From 4a45b4edac6976ecf7d1b6b018a077ce64d61400 Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Mon, 16 Sep 2024 09:20:29 +0000
Subject: [PATCH 01/78] Update to new names and add history api
---
.env.example | 8 ++++----
balance_tracker/rewrite_rules.conf | 26 ++++----------------------
hafah_rest/rewrite_rules.conf | 2 +-
hivemind_rtracker/rewrite_rules.conf | 4 ++--
reputation_tracker/rewrite_rules.conf | 4 ++--
5 files changed, 13 insertions(+), 31 deletions(-)
diff --git a/.env.example b/.env.example
index 3c94b6f..1177019 100644
--- a/.env.example
+++ b/.env.example
@@ -118,10 +118,10 @@ HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
HAF_VERSION=f2cec24d
-HAFAH_VERSION=5a07a805
-BALANCE_TRACKER_VERSION=b51cb031
-HAF_BLOCK_EXPLORER_VERSION=9e5a1bf2
-REPUTATION_TRACKER_VERSION=6dfbe231
+HAFAH_VERSION=34991926
+BALANCE_TRACKER_VERSION=0ac33077
+HAF_BLOCK_EXPLORER_VERSION=9b7b96a8
+REPUTATION_TRACKER_VERSION=81a18138
HIVEMIND_INSTANCE_VERSION=fa851ba1
diff --git a/balance_tracker/rewrite_rules.conf b/balance_tracker/rewrite_rules.conf
index 9aebbd4..ff134bf 100644
--- a/balance_tracker/rewrite_rules.conf
+++ b/balance_tracker/rewrite_rules.conf
@@ -1,26 +1,8 @@
-rewrite ^/account-balances/([^/]+)/rewards/info /rpc/get_account_info_rewards?account-name=$1 break;
-# endpoint for get /account-balances/{account-name}/rewards/info
+rewrite ^/accounts/([^/]+)/balance-history /rpc/get_balance_history?account-name=$1 break;
+# endpoint for get /accounts/{account-name}/balance-history
-rewrite ^/account-balances/([^/]+)/rewards /rpc/get_account_rewards?account-name=$1 break;
-# endpoint for get /account-balances/{account-name}/rewards
-
-rewrite ^/account-balances/([^/]+)/withdrawals /rpc/get_account_withdraws?account-name=$1 break;
-# endpoint for get /account-balances/{account-name}/withdrawals
-
-rewrite ^/account-balances/([^/]+)/savings /rpc/get_account_savings?account-name=$1 break;
-# endpoint for get /account-balances/{account-name}/savings
-
-rewrite ^/account-balances/([^/]+)/delegations /rpc/get_account_delegations?account-name=$1 break;
-# endpoint for get /account-balances/{account-name}/delegations
-
-rewrite ^/account-balances/([^/]+) /rpc/get_account_balances?account-name=$1 break;
-# endpoint for get /account-balances/{account-name}
-
-rewrite ^/balance-for-coins/([^/]+)/by-time /rpc/get_balance_for_coin_by_time?account-name=$1 break;
-# endpoint for get /balance-for-coins/{account-name}/by-time
-
-rewrite ^/balance-for-coins/([^/]+) /rpc/get_balance_for_coin_by_block?account-name=$1 break;
-# endpoint for get /balance-for-coins/{account-name}
+rewrite ^/accounts/([^/]+)/balances /rpc/get_account_balances?account-name=$1 break;
+# endpoint for get /accounts/{account-name}/balances
rewrite ^/$ / break;
# endpoint for openapi spec itself
diff --git a/hafah_rest/rewrite_rules.conf b/hafah_rest/rewrite_rules.conf
index e431f93..d34d5f1 100644
--- a/hafah_rest/rewrite_rules.conf
+++ b/hafah_rest/rewrite_rules.conf
@@ -1,7 +1,7 @@
rewrite ^/block-number-by-date/([^/]+) /rpc/get_block_by_time?timestamp=$1 break;
# endpoint for get /block-number-by-date/{timestamp}
-rewrite ^/global-state /rpc/get_block break;
+rewrite ^/global-state /rpc/get_global_state break;
# endpoint for get /global-state
rewrite ^/headblock /rpc/get_head_block_num break;
diff --git a/hivemind_rtracker/rewrite_rules.conf b/hivemind_rtracker/rewrite_rules.conf
index d93aa9c..da7bfd8 100644
--- a/hivemind_rtracker/rewrite_rules.conf
+++ b/hivemind_rtracker/rewrite_rules.conf
@@ -1,5 +1,5 @@
-rewrite ^/reputation/([^/]+) /rpc/get_account_reputation?account-name=$1 break;
-# endpoint for get /reptracker/reputation/{account-name}
+rewrite ^/accounts/([^/]+)/reputation /rpc/get_account_reputation?account-name=$1 break;
+# endpoint for get /accounts/{account-name}/reputation
rewrite ^/$ / break;
# endpoint for openapi spec itself
diff --git a/reputation_tracker/rewrite_rules.conf b/reputation_tracker/rewrite_rules.conf
index d93aa9c..da7bfd8 100644
--- a/reputation_tracker/rewrite_rules.conf
+++ b/reputation_tracker/rewrite_rules.conf
@@ -1,5 +1,5 @@
-rewrite ^/reputation/([^/]+) /rpc/get_account_reputation?account-name=$1 break;
-# endpoint for get /reptracker/reputation/{account-name}
+rewrite ^/accounts/([^/]+)/reputation /rpc/get_account_reputation?account-name=$1 break;
+# endpoint for get /accounts/{account-name}/reputation
rewrite ^/$ / break;
# endpoint for openapi spec itself
--
GitLab
From ac190b4a92bf41efda120e379bb72c9703e67bd5 Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Mon, 16 Sep 2024 09:58:53 +0000
Subject: [PATCH 02/78] Use rc versioning in example script
---
.env.example | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/.env.example b/.env.example
index 1177019..e6d451e 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ HIVE_API_NODE_REGISTRY=registry.gitlab.syncad.com/hive
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.5
+HIVE_API_NODE_VERSION=1.27.6rc1
# Global settings
@@ -118,11 +118,6 @@ HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
HAF_VERSION=f2cec24d
-HAFAH_VERSION=34991926
-BALANCE_TRACKER_VERSION=0ac33077
-HAF_BLOCK_EXPLORER_VERSION=9b7b96a8
-REPUTATION_TRACKER_VERSION=81a18138
-
HIVEMIND_INSTANCE_VERSION=fa851ba1
# REPTRACKER_SCHEMA="hafbe-rt"
--
GitLab
From 56051bea90c9fb80fa5dd4388c862392d934b3f6 Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Mon, 23 Sep 2024 12:04:05 +0000
Subject: [PATCH 03/78] Share rewrite rules between hafbe's reptracker and
hivemind's
---
hivemind_rtracker/rewrite_rules.conf | 8 --------
.../hivemind_rtracker_nginx.conf | 0
2 files changed, 8 deletions(-)
delete mode 100644 hivemind_rtracker/rewrite_rules.conf
rename {hivemind_rtracker => reputation_tracker}/hivemind_rtracker_nginx.conf (100%)
diff --git a/hivemind_rtracker/rewrite_rules.conf b/hivemind_rtracker/rewrite_rules.conf
deleted file mode 100644
index da7bfd8..0000000
--- a/hivemind_rtracker/rewrite_rules.conf
+++ /dev/null
@@ -1,8 +0,0 @@
-rewrite ^/accounts/([^/]+)/reputation /rpc/get_account_reputation?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/reputation
-
-rewrite ^/$ / break;
-# endpoint for openapi spec itself
-
-rewrite ^/(.*)$ /rpc/$1 break;
-# default endpoint for everything else
diff --git a/hivemind_rtracker/hivemind_rtracker_nginx.conf b/reputation_tracker/hivemind_rtracker_nginx.conf
similarity index 100%
rename from hivemind_rtracker/hivemind_rtracker_nginx.conf
rename to reputation_tracker/hivemind_rtracker_nginx.conf
--
GitLab
From e4c31d645066532a7aada5797b81e6cc13a66ae5 Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Mon, 23 Sep 2024 12:04:49 +0000
Subject: [PATCH 04/78] Use "apps" as universal profile for all haf apps
---
.env.example | 7 +++++--
.env.swp | Bin 0 -> 16384 bytes
hafah.yaml | 2 ++
hivemind_reptracker.yaml | 8 ++++++--
swagger.yaml | 1 +
5 files changed, 14 insertions(+), 4 deletions(-)
create mode 100644 .env.swp
diff --git a/.env.example b/.env.example
index e6d451e..527eaab 100644
--- a/.env.example
+++ b/.env.example
@@ -20,7 +20,7 @@ TOP_LEVEL_DATASET_MOUNTPOINT="${ZPOOL_MOUNT_POINT}/${TOP_LEVEL_DATASET}"
# - monitoring: services for Prometheus, Grafana, Loki, Cadvisor , Nodeexporter, Promtail, Postresexporter, Blackboxexporter...
# COMPOSE_PROFILES="core,admin,hafah,hivemind,servers"
# COMPOSE_PROFILES="core,admin,hafah,hafbe,hivemind,servers,monitoring"
-COMPOSE_PROFILES="core,admin,hafah,hafbe,servers"
+COMPOSE_PROFILES="core,admin,apps,servers"
# The registry where Hive docker images are pulled from. Normally, you
# should set this to the default, `registry.hive.blog` or Docker Hub,
@@ -117,8 +117,11 @@ HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
#HAF_REPUTATION_TRACKER_VERSION=f9f74604
HAF_VERSION=f2cec24d
+#HAF_VERSION=v1.27.6rc2
-HIVEMIND_INSTANCE_VERSION=fa851ba1
+HAF_BLOCK_EXPLORER_VERSION=1.27.6rc2
+
+HIVEMIND_INSTANCE_VERSION=f7df13df
# REPTRACKER_SCHEMA="hafbe-rt"
# BTRACKER_SCHEMA="hafbe-bt"
diff --git a/.env.swp b/.env.swp
new file mode 100644
index 0000000000000000000000000000000000000000..cdefd5914f833ea85a4ad4b703e0aff0ffb9286c
GIT binary patch
literal 16384
zcmeI3U5q2yRmW@DK-jQZ5D^F|FPEN?W+w7Sx4UO&rjfI_yWQQ+y6sNeGn<J(p0exO
zuI+MFZGH5I!ipCpLY^Q}@&W_|h^#~&K!PB6Nq7OoBHrPJ2ZDl-K)?$kAcfz#RpsvX
z^h~k|5+K={|G4e@b?&+6p7TG)_Hgx4>p<PF)GQu9WLY2n=)aENZT;B`$3OOhHMPeB
z+nKq(v_0pszNYWj?$$OQtPY3I>1I_=gXlO8?UAlB_0ar3utRs?2ac{*w14uv(aW}_
z29_H5?lka0YrVF%Ds$Xe`Em83AAb1VO}8Aq)WA{$OARbFu++d(14|7oHL%pcQUm|5
zG?0WZT0e`#-;+yykUzh$@chmEahSi~S!jPTf4!AIe_^3Lf4O{GYGA2>r3RK7SZZLY
zfu#nP8dz#zsez>imKs=UV5x!s1r6AaWwlxNS0v%*^Zz3M|EnLgtS^B#!A<bBS1ju*
z;Mc$xz+0dPJ^)?<U;UtEC14E7;GbTmFW3OT{{xowHn<DE@d3;F5Ab{77_5N5e#x@F
z3_b&TU>(%JzkI)C{VsS1d>Z^bNI?SD!8hJ-S$_xq61)pO4}Kba2>ki?S=Je-fNy@U
zWqlER9NY$9d!J?fF*pOKpbUN#`~^jfp93jyzzF;V_!01*6iEIC{08`Sa0*iJhv@AO
zfavd+MTh@`j~nX1_8mJ3qH`65i5vK_vLn4|-LP(`#?juxgJ!4StFM&Hkq$lkygc-R
z(eY?%yS^%y<0J@+20O{x)ZJC{pjn@}ys5`Qq|5QSKjJMjoN;y%%fKK?%0ZfhX;SZW
z`>ox_*LO+ILtWo}*lyG82Z=j&N4Al*tbHd8TtBH#-4pGU!<pWEuy+6Mijlv0X3s)T
zt7&kml0c=gR<po&*{thNRIH;Dccf#wetR2T+#!oRf8b(=_}C50M$p9;C!K}m*tf%Y
z8YK1FYHfY>er=6js_n*!3dS7DB%NtLi52Qp$yBTT#%{3(%jz`yZ*-4dA9NZA%{t$F
zVPxvNGf}YZCly{!P>_t9DaRd;b;MlC*NJ>zM)~$k$D*QRt;4L78wZ}9Xs4o7mr>EF
z8*66iv#^5&=;+Gel5)rDJV;fP`ieP?aIVie`)F7Nyve|rWsJx=Dp`j|-H$c5`*VrQ
zNa8tHmly@?fKv;j;7vV3UM481iAD=yq{r@=c9<KvwLO}u5r1NrA{zZH3k3~&`v-%a
z))5PJj~>@oZhorYJsh-~kDBelPNU!GHT#2u?!!+1u-oeN->$|}jNci|^o$E3>c3`=
z>da5%%SD5W&d=!io>zO4tG3v88mhAML=Gm>=rT$~9y!dhi&^uw(duQHIot?MA$>_{
z$4%(rVlsJmnT5-!xBwH^w>@=Yd#R2~3Ln6%jd6nV%$#ytr?!mr-FT{<5~r>f#dCsF
zAL*vr=U!#jr7o0$43!~Y?7Bf?im}<gvO`WXj9d&$C9`mxNfw{S31-PWk(SN%16fQ2
zFL9<bv3iA*FoJoUuNrEP8;#7^c+o+m4rsGw55<OzXkwNl7B?PfY_nFRhUbbUxgB=o
zdgpisaR~Hr{c?8pPwzt{Yh_06YwQl~&+IssTV-aJ>v-F(=!&@qn32L$6*H?(?)Fr1
zzweZcX};0G2ERkoH1>CDdn;z9c6aZ+%;W`=I2$Ne;6037;X6I%Mm%aB^;+Fdy|(_U
zGrWIayp3Fx;_Y?&-p1Wm$2B9|QFE`=>vQ>Termt<s5xjHwg#Q<PIFK+ye*rl4((VE
z3Xj2j*i#g_j&WHsIhKG*-I+bnw_-(OOsq0C5a|g%lK5pBl~~cH2)gG5Lrgr@Ny6#I
z_=79U7s9SpYWFIyMx!-`7{lV3F^aom3{t7bxdB}fDX}M+D|UUyJ#n3srHlZ?6R}_k
zHe!>;=h1nB$rMtHJNG=(yf5~HN-!4onoUqLYQ7?Dp(_*S++blV^VUKYm5G~p_OK#`
zX*(6}NL9iPl3mro(*vbjg&;i<ZKST}7&@TrGzn&e+!1o0t1uOpAQvJRSV0K<iEJ2W
z!^L>=Ed+U@PDQtB7$j4*-D0BbH9l^LW6Ne$r!Gnw;{DU1XPgkRD8GTH1U2Ip2{%Ln
zsD!66ASNP!rk-O?u#iTM!O>$o^^(#h(dJuyf<II{Sy0<ghb3m|Go;u<E}J>2+&||6
z$|Vm|&*NyvQ7}WiP7uvRMI~_pmbKkIIPCVC17h87tKEc0i4~X37hX$Py*-uxhq@%P
z@FiHMB%}n}mVojd4=!DhoNsrYYnCrx;!2w)Y71F1?BkHlYP6c-p!XuS;oBwE4vyWD
z+P0k&s6(Vm=)%$ZOc;nbWEn$r_V_E{w(9tzY0I-m$HU;Pc&Abk(<+-a(Z#~z5ut$l
zTpbbe#cRvu5woP<)F!-~q5L;f9J9Lpv0k@Bo9TC_M6in_n5o-G&0c^0zFCUE#CYRk
z#Mfjm=ZC_?a#Zq!Io`&r2(2k65pokuuz6d>>5x%(vf1Pd&70x^#wpi42)s;h=G?_s
zag&gol{Qs4$*h4txDtUGFLG=e_rN^Znr9qf9wx<kC>Xmf9@98LkyFBVVjmNU#S)?K
zb2hdZwA1j3(#c5fQi3P7O^CJ2u?}tI!j!pdhH;`x;SJ)?7EMu71S4P3SCm%pX|r;1
zL4}h=fhTv!x2)%pFCk~2JnVMc86g|wjO-c?_4bPORLA;CH6KnZs+eM6#ifrq3EVS|
z%L)8j!g|@I@mQR~HA2-_IOwvd7&0ZZg`A*JhGL9*Y9Dlp1G8~q8dLiOnyM?4N=eP|
zI<6!kq9PODJb1Oqvo5h>#!LIDkuTD=Q$3AbNRsWJ<HfKQj6RlV$m!oSo3Gw{N{GU7
z3Hz0f#_zEUiDs;ojVrvFtS0_7BZMxDS~i?17G+-ZHTtrsLDV76@$wpm+rTRc&6%ns
zLy_$e3Ah86<o~})-ufrxvXcLA<@xIu$@#nB-^uO&8GH`B2EI;i{|@MZDtH<E13CWJ
z!Mnf%Yv6t0Meq(e|1X03;3L2Sze&#jSzv=c*Z?nq-y!G!EZ7DAMSlNB;Pc>f;5G2~
z<n_M<w!w#i<oG|6=lFk1UjHTV8So3>9{4wM`riUS1zO-E;0|~NybS)4y#7zYZ-Y+*
z9~^=Tcm;?~zAAcIKK}1&APY>&u;pI&;n8-pzCvD_ObBCdd7IU0@!}_O9GiDH)@$>3
zUW;RIMj)&^ZY<np#mXn!YB#oD-|Bv1PMS|vE47WvYSAu4ZD(+J(42SJcwpaKU#mGq
z4@kv9ztL*XJJc!<DjP*>yZd_U`j)+;#%`n2m^ZGiR@N))#klRp&ZAbZdo*uGu8n*A
zb|;tHQ1|914!cMF=A}++m3tSmi)79Ie)D1PVwPHE<3R<EjsG~fsKM}i90|*=s0M~k
z3c7R9>I|Bl9pnCmIQ6^R-FAJNBq7ZvR;r@t6AMpBL!lG)Ou{fB3o0)Dve#=H4rVgH
z1$Ig(ko0i`oiS_`Kgm4%!KsFN6_t!z@N$CJ*qx+Q1jMyRIUy0Z7WhOH!_|~XX7WuU
zfUqKoDdHm$hkrzcWNIh#&=8U`;#;XyML{AZ4Hlo_iQ&Qf{r;gMs6(>TWGY*OOgZP7
zn>GuI$YBC8eT$?WkDUjQs?kdILM<kFGJ?etwsP)U!4tN7gI=>uS){es!Mu8%*5P5Z
zU#~j)r0T<DtsBqkLLT9qS7Yss%dtE0q5rY(hM`W76NONdQBVmT<;6MTpXBFe?<UTS
zvKqjI0uA({O$n#zkD~KXIEe5LXvd5wB-<Wpl;|+sm|6x&t|$Dw)99Dv`S9UZyR|*o
z@Ampq>k=K$-1y?8IKL!Q*%-FTY}kf(S+I=3F3cjN>9SB+reP9@jFGbrbG^|x>a;q0
zn?<!NztkquUQ*5F<qRb*PSH-J>R>V#qcEm46boWt=E#+tDVB&_)N-VsNNA4?apX)*
zNkt5OesnU2gmJ30BeaiQksdqdG8p5fR+Aevvt2r<Z01V`B3dRd%SwJ1mxwg$L@F_~
zVEJ=1xR}!vnM?*<f@HW9<Pr@n&#l<l2(>vmYS{AS2`-H04tno%yM|K7qnn%^(r(Rd
zn3?r0xaY-r=OklcPOkrzJ>EuTu0QfpqYt(~6uxQ#^%V-+wpSUBD&w~tzY-d%qj)^k
zQPf$P;{SyNNpfwJ6jhAU%JT+_tHPGbB4Oq#xUX|MSac(WDyX-pHunWND>rw@{F+f1
z{k6h~`ym2dBa^hm*ENY$3WJVQF(A=$W@vY0k*nfr91|IiaZN(%a8bRi%2i@m90qd7
zsrHKNn!BaKm`TaSopRqHv=Gv4*{p<C<Ojths0OY}r9RW@x7cmA0|XM2e9x-MMx!wk
z9AKym`_8I?#^aie-Og*<LhF@J;q*+d$lTj2?U8Fx0C6Zzn#hCUM`Uh>=2Pr@#>qW(
zWSLxr3KoQtG&d8>90C_XV#d5KGe5-aIvcsmAI%QT<ipAQ0|4X^$HrCb5wMibJ|u}{
zy*_ulSYJZcEqpH-FGYP{wszSc2EN=?Q^k>FFqYVfqdm3H<9V=~-%^R}5R0A1b3Ram
zFKm=;FQlGZmaUXWUXVItkICkpTL+E3=JV@@7r*T_zLTN(VHGu6`9Ol(l#=H_5u|*m
zZcBcWZyQIM)#@7C1e2XkPm(+;I#jb_N(OPLNA4Kkkre}(ewVR@K8R4nOj0Q70H);F
zd)(O`Y#ntQJKK$3f3O9`gUSr>Ms+_ZLj{@O8GVFM)$gx9kTT>my7Zby6fN^lces~%
z^**awW-$-yfGtYdZyE)DkP`o+z;;Gdy$#jjFpZ(YXz++)flA`Syn<lTW>ZZOy-;VE
ze^iu(K|(O+T-0S*YEIt7d=`px*-GNo=$K8KS+H5xxZ7-HQ~Wg1ofb?etL<@0r*<TT
zjm(ZDaLXHPB+KsII7<1*QivKyX7*=i#llhMa3iRXBOevAUA7}C3)4tGQ4*TbXM-n!
zOYJU*Mq=J#!nh#$MSA{&aXu22h{cgJJhP-&BaM^GT_{Cn&RLd-E7HgckUy-EMoI_f
zV3>`?W>P&8t_ta$;r}r%<81i2jsOp>V?-pGa|E}sk43agC(690AUZOYuS^0aIi(<T
z9_9twLe#9JlM8E#53({MT%ItKSKO3PEn^B7MKvz@u54d8EYUa)d@jXx=9Qm;_yR}6
M6^zyy-Sw^i1`zqZxBvhE
literal 0
HcmV?d00001
diff --git a/hafah.yaml b/hafah.yaml
index a24ad73..c46a88a 100644
--- a/hafah.yaml
+++ b/hafah.yaml
@@ -34,6 +34,7 @@ services:
profiles:
- apps
- hafah
+ - hafbe
environment:
PGRST_ADMIN_SERVER_PORT: 3001
PGRST_DB_URI: postgresql://hafah_user@haf/haf_block_log
@@ -61,6 +62,7 @@ services:
profiles:
- apps
- hafah
+ - hafbe
volumes:
- type: bind
source: hafah_rest/hafah_rest_nginx.conf
diff --git a/hivemind_reptracker.yaml b/hivemind_reptracker.yaml
index 6025b86..0f8a734 100644
--- a/hivemind_reptracker.yaml
+++ b/hivemind_reptracker.yaml
@@ -3,6 +3,7 @@ services:
image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- hivemind
+ - apps
environment:
POSTGRES_HOST: haf
networks:
@@ -31,12 +32,14 @@ services:
image: ${POSTGREST_REGISTRY:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
- hivemind
+ - apps
environment:
PGRST_ADMIN_SERVER_PORT: 3001
PGRST_DB_URI: postgresql://reptracker_user@haf/haf_block_log
PGRST_DB_SCHEMA: reptracker_endpoints
PGRST_DB_ANON_ROLE: reptracker_user
PGRST_DB_POOL: 20
+ PGRST_DB_ROOT_SPEC: root
PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
PGRST_DB_EXTRA_SEARCH_PATH: reptracker_app
# PGRST_OPENAPI_SERVER_PROXY_URI: http://${PUBLIC_HOSTNAME}/reptracker_user/
@@ -57,13 +60,14 @@ services:
image: nginx
profiles:
- hivemind
+ - apps
volumes:
- type: bind
- source: hivemind_rtracker/hivemind_rtracker_nginx.conf
+ source: reputation_tracker/hivemind_rtracker_nginx.conf
target: /etc/nginx/nginx.conf
read_only: true
- type: bind
- source: hivemind_rtracker/rewrite_rules.conf
+ source: reputation_tracker/rewrite_rules.conf
target: /etc/nginx/rewrite_rules.conf
read_only: true
depends_on:
diff --git a/swagger.yaml b/swagger.yaml
index 215d0b0..f679823 100644
--- a/swagger.yaml
+++ b/swagger.yaml
@@ -3,6 +3,7 @@ services:
image: swaggerapi/swagger-ui
profiles:
- hafbe
+ - apps
environment:
URLS: >-
[{url: "https://${PUBLIC_HOSTNAME}/hafbe-api/", name: "HAF Block Explorer"},
--
GitLab
From 0a5e6f02a153aad6f63e6912e1bed9e77f51cbd4 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 20 Sep 2024 16:03:43 +0000
Subject: [PATCH 05/78] Remove hard-coded revisions, and comment out as much as
possible in the example environment file.
---
.env.example | 77 ++++++++++++++++++-----------------------
caddy.yaml | 4 +--
clear_pool.sh | 6 ++--
drone.yaml | 2 +-
haf_base.yaml | 18 +++++-----
hafah.yaml | 6 ++--
jussi.yaml | 2 +-
monitoring.yaml | 18 +++++-----
reputation_tracker.yaml | 1 +
9 files changed, 63 insertions(+), 71 deletions(-)
diff --git a/.env.example b/.env.example
index 527eaab..b40bee0 100644
--- a/.env.example
+++ b/.env.example
@@ -20,7 +20,7 @@ TOP_LEVEL_DATASET_MOUNTPOINT="${ZPOOL_MOUNT_POINT}/${TOP_LEVEL_DATASET}"
# - monitoring: services for Prometheus, Grafana, Loki, Cadvisor , Nodeexporter, Promtail, Postresexporter, Blackboxexporter...
# COMPOSE_PROFILES="core,admin,hafah,hivemind,servers"
# COMPOSE_PROFILES="core,admin,hafah,hafbe,hivemind,servers,monitoring"
-COMPOSE_PROFILES="core,admin,apps,servers"
+COMPOSE_PROFILES="core,admin,servers,apps"
# The registry where Hive docker images are pulled from. Normally, you
# should set this to the default, `registry.hive.blog` or Docker Hub,
@@ -28,8 +28,8 @@ COMPOSE_PROFILES="core,admin,apps,servers"
# images, change this to `registry.gitlab.syncad.com/hive` where both CI
# builds are automatically pushed.
# HIVE_API_NODE_REGISTRY=registry.hive.blog
-HIVE_API_NODE_REGISTRY=registry.gitlab.syncad.com/hive
# HIVE_API_NODE_REGISTRY=hiveio
+# HIVE_API_NODE_REGISTRY=registry.gitlab.syncad.com/hive
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
@@ -39,13 +39,12 @@ HIVE_API_NODE_VERSION=1.27.6rc1
# Global settings
# override the HAF core image's version and registry image here:
-HAF_REGISTRY=${HIVE_API_NODE_REGISTRY}/haf/base_instance
-# HAF_VERSION=3a7456f2
-# HAF_VERSION=236db88f
+# HAF_REGISTRY=${HIVE_API_NODE_REGISTRY}/haf
+# HAF_VERSION=${HIVE_API_NODE_VERSION}
-HAF_DATA_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}"
-HAF_LOG_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/logs"
-HAF_WAL_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory/haf_wal"
+# HAF_DATA_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}"
+# HAF_LOG_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/logs"
+# HAF_WAL_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory/haf_wal"
# If you need to massive sync HAF (i.e. you are not using a ZFS snapshot),
# then you can sync faster by temporarily using an in-memory shared_memory.bin.
# To do this, comment out the line below and uncomment the one after, and
@@ -55,7 +54,6 @@ HAF_WAL_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory/haf_wal"
# `docker compose up -d` to restart HAF.
# HAF_SHM_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory"
# HAF_SHM_DIRECTORY="/mnt/haf_shared_mem"
-HAF_SHM_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory"
# The docker compose project name, gets prefixed onto each container name
PROJECT_NAME=haf-world
@@ -65,10 +63,10 @@ PROJECT_NAME=haf-world
NETWORK_NAME=haf
# List of arguments for the HAF service
-#ARGUMENTS="--dump-snapshot=20230821"
-#ARGUMENTS="--skip-hived"
-#ARGUMENTS="--replay-blockchain"
-ARGUMENTS="--replay-blockchain --stop-replay-at-block 5000000"
+# ARGUMENTS=""
+# ARGUMENTS="--replay-blockchain"
+# ARGUMENTS="--dump-snapshot=20230821"
+# ARGUMENTS="--skip-hived"
#
# Example how to use monitoring services
#
@@ -89,48 +87,39 @@ ARGUMENTS="--replay-blockchain --stop-replay-at-block 5000000"
# Additional logs are collected from all containers in the stack via Loki and Promtail
# Default login and password for Grafana is admin/admin - remember to change it after first login
# Statistics provided by Grafana are available at the host address on port 3000 (http(s)://hostname:3000)
-#
+
# The default setup will run the recommended version of HAfAH,
# you can run a custom version by un-commenting and modifying the
# values below
-HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah/setup
-# HAFAH_VERSION=9556cd0f
-# HAFAH_VERSION=a85fc0b4
+# HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah
+# HAFAH_VERSION=${HIVE_API_NODE_VERSION}
# The default setup will run the recommended version of Hivemind using the values
# below. You can override them here to run a custom version of Hivemind
-HIVEMIND_INSTANCE_IMAGE=${HIVE_API_NODE_REGISTRY}/hivemind/instance
-# HIVEMIND_INSTANCE_VERSION=f09bd298
+# HIVEMIND_INSTANCE_IMAGE=${HIVE_API_NODE_REGISTRY}/hivemind
+# HIVEMIND_INSTANCE_VERSION=${HIVE_API_NODE_VERSION}
# The default setup will run the recommended version of balance tracker,
# you can run a custom version by un-commenting and modifying the
# values below
-BALANCE_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/balance_tracker
-# BALANCE_TRACKER_VERSION=66452803
-# BALANCE_TRACKER_VERSION=c9906c48
+# BALANCE_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/balance_tracker
+# BALANCE_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
# REPUTATION_TRACKER_ADDON
-HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
-#HAF_REPUTATION_TRACKER_VERSION=f9f74604
-
-HAF_VERSION=f2cec24d
-#HAF_VERSION=v1.27.6rc2
-
-HAF_BLOCK_EXPLORER_VERSION=1.27.6rc2
+# HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
+# HAF_REPUTATION_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
-HIVEMIND_INSTANCE_VERSION=f7df13df
# REPTRACKER_SCHEMA="hafbe-rt"
# BTRACKER_SCHEMA="hafbe-bt"
# The default setup will run the recommended version of HAF block explorer,
# you can run a custom version by un-commenting and modifying the
# values below
-HAF_BLOCK_EXPLORER_REGISTRY=${HIVE_API_NODE_REGISTRY}/haf_block_explorer
-# HAF_BLOCK_EXPLORER_VERSION=b268faed
-# HAF_BLOCK_EXPLORER_VERSION=354960a1
+# HAF_BLOCK_EXPLORER_REGISTRY=${HIVE_API_NODE_REGISTRY}/haf_block_explorer
+# HAF_BLOCK_EXPLORER_VERSION=${HIVE_API_NODE_VERSION}
# The default setup uses "Jussi" as the API reverse proxy & cache for the old JSON-RPC-style
@@ -173,7 +162,7 @@ HAF_BLOCK_EXPLORER_REGISTRY=${HIVE_API_NODE_REGISTRY}/haf_block_explorer
# For running a full stack:
# if you need to run a custom image (for example, to use ACME DNS challenges), specify it here
# CADDY_IMAGE=${HIVE_API_NODE_REGISTRY}/haf_api_node/caddy
-# CADDY_VERSION=2.7.4-alpine-with-cloudflare
+# CADDY_VERSION=latest
# The hostname you'll be running this server on. There are several ways you can configure
# this. Some examples:
@@ -202,7 +191,7 @@ PUBLIC_HOSTNAME="your.hostname.com"
# real certificate for PUBLIC_HOSTNAME from LetsEncrypt. If this server is
# behind a firewall or NAT, or PUBLIC_HOSTNAME is misconfigured, it will fail
# to get a certificate, and that will count against LetsEncrypt's rate limits.
-#TLS_SELF_SIGNED_SNIPPET=caddy/self-signed.snippet
+# TLS_SELF_SIGNED_SNIPPET=caddy/self-signed.snippet
TLS_SELF_SIGNED_SNIPPET=/dev/null
# Caddy will only accept requests on the /admin/ endpoints over https by default.
@@ -214,12 +203,12 @@ TLS_SELF_SIGNED_SNIPPET=/dev/null
# Monitoring env variables
#
-export PROMETHEUS_VERSION=v2.49.1
-export NODE_EXPORTER_VERSION=v1.7.0
-export CADVISOR_VERSION=v0.47.2
-export GRAFANA_VERSION=10.3.3
-export LOKI_VERSION=2.9.4
-export PROMTAIL_VERSION=2.9.4
-export HIVED_PME_VERSION=49a7312d
-export BLACKBOX_VERSION=v0.24.0
-export DATA_SOURCE="postgresql://postgres@haf:5432/postgres?sslmode=disable"
+# PROMETHEUS_VERSION=v2.49.1
+# NODE_EXPORTER_VERSION=v1.7.0
+# CADVISOR_VERSION=v0.47.2
+# GRAFANA_VERSION=10.3.3
+# LOKI_VERSION=2.9.4
+# PROMTAIL_VERSION=2.9.4
+# HIVED_PME_VERSION=49a7312d
+# BLACKBOX_VERSION=v0.24.0
+# DATA_SOURCE="postgresql://postgres@haf:5432/postgres?sslmode=disable"
diff --git a/caddy.yaml b/caddy.yaml
index a56f2b7..d130513 100644
--- a/caddy.yaml
+++ b/caddy.yaml
@@ -26,11 +26,11 @@ services:
target: /etc/caddy/admin_html
read_only: true
- type: bind
- source: ${HAF_DATA_DIRECTORY}/logs/pgbadger
+ source: ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/logs/pgbadger
target: /etc/caddy/pgbadger
read_only: true
- type: bind
- source: ${HAF_DATA_DIRECTORY}/logs/caddy
+ source: ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/logs/caddy
target: /var/log/caddy
read_only: false
- type: bind
diff --git a/clear_pool.sh b/clear_pool.sh
index 90182c9..1f75823 100755
--- a/clear_pool.sh
+++ b/clear_pool.sh
@@ -1,5 +1,5 @@
#brain dead script that needs improvement, but useful for me
. ./.env
-sudo rm -rf ${HAF_LOG_DIRECTORY}/postgresql/*
-sudo rm -rf ${HAF_DATA_DIRECTORY}/haf_db_store/*
-rm -rf ${HAF_SHM_DIRECTORY}/shared_memory.bin ${HAF_SHM_DIRECTORY}/haf_wal/*
+sudo rm -rf ${HAF_LOG_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/logs}/postgresql/*
+sudo rm -rf ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/haf_db_store/*
+rm -rf ${HAF_SHM_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory}/shared_memory.bin ${HAF_SHM_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory}/haf_wal/*
diff --git a/drone.yaml b/drone.yaml
index 5009222..4c7f6da 100644
--- a/drone.yaml
+++ b/drone.yaml
@@ -28,7 +28,7 @@ services:
# target: /etc/nginx/nginx.conf
# read_only: true
# - type: bind
- # source: ${HAF_LOG_DIRECTORY}/drone
+ # source: ${HAF_LOG_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/logs}/drone
# target: /var/log/drone
# depends_on:
# drone-real:
diff --git a/haf_base.yaml b/haf_base.yaml
index 5c9ef7d..bff2ebd 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -11,7 +11,7 @@ services:
init: true
entrypoint:
- /home/haf_admin/docker_entrypoint.sh
- - ${ARGUMENTS}
+ - ${ARGUMENTS:-}
# after requesting a shutdown with SIGTERM, allow the container two minutes to exit
# before killing it. The default of ten seconds isn't enough for postgresql to
# cleanly shut down, and would often make PostgreSQL perform crash recovery at the
@@ -82,6 +82,8 @@ services:
# 5 minutes is probably long enough that anyone using NVMes will be able to recover their database at
# startup. If you're on slower drives or older hardware, you may need to increase this.
PGCTLTIMEOUT: 300
+ # HACK, do not commit
+ OVERRIDE_LD_PRELOAD: ""
# when mounting volumes, declare the directories you want to bind-mount here,
# using either the short or long syntax. Don't declare them as top-level named
# volumes and mount them by name. Declaring them inline here always results
@@ -89,16 +91,16 @@ services:
# seems to vary from system to system (even when `o: rbind` is specified).
volumes:
- type: bind
- source: ${HAF_DATA_DIRECTORY}/
+ source: ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/
target: /home/hived/datadir
- type: bind
- source: ${HAF_LOG_DIRECTORY}/postgresql
+ source: ${HAF_LOG_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/logs}/postgresql
target: /home/hived/postgresql_logs/
- type: bind
- source: ${HAF_SHM_DIRECTORY}/
+ source: ${HAF_SHM_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory}/
target: /home/hived/shm_dir
- type: bind
- source: ${HAF_WAL_DIRECTORY:-${HAF_SHM_DIRECTORY}/haf_wal}
+ source: ${HAF_WAL_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory/haf_wal}
target: /home/hived/wal_dir
shm_size: 4gb
healthcheck:
@@ -114,7 +116,7 @@ services:
init: true
volumes:
- type: bind
- source: ${HAF_LOG_DIRECTORY}/postgresql
+ source: ${HAF_LOG_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/logs}/postgresql
target: /var/log/postgresql
- type: bind
source: ./logrotate/logrotate.d
@@ -128,10 +130,10 @@ services:
command: infinity
volumes:
- type: bind
- source: ${HAF_DATA_DIRECTORY}/logs/pgbadger
+ source: ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/logs/pgbadger
target: /pgbadger_output
- type: bind
- source: ${HAF_LOG_DIRECTORY}/postgresql
+ source: ${HAF_LOG_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/logs}/postgresql
target: /postgresql_logs
read_only: true
labels:
diff --git a/hafah.yaml b/hafah.yaml
index c46a88a..55f9041 100644
--- a/hafah.yaml
+++ b/hafah.yaml
@@ -77,7 +77,7 @@ services:
condition: service_healthy
networks:
- haf-network
- hafah-benchmarks-old-style: #NOTE: need to first run mkdir -p ${HAF_DATA_DIRECTORY}/tests/hafah_api_benchmarks/old-style
+ hafah-benchmarks-old-style: #NOTE: need to first run mkdir -p ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/tests/hafah_api_benchmarks/old-style
image: registry.gitlab.syncad.com/hive/tests_api/benchmark_aio:latest
profiles:
- hafah-benchmarks
@@ -91,7 +91,7 @@ services:
WDIR: /workspace
volumes:
- type: bind
- source: ${HAF_DATA_DIRECTORY}/tests/hafah_api_benchmarks/old-style
+ source: ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/tests/hafah_api_benchmarks/old-style
target: /workspace
networks:
haf-network:
@@ -116,7 +116,7 @@ services:
# WDIR: /workspace
# volumes:
# - type: bind
- # source: ${HAF_DATA_DIRECTORY}/tests/hafah_api_benchmarks/new-style
+ # source: ${HAF_DATA_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}}/tests/hafah_api_benchmarks/new-style
# target: /workspace
# networks:
# haf-network:
diff --git a/jussi.yaml b/jussi.yaml
index ff6f0bf..f2e178b 100644
--- a/jussi.yaml
+++ b/jussi.yaml
@@ -52,7 +52,7 @@ services:
# target: /etc/nginx/nginx.conf
# read_only: true
# - type: bind
- # source: ${HAF_LOG_DIRECTORY}/jussi
+ # source: ${HAF_LOG_DIRECTORY:-${TOP_LEVEL_DATASET_MOUNTPOINT}/logs}/jussi
# target: /var/log/jussi
# depends_on:
# jussi-real:
diff --git a/monitoring.yaml b/monitoring.yaml
index c92312e..cd45ea1 100644
--- a/monitoring.yaml
+++ b/monitoring.yaml
@@ -1,6 +1,6 @@
services:
prometheus:
- image: prom/prometheus:${PROMETHEUS_VERSION}
+ image: prom/prometheus:${PROMETHEUS_VERSION:-v2.49.1}
profiles:
- monitoring
volumes:
@@ -22,7 +22,7 @@ services:
org.label-schema.group: "monitoring"
nodeexporter:
- image: prom/node-exporter:${NODE_EXPORTER_VERSION}
+ image: prom/node-exporter:${NODE_EXPORTER_VERSION:-v1.7.0}
profiles:
- monitoring
volumes:
@@ -43,7 +43,7 @@ services:
org.label-schema.group: "monitoring"
cadvisor:
- image: gcr.io/cadvisor/cadvisor:${CADVISOR_VERSION}
+ image: gcr.io/cadvisor/cadvisor:${CADVISOR_VERSION:-v0.47.2}
profiles:
- monitoring
privileged: true
@@ -64,7 +64,7 @@ services:
org.label-schema.group: "monitoring"
grafana:
- image: grafana/grafana:${GRAFANA_VERSION}
+ image: grafana/grafana:${GRAFANA_VERSION:-10.3.3}
profiles:
- monitoring
depends_on:
@@ -89,7 +89,7 @@ services:
org.label-schema.group: "monitoring"
loki:
- image: grafana/loki:${LOKI_VERSION}
+ image: grafana/loki:${LOKI_VERSION:-2.9.4}
profiles:
- monitoring
volumes:
@@ -105,7 +105,7 @@ services:
org.label-schema.group: "monitoring"
promtail:
- image: grafana/promtail:${PROMTAIL_VERSION}
+ image: grafana/promtail:${PROMTAIL_VERSION:-2.9.4}
profiles:
- monitoring
depends_on:
@@ -121,7 +121,7 @@ services:
org.label-schema.group: "monitoring"
hived-pme:
- image: registry.gitlab.syncad.com/hive/hived-pme:${HIVED_PME_VERSION}
+ image: registry.gitlab.syncad.com/hive/hived-pme:${HIVED_PME_VERSION:-49a7312d}
profiles:
- monitoring
restart: unless-stopped
@@ -154,12 +154,12 @@ services:
networks:
- haf-network
environment:
- - DATA_SOURCE_NAME=${DATA_SOURCE}
+ - DATA_SOURCE_NAME=${DATA_SOURCE:-postgresql://postgres@haf:5432/postgres?sslmode=disable}
labels:
org.label-schema.group: "monitoring"
blackboxexporter:
- image: prom/blackbox-exporter:${BLACKBOX_VERSION}
+ image: prom/blackbox-exporter:${BLACKBOX_VERSION:-v0.24.0}
profiles:
- monitoring
restart: unless-stopped
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index fd368b9..750c8c4 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -33,6 +33,7 @@ services:
reputation-tracker-block-processing:
image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
+ - apps
- reputation-tracker-standalone
environment:
POSTGRES_HOST: haf
--
GitLab
From 8b897592a0cc9b767d5e0efc6664b09138fcda3e Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 20 Sep 2024 19:46:05 +0000
Subject: [PATCH 06/78] Make local-admin-only something you can disable via
.env instead of by deleting or modifying a version-controlled file
---
.env.example | 12 ++++++++++--
caddy.yaml | 4 ++++
caddy/Caddyfile | 4 ++++
...l_admin_only.snippet => local-admin-only.snippet} | 0
4 files changed, 18 insertions(+), 2 deletions(-)
rename caddy/{snippets/local_admin_only.snippet => local-admin-only.snippet} (100%)
diff --git a/.env.example b/.env.example
index b40bee0..d291056 100644
--- a/.env.example
+++ b/.env.example
@@ -191,8 +191,16 @@ PUBLIC_HOSTNAME="your.hostname.com"
# real certificate for PUBLIC_HOSTNAME from LetsEncrypt. If this server is
# behind a firewall or NAT, or PUBLIC_HOSTNAME is misconfigured, it will fail
# to get a certificate, and that will count against LetsEncrypt's rate limits.
-# TLS_SELF_SIGNED_SNIPPET=caddy/self-signed.snippet
-TLS_SELF_SIGNED_SNIPPET=/dev/null
+TLS_SELF_SIGNED_SNIPPET=caddy/self-signed.snippet
+# TLS_SELF_SIGNED_SNIPPET=/dev/null
+
+# By default, we restrict access to the /admin URLs to localhost. You can allow
+# connections by switching the following variable to /dev/null. First, though,
+# you should protect the admin endpoint by a password or to a local network.
+# Read caddy/snippets/README.md for how
+LOCAL_ADMIN_ONLY_SNIPPET=caddy/local-admin-only.snippet
+# LOCAL_ADMIN_ONLY_SNIPPET=/dev/null
+
# Caddy will only accept requests on the /admin/ endpoints over https by default.
# This is so that you can password-protect them with HTTP basicauth.
diff --git a/caddy.yaml b/caddy.yaml
index d130513..9016c4d 100644
--- a/caddy.yaml
+++ b/caddy.yaml
@@ -37,6 +37,10 @@ services:
source: ${TLS_SELF_SIGNED_SNIPPET}
target: /etc/caddy/tls-self-signed-snippets/self-signed.snippet
read_only: true
+ - type: bind
+ source: ${LOCAL_ADMIN_ONLY_SNIPPET}
+ target: /etc/caddy/local-admin-only-snippets/local-admin-only.snippet
+ read_only: true
- type: volume
source: caddy_data
target: /data
diff --git a/caddy/Caddyfile b/caddy/Caddyfile
index 10f8889..41bb0d3 100644
--- a/caddy/Caddyfile
+++ b/caddy/Caddyfile
@@ -9,6 +9,10 @@
# put your real TLS config in a file in the snippets directory
import tls-self-signed-snippets/*.snippet
+ # Import a snippet that will restrict connections to the admin endpoint
+ # to the localhost
+ import local-admin-only-snippets/*.snippet
+
import snippets/*.snippet
# Block API abusers outright
diff --git a/caddy/snippets/local_admin_only.snippet b/caddy/local-admin-only.snippet
similarity index 100%
rename from caddy/snippets/local_admin_only.snippet
rename to caddy/local-admin-only.snippet
--
GitLab
From 87f96ef699afadfcb3691c6f7a8296da31c8a073 Mon Sep 17 00:00:00 2001
From: Eric Frias <efrias@syncad.com>
Date: Mon, 23 Sep 2024 14:48:55 +0000
Subject: [PATCH 07/78] Remove weird path stuff from ZPOOL in example env file
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index d291056..0ce20da 100644
--- a/.env.example
+++ b/.env.example
@@ -1,5 +1,5 @@
# The name of the ZFS storage pool for HAF to use
-ZPOOL="/workspace/hafbe/haf-pool"
+ZPOOL="haf-pool"
# The name of the dataset on $ZPOOL where HAF will store its data
# HAF won't read/write anything outside of $ZPOOL/$TOP_LEVEL_DATASET,
# so you can have, e.g., multiple HAF installations on the same
--
GitLab
From 6bbc35d7547461ca7ab1d2a88aa19633469866dc Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 23 Sep 2024 21:35:54 +0000
Subject: [PATCH 08/78] Update default images to rc3
---
.env.example | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 0ce20da..91e535f 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,8 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.6rc1
+HIVE_API_NODE_VERSION=1.27.6rc3
+
# Global settings
--
GitLab
From 58f29a2d2803a9cd0f6e39491b77e8a003fd1ee8 Mon Sep 17 00:00:00 2001
From: DanEric <dan@syncad.com>
Date: Tue, 24 Sep 2024 18:25:55 +0000
Subject: [PATCH 09/78] change hivemind healtcheck to start after 3 days to
ensure haf has replayed
---
hivemind.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hivemind.yaml b/hivemind.yaml
index 2bbd32a..585862c 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -17,7 +17,7 @@ services:
interval: 10s
timeout: 2s
retries: 10
- start_period: 5m
+ start_period: 72h
depends_on:
haf:
condition: service_healthy
--
GitLab
From eb0ee44ed08a7a4b4648df2591dbe982c74f5c01 Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Wed, 25 Sep 2024 12:57:46 +0000
Subject: [PATCH 10/78] Update to new block-range setup
---
.env.example | 12 ++++++++++++
.env.swp | Bin 16384 -> 0 bytes
hafah_rest/rewrite_rules.conf | 3 ---
3 files changed, 12 insertions(+), 3 deletions(-)
delete mode 100644 .env.swp
diff --git a/.env.example b/.env.example
index 91e535f..a3ca6eb 100644
--- a/.env.example
+++ b/.env.example
@@ -95,6 +95,10 @@ NETWORK_NAME=haf
# values below
# HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah
# HAFAH_VERSION=${HIVE_API_NODE_VERSION}
+HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah/setup
+
+# HAFAH_VERSION=9556cd0f
+# HAFAH_VERSION=a85fc0b4
# The default setup will run the recommended version of Hivemind using the values
# below. You can override them here to run a custom version of Hivemind
@@ -113,7 +117,15 @@ NETWORK_NAME=haf
# HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
# HAF_REPUTATION_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
+HAF_VERSION=a5fd4e7d
+#HAF_VERSION=v1.27.6rc2
+
+HAF_BLOCK_EXPLORER_VERSION=40b14fca
+REPUTATION_TRACKER_VERSION=0367602f
+HAFAH_VERSION=d619d73b
+BALANCE_TRACKER_VERSION=9c105a40
+HIVEMIND_INSTANCE_VERSION=d29e5480
# REPTRACKER_SCHEMA="hafbe-rt"
# BTRACKER_SCHEMA="hafbe-bt"
# The default setup will run the recommended version of HAF block explorer,
diff --git a/.env.swp b/.env.swp
deleted file mode 100644
index cdefd5914f833ea85a4ad4b703e0aff0ffb9286c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 16384
zcmeI3U5q2yRmW@DK-jQZ5D^F|FPEN?W+w7Sx4UO&rjfI_yWQQ+y6sNeGn<J(p0exO
zuI+MFZGH5I!ipCpLY^Q}@&W_|h^#~&K!PB6Nq7OoBHrPJ2ZDl-K)?$kAcfz#RpsvX
z^h~k|5+K={|G4e@b?&+6p7TG)_Hgx4>p<PF)GQu9WLY2n=)aENZT;B`$3OOhHMPeB
z+nKq(v_0pszNYWj?$$OQtPY3I>1I_=gXlO8?UAlB_0ar3utRs?2ac{*w14uv(aW}_
z29_H5?lka0YrVF%Ds$Xe`Em83AAb1VO}8Aq)WA{$OARbFu++d(14|7oHL%pcQUm|5
zG?0WZT0e`#-;+yykUzh$@chmEahSi~S!jPTf4!AIe_^3Lf4O{GYGA2>r3RK7SZZLY
zfu#nP8dz#zsez>imKs=UV5x!s1r6AaWwlxNS0v%*^Zz3M|EnLgtS^B#!A<bBS1ju*
z;Mc$xz+0dPJ^)?<U;UtEC14E7;GbTmFW3OT{{xowHn<DE@d3;F5Ab{77_5N5e#x@F
z3_b&TU>(%JzkI)C{VsS1d>Z^bNI?SD!8hJ-S$_xq61)pO4}Kba2>ki?S=Je-fNy@U
zWqlER9NY$9d!J?fF*pOKpbUN#`~^jfp93jyzzF;V_!01*6iEIC{08`Sa0*iJhv@AO
zfavd+MTh@`j~nX1_8mJ3qH`65i5vK_vLn4|-LP(`#?juxgJ!4StFM&Hkq$lkygc-R
z(eY?%yS^%y<0J@+20O{x)ZJC{pjn@}ys5`Qq|5QSKjJMjoN;y%%fKK?%0ZfhX;SZW
z`>ox_*LO+ILtWo}*lyG82Z=j&N4Al*tbHd8TtBH#-4pGU!<pWEuy+6Mijlv0X3s)T
zt7&kml0c=gR<po&*{thNRIH;Dccf#wetR2T+#!oRf8b(=_}C50M$p9;C!K}m*tf%Y
z8YK1FYHfY>er=6js_n*!3dS7DB%NtLi52Qp$yBTT#%{3(%jz`yZ*-4dA9NZA%{t$F
zVPxvNGf}YZCly{!P>_t9DaRd;b;MlC*NJ>zM)~$k$D*QRt;4L78wZ}9Xs4o7mr>EF
z8*66iv#^5&=;+Gel5)rDJV;fP`ieP?aIVie`)F7Nyve|rWsJx=Dp`j|-H$c5`*VrQ
zNa8tHmly@?fKv;j;7vV3UM481iAD=yq{r@=c9<KvwLO}u5r1NrA{zZH3k3~&`v-%a
z))5PJj~>@oZhorYJsh-~kDBelPNU!GHT#2u?!!+1u-oeN->$|}jNci|^o$E3>c3`=
z>da5%%SD5W&d=!io>zO4tG3v88mhAML=Gm>=rT$~9y!dhi&^uw(duQHIot?MA$>_{
z$4%(rVlsJmnT5-!xBwH^w>@=Yd#R2~3Ln6%jd6nV%$#ytr?!mr-FT{<5~r>f#dCsF
zAL*vr=U!#jr7o0$43!~Y?7Bf?im}<gvO`WXj9d&$C9`mxNfw{S31-PWk(SN%16fQ2
zFL9<bv3iA*FoJoUuNrEP8;#7^c+o+m4rsGw55<OzXkwNl7B?PfY_nFRhUbbUxgB=o
zdgpisaR~Hr{c?8pPwzt{Yh_06YwQl~&+IssTV-aJ>v-F(=!&@qn32L$6*H?(?)Fr1
zzweZcX};0G2ERkoH1>CDdn;z9c6aZ+%;W`=I2$Ne;6037;X6I%Mm%aB^;+Fdy|(_U
zGrWIayp3Fx;_Y?&-p1Wm$2B9|QFE`=>vQ>Termt<s5xjHwg#Q<PIFK+ye*rl4((VE
z3Xj2j*i#g_j&WHsIhKG*-I+bnw_-(OOsq0C5a|g%lK5pBl~~cH2)gG5Lrgr@Ny6#I
z_=79U7s9SpYWFIyMx!-`7{lV3F^aom3{t7bxdB}fDX}M+D|UUyJ#n3srHlZ?6R}_k
zHe!>;=h1nB$rMtHJNG=(yf5~HN-!4onoUqLYQ7?Dp(_*S++blV^VUKYm5G~p_OK#`
zX*(6}NL9iPl3mro(*vbjg&;i<ZKST}7&@TrGzn&e+!1o0t1uOpAQvJRSV0K<iEJ2W
z!^L>=Ed+U@PDQtB7$j4*-D0BbH9l^LW6Ne$r!Gnw;{DU1XPgkRD8GTH1U2Ip2{%Ln
zsD!66ASNP!rk-O?u#iTM!O>$o^^(#h(dJuyf<II{Sy0<ghb3m|Go;u<E}J>2+&||6
z$|Vm|&*NyvQ7}WiP7uvRMI~_pmbKkIIPCVC17h87tKEc0i4~X37hX$Py*-uxhq@%P
z@FiHMB%}n}mVojd4=!DhoNsrYYnCrx;!2w)Y71F1?BkHlYP6c-p!XuS;oBwE4vyWD
z+P0k&s6(Vm=)%$ZOc;nbWEn$r_V_E{w(9tzY0I-m$HU;Pc&Abk(<+-a(Z#~z5ut$l
zTpbbe#cRvu5woP<)F!-~q5L;f9J9Lpv0k@Bo9TC_M6in_n5o-G&0c^0zFCUE#CYRk
z#Mfjm=ZC_?a#Zq!Io`&r2(2k65pokuuz6d>>5x%(vf1Pd&70x^#wpi42)s;h=G?_s
zag&gol{Qs4$*h4txDtUGFLG=e_rN^Znr9qf9wx<kC>Xmf9@98LkyFBVVjmNU#S)?K
zb2hdZwA1j3(#c5fQi3P7O^CJ2u?}tI!j!pdhH;`x;SJ)?7EMu71S4P3SCm%pX|r;1
zL4}h=fhTv!x2)%pFCk~2JnVMc86g|wjO-c?_4bPORLA;CH6KnZs+eM6#ifrq3EVS|
z%L)8j!g|@I@mQR~HA2-_IOwvd7&0ZZg`A*JhGL9*Y9Dlp1G8~q8dLiOnyM?4N=eP|
zI<6!kq9PODJb1Oqvo5h>#!LIDkuTD=Q$3AbNRsWJ<HfKQj6RlV$m!oSo3Gw{N{GU7
z3Hz0f#_zEUiDs;ojVrvFtS0_7BZMxDS~i?17G+-ZHTtrsLDV76@$wpm+rTRc&6%ns
zLy_$e3Ah86<o~})-ufrxvXcLA<@xIu$@#nB-^uO&8GH`B2EI;i{|@MZDtH<E13CWJ
z!Mnf%Yv6t0Meq(e|1X03;3L2Sze&#jSzv=c*Z?nq-y!G!EZ7DAMSlNB;Pc>f;5G2~
z<n_M<w!w#i<oG|6=lFk1UjHTV8So3>9{4wM`riUS1zO-E;0|~NybS)4y#7zYZ-Y+*
z9~^=Tcm;?~zAAcIKK}1&APY>&u;pI&;n8-pzCvD_ObBCdd7IU0@!}_O9GiDH)@$>3
zUW;RIMj)&^ZY<np#mXn!YB#oD-|Bv1PMS|vE47WvYSAu4ZD(+J(42SJcwpaKU#mGq
z4@kv9ztL*XJJc!<DjP*>yZd_U`j)+;#%`n2m^ZGiR@N))#klRp&ZAbZdo*uGu8n*A
zb|;tHQ1|914!cMF=A}++m3tSmi)79Ie)D1PVwPHE<3R<EjsG~fsKM}i90|*=s0M~k
z3c7R9>I|Bl9pnCmIQ6^R-FAJNBq7ZvR;r@t6AMpBL!lG)Ou{fB3o0)Dve#=H4rVgH
z1$Ig(ko0i`oiS_`Kgm4%!KsFN6_t!z@N$CJ*qx+Q1jMyRIUy0Z7WhOH!_|~XX7WuU
zfUqKoDdHm$hkrzcWNIh#&=8U`;#;XyML{AZ4Hlo_iQ&Qf{r;gMs6(>TWGY*OOgZP7
zn>GuI$YBC8eT$?WkDUjQs?kdILM<kFGJ?etwsP)U!4tN7gI=>uS){es!Mu8%*5P5Z
zU#~j)r0T<DtsBqkLLT9qS7Yss%dtE0q5rY(hM`W76NONdQBVmT<;6MTpXBFe?<UTS
zvKqjI0uA({O$n#zkD~KXIEe5LXvd5wB-<Wpl;|+sm|6x&t|$Dw)99Dv`S9UZyR|*o
z@Ampq>k=K$-1y?8IKL!Q*%-FTY}kf(S+I=3F3cjN>9SB+reP9@jFGbrbG^|x>a;q0
zn?<!NztkquUQ*5F<qRb*PSH-J>R>V#qcEm46boWt=E#+tDVB&_)N-VsNNA4?apX)*
zNkt5OesnU2gmJ30BeaiQksdqdG8p5fR+Aevvt2r<Z01V`B3dRd%SwJ1mxwg$L@F_~
zVEJ=1xR}!vnM?*<f@HW9<Pr@n&#l<l2(>vmYS{AS2`-H04tno%yM|K7qnn%^(r(Rd
zn3?r0xaY-r=OklcPOkrzJ>EuTu0QfpqYt(~6uxQ#^%V-+wpSUBD&w~tzY-d%qj)^k
zQPf$P;{SyNNpfwJ6jhAU%JT+_tHPGbB4Oq#xUX|MSac(WDyX-pHunWND>rw@{F+f1
z{k6h~`ym2dBa^hm*ENY$3WJVQF(A=$W@vY0k*nfr91|IiaZN(%a8bRi%2i@m90qd7
zsrHKNn!BaKm`TaSopRqHv=Gv4*{p<C<Ojths0OY}r9RW@x7cmA0|XM2e9x-MMx!wk
z9AKym`_8I?#^aie-Og*<LhF@J;q*+d$lTj2?U8Fx0C6Zzn#hCUM`Uh>=2Pr@#>qW(
zWSLxr3KoQtG&d8>90C_XV#d5KGe5-aIvcsmAI%QT<ipAQ0|4X^$HrCb5wMibJ|u}{
zy*_ulSYJZcEqpH-FGYP{wszSc2EN=?Q^k>FFqYVfqdm3H<9V=~-%^R}5R0A1b3Ram
zFKm=;FQlGZmaUXWUXVItkICkpTL+E3=JV@@7r*T_zLTN(VHGu6`9Ol(l#=H_5u|*m
zZcBcWZyQIM)#@7C1e2XkPm(+;I#jb_N(OPLNA4Kkkre}(ewVR@K8R4nOj0Q70H);F
zd)(O`Y#ntQJKK$3f3O9`gUSr>Ms+_ZLj{@O8GVFM)$gx9kTT>my7Zby6fN^lces~%
z^**awW-$-yfGtYdZyE)DkP`o+z;;Gdy$#jjFpZ(YXz++)flA`Syn<lTW>ZZOy-;VE
ze^iu(K|(O+T-0S*YEIt7d=`px*-GNo=$K8KS+H5xxZ7-HQ~Wg1ofb?etL<@0r*<TT
zjm(ZDaLXHPB+KsII7<1*QivKyX7*=i#llhMa3iRXBOevAUA7}C3)4tGQ4*TbXM-n!
zOYJU*Mq=J#!nh#$MSA{&aXu22h{cgJJhP-&BaM^GT_{Cn&RLd-E7HgckUy-EMoI_f
zV3>`?W>P&8t_ta$;r}r%<81i2jsOp>V?-pGa|E}sk43agC(690AUZOYuS^0aIi(<T
z9_9twLe#9JlM8E#53({MT%ItKSKO3PEn^B7MKvz@u54d8EYUa)d@jXx=9Qm;_yR}6
M6^zyy-Sw^i1`zqZxBvhE
diff --git a/hafah_rest/rewrite_rules.conf b/hafah_rest/rewrite_rules.conf
index d34d5f1..6b6b8a6 100644
--- a/hafah_rest/rewrite_rules.conf
+++ b/hafah_rest/rewrite_rules.conf
@@ -1,6 +1,3 @@
-rewrite ^/block-number-by-date/([^/]+) /rpc/get_block_by_time?timestamp=$1 break;
-# endpoint for get /block-number-by-date/{timestamp}
-
rewrite ^/global-state /rpc/get_global_state break;
# endpoint for get /global-state
--
GitLab
From 76449f64c38cf518bd15e92de24bcb5205b1dec7 Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Wed, 25 Sep 2024 12:59:39 +0000
Subject: [PATCH 11/78] Ignore sw files
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index 4c49bd7..13d1d46 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
.env
+*.sw?
\ No newline at end of file
--
GitLab
From dcfd97765ed1760b1b0662e46e0368cbc95ad1d4 Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Fri, 27 Sep 2024 07:30:30 +0000
Subject: [PATCH 12/78] Update example env
---
.env.example | 21 ++++++++++++---------
1 file changed, 12 insertions(+), 9 deletions(-)
diff --git a/.env.example b/.env.example
index a3ca6eb..c732970 100644
--- a/.env.example
+++ b/.env.example
@@ -93,9 +93,12 @@ NETWORK_NAME=haf
# The default setup will run the recommended version of HAfAH,
# you can run a custom version by un-commenting and modifying the
# values below
-# HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah
-# HAFAH_VERSION=${HIVE_API_NODE_VERSION}
-HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah/setup
+
+# when using rc tag
+#HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah
+
+# when using commit hash
+#HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah/setup
# HAFAH_VERSION=9556cd0f
# HAFAH_VERSION=a85fc0b4
@@ -117,15 +120,15 @@ HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah/setup
# HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
# HAF_REPUTATION_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
-HAF_VERSION=a5fd4e7d
+HAF_VERSION=34e90932
#HAF_VERSION=v1.27.6rc2
-HAF_BLOCK_EXPLORER_VERSION=40b14fca
-REPUTATION_TRACKER_VERSION=0367602f
-HAFAH_VERSION=d619d73b
-BALANCE_TRACKER_VERSION=9c105a40
+HAF_BLOCK_EXPLORER_VERSION=1.27.6rc4
+REPUTATION_TRACKER_VERSION=1.27.6rc3
+HAFAH_VERSION=1.27.6rc4
+BALANCE_TRACKER_VERSION=1.27.6rc4
-HIVEMIND_INSTANCE_VERSION=d29e5480
+HIVEMIND_INSTANCE_VERSION=67fa2149
# REPTRACKER_SCHEMA="hafbe-rt"
# BTRACKER_SCHEMA="hafbe-bt"
# The default setup will run the recommended version of HAF block explorer,
--
GitLab
From 105d3f35f8a4fdfa050878b82fbd63f1c5f19eff Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Mon, 7 Oct 2024 11:30:25 +0000
Subject: [PATCH 13/78] Set reptracker as seperate app in hafbe and hivemind
---
.env.example | 10 +--
compose.yml | 1 -
exposed/hivemind_rtracker.exposed.yaml | 4 -
haf_base.yaml | 4 -
haf_block_explorer.yaml | 4 +-
haproxy/haproxy.cfg | 8 --
healthchecks/checks/hafbe.sh | 4 +-
healthchecks/docker_entrypoint.sh | 1 -
hivemind.yaml | 5 +-
hivemind_reptracker.yaml | 83 -------------------
reputation_tracker.yaml | 14 ++--
.../hivemind_rtracker_nginx.conf | 32 -------
reputation_tracker/rewrite_rules.conf | 3 +
13 files changed, 17 insertions(+), 156 deletions(-)
delete mode 100644 exposed/hivemind_rtracker.exposed.yaml
delete mode 100644 hivemind_reptracker.yaml
delete mode 100644 reputation_tracker/hivemind_rtracker_nginx.conf
diff --git a/.env.example b/.env.example
index c732970..3577388 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.6rc3
+HIVE_API_NODE_VERSION=1.27.6rc5
# Global settings
@@ -120,15 +120,7 @@ NETWORK_NAME=haf
# HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
# HAF_REPUTATION_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
-HAF_VERSION=34e90932
-#HAF_VERSION=v1.27.6rc2
-HAF_BLOCK_EXPLORER_VERSION=1.27.6rc4
-REPUTATION_TRACKER_VERSION=1.27.6rc3
-HAFAH_VERSION=1.27.6rc4
-BALANCE_TRACKER_VERSION=1.27.6rc4
-
-HIVEMIND_INSTANCE_VERSION=67fa2149
# REPTRACKER_SCHEMA="hafbe-rt"
# BTRACKER_SCHEMA="hafbe-bt"
# The default setup will run the recommended version of HAF block explorer,
diff --git a/compose.yml b/compose.yml
index dcc8b4f..6b5cfb0 100644
--- a/compose.yml
+++ b/compose.yml
@@ -5,7 +5,6 @@ include:
- backend.yaml
- hafah.yaml
- hivemind.yaml
- - hivemind_reptracker.yaml
- balance_tracker.yaml
- reputation_tracker.yaml
- haf_block_explorer.yaml
diff --git a/exposed/hivemind_rtracker.exposed.yaml b/exposed/hivemind_rtracker.exposed.yaml
deleted file mode 100644
index b53e64b..0000000
--- a/exposed/hivemind_rtracker.exposed.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-services:
- hivemind-rtracker-postgrest:
- ports:
- - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7010:3000"
\ No newline at end of file
diff --git a/haf_base.yaml b/haf_base.yaml
index bff2ebd..8d17cb0 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -57,10 +57,6 @@ services:
host haf_block_log hivemind ${COMPOSE_PROJECT_NAME}-hivemind-server-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-hivemind-uninstall-1.${NETWORK_NAME} trust\n
- host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-hivemind-rtracker-install-1.${NETWORK_NAME} trust\n
- host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-hivemind-rtracker-uninstall-1.${NETWORK_NAME} trust\n
- host haf_block_log reptracker_user ${COMPOSE_PROJECT_NAME}-hivemind-rtracker-postgrest-1.${NETWORK_NAME} trust\n
-
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-pgadmin-1.${NETWORK_NAME} trust\n
host postgres haf_admin ${COMPOSE_PROJECT_NAME}-pgadmin-1.${NETWORK_NAME} trust\n
host haf_block_log pghero ${COMPOSE_PROJECT_NAME}-pghero-1.${NETWORK_NAME} trust\n
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index d22c2d7..b068bb5 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -16,8 +16,6 @@ services:
condition: service_healthy
balance-tracker-install:
condition: service_completed_successfully
- reputation-tracker-install:
- condition: service_completed_successfully
hafah-install:
condition: service_completed_successfully
block-explorer-uninstall:
@@ -98,7 +96,7 @@ services:
PGRST_DB_POOL: 20
PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
PGRST_DB_ROOT_SPEC: root
- PGRST_DB_EXTRA_SEARCH_PATH: ${BTRACKER_SCHEMA:-hafbe_bal}, ${REPTRACKER_SCHEMA:-hafbe_rep}
+ PGRST_DB_EXTRA_SEARCH_PATH: ${BTRACKER_SCHEMA:-hafbe_bal}, ${REPTRACKER_SCHEMA:-reptracker_app}
# PGRST_OPENAPI_SERVER_PROXY_URI: http://${PUBLIC_HOSTNAME}/hafbe_api/
networks:
haf-network:
diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg
index f9c41be..4e4d0f6 100644
--- a/haproxy/haproxy.cfg
+++ b/haproxy/haproxy.cfg
@@ -87,11 +87,6 @@ frontend reputation-tracker-in-7009
option http-server-close
default_backend reputation-tracker
-frontend hivemind-rtracker-in-7010
- bind *:7010
- option http-server-close
- default_backend hivemind-rtracker
-
backend hived
balance roundrobin
server haf haf:8090 check agent-check agent-addr haproxy-healthchecks agent-port 7001 agent-inter 10s
@@ -108,9 +103,6 @@ backend reputation-tracker
balance roundrobin
server reputation-tracker reputation-tracker-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7009 agent-inter 10s
-backend hivemind-rtracker
- balance roundrobin
- server hivemind-rtracker hivemind-rtracker-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7010 agent-inter 10s
backend hafah
balance roundrobin
server hafah-postgrest hafah-postgrest-rewriter:80
diff --git a/healthchecks/checks/hafbe.sh b/healthchecks/checks/hafbe.sh
index 64bf318..a066c91 100755
--- a/healthchecks/checks/hafbe.sh
+++ b/healthchecks/checks/hafbe.sh
@@ -8,10 +8,10 @@ trap "trap - 2 15 && kill -- -\$\$" 2 15
check_haf_lib
-REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select hive.app_get_current_block_num('hafbe_rep'))")
+REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select hive.app_get_current_block_num('reptracker_app'))")
if [ "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE")
- echo "down #hafbe_rep block over a minute old ($age_string)"
+ echo "down #reptracker_app block over a minute old ($age_string)"
exit 3
fi
diff --git a/healthchecks/docker_entrypoint.sh b/healthchecks/docker_entrypoint.sh
index 9fd84b3..01f1315 100755
--- a/healthchecks/docker_entrypoint.sh
+++ b/healthchecks/docker_entrypoint.sh
@@ -6,6 +6,5 @@ nc -lk -p 7003 -e /checks/hafah.sh &
nc -lk -p 7004 -e /checks/hafbe_btracker.sh &
nc -lk -p 7005 -e /checks/hafbe.sh &
nc -lk -p 7009 -e /checks/hafbe_reptracker.sh &
-nc -lk -p 7010 -e /checks/hivemind_rtracker.sh &
wait
diff --git a/hivemind.yaml b/hivemind.yaml
index 585862c..351b9e2 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -11,7 +11,6 @@ services:
- "--database-url=postgresql://hivemind@haf/haf_block_log"
- "--database-admin-url=postgresql://haf_admin@haf/haf_block_log"
- "--install-app"
- - "--only-hivemind"
healthcheck:
test: ["CMD-SHELL","/home/hivemind/block-processing-healthcheck.sh || exit 1"]
interval: 10s
@@ -21,7 +20,7 @@ services:
depends_on:
haf:
condition: service_healthy
- hivemind-rtracker-install:
+ reputation-tracker-install:
condition: service_completed_successfully
hivemind-server:
@@ -56,7 +55,7 @@ services:
depends_on:
haf:
condition: service_healthy
- hivemind-rtracker-install:
+ reputation-tracker-install:
condition: service_completed_successfully
hivemind-uninstall:
diff --git a/hivemind_reptracker.yaml b/hivemind_reptracker.yaml
deleted file mode 100644
index 0f8a734..0000000
--- a/hivemind_reptracker.yaml
+++ /dev/null
@@ -1,83 +0,0 @@
-services:
- hivemind-rtracker-install:
- image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
- profiles:
- - hivemind
- - apps
- environment:
- POSTGRES_HOST: haf
- networks:
- haf-network:
- command:
- - install_app
- - --is_forking=false
- - --swagger-url=${PUBLIC_HOSTNAME}
- depends_on:
- haf:
- condition: service_healthy
- hivemind-rtracker-uninstall:
- image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
- profiles:
- - reputation-tracker-uninstall
- environment:
- POSTGRES_HOST: haf
- networks:
- haf-network:
- command:
- - uninstall_app
- depends_on:
- haf:
- condition: service_healthy
- hivemind-rtracker-postgrest:
- image: ${POSTGREST_REGISTRY:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
- profiles:
- - hivemind
- - apps
- environment:
- PGRST_ADMIN_SERVER_PORT: 3001
- PGRST_DB_URI: postgresql://reptracker_user@haf/haf_block_log
- PGRST_DB_SCHEMA: reptracker_endpoints
- PGRST_DB_ANON_ROLE: reptracker_user
- PGRST_DB_POOL: 20
- PGRST_DB_ROOT_SPEC: root
- PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
- PGRST_DB_EXTRA_SEARCH_PATH: reptracker_app
- # PGRST_OPENAPI_SERVER_PROXY_URI: http://${PUBLIC_HOSTNAME}/reptracker_user/
- healthcheck:
- test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:3001/ready || exit 1"]
- interval: 10s
- timeout: 3s
- retries: 10
- start_period: 1m
- networks:
- haf-network:
- depends_on:
- hivemind-rtracker-install:
- condition: service_completed_successfully
- haf:
- condition: service_healthy
- hivemind-rtracker-postgrest-rewriter:
- image: nginx
- profiles:
- - hivemind
- - apps
- volumes:
- - type: bind
- source: reputation_tracker/hivemind_rtracker_nginx.conf
- target: /etc/nginx/nginx.conf
- read_only: true
- - type: bind
- source: reputation_tracker/rewrite_rules.conf
- target: /etc/nginx/rewrite_rules.conf
- read_only: true
- depends_on:
- hivemind-rtracker-postgrest:
- condition: service_healthy
- networks:
- - haf-network
- semaphore:
- image: busybox
- depends_on:
- haf:
- condition: service_healthy
- command: ["sleep", "20"]
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index 750c8c4..87f0794 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -4,6 +4,7 @@ services:
profiles:
- apps
- hafbe
+ - hivemind
- reputation-tracker-setup
environment:
POSTGRES_HOST: haf
@@ -11,11 +12,10 @@ services:
haf-network:
command:
- install_app
- - --schema=${REPTRACKER_SCHEMA:-hafbe_rep}
- --swagger-url=${PUBLIC_HOSTNAME}
depends_on:
- semaphore:
- condition: service_started
+ haf:
+ condition: service_healthy
reputation-tracker-uninstall:
image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
@@ -26,7 +26,6 @@ services:
haf-network:
command:
- uninstall_app
- - --schema=${REPTRACKER_SCHEMA:-hafbe_rep}
depends_on:
haf:
condition: service_healthy
@@ -34,6 +33,8 @@ services:
image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
+ - hafbe
+ - hivemind
- reputation-tracker-standalone
environment:
POSTGRES_HOST: haf
@@ -41,7 +42,6 @@ services:
haf-network:
command:
- process_blocks
- - --schema=${REPTRACKER_SCHEMA:-hafbe_rep}
healthcheck:
test: ["CMD-SHELL","/app/block-processing-healthcheck.sh"]
interval: 60s
@@ -57,6 +57,7 @@ services:
image: ${POSTGREST_REGISTRY:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
- apps
+ - hivemind
- reputation-tracker-standalone
- hafbe
environment:
@@ -67,7 +68,7 @@ services:
PGRST_DB_POOL: 20
PGRST_DB_ROOT_SPEC: root
PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
- PGRST_DB_EXTRA_SEARCH_PATH: ${REPTRACKER_SCHEMA:-hafbe_rep}
+ PGRST_DB_EXTRA_SEARCH_PATH: reptracker_app
# PGRST_OPENAPI_SERVER_PROXY_URI: http://${PUBLIC_HOSTNAME}/reptracker_user/
healthcheck:
test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:3001/ready || exit 1"]
@@ -88,6 +89,7 @@ services:
- apps
- reputation-tracker-standalone
- hafbe
+ - hivemind
volumes:
- type: bind
source: reputation_tracker/reputation_tracker_nginx.conf
diff --git a/reputation_tracker/hivemind_rtracker_nginx.conf b/reputation_tracker/hivemind_rtracker_nginx.conf
deleted file mode 100644
index 846172d..0000000
--- a/reputation_tracker/hivemind_rtracker_nginx.conf
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Homepage and endpoints of the API "HAF Block Explorer".
-#
-worker_processes 5;
-error_log /dev/stdout info;
-worker_rlimit_nofile 8192;
-
-events {
- worker_connections 4096;
-}
-http {
- access_log /dev/stdout;
- server {
- listen 0.0.0.0:80 default_server;
- server_name _;
-
- location / {
- include rewrite_rules.conf;
- rewrite_log on;
-
- proxy_pass http://hivemind-rtracker-postgrest:3000; # my PostREST is here!
-
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- #default_type;
- proxy_hide_header Content-Location;
- proxy_set_header Connection "";
- proxy_http_version 1.1;
- }
- }
-}
diff --git a/reputation_tracker/rewrite_rules.conf b/reputation_tracker/rewrite_rules.conf
index da7bfd8..e9c56f2 100644
--- a/reputation_tracker/rewrite_rules.conf
+++ b/reputation_tracker/rewrite_rules.conf
@@ -1,3 +1,6 @@
+rewrite ^/last-synced-block /rpc/get_rep_last_synced_block break;
+# endpoint for get /last-synced-block
+
rewrite ^/accounts/([^/]+)/reputation /rpc/get_account_reputation?account-name=$1 break;
# endpoint for get /accounts/{account-name}/reputation
--
GitLab
From 43017d5d9a9905224fc6be3e096764dbef19af79 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 7 Oct 2024 18:35:53 +0000
Subject: [PATCH 14/78] Uncomment HAF_SHM_DIRECTORY so that
.assisted_startup.sh works properly
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 3577388..c420b27 100644
--- a/.env.example
+++ b/.env.example
@@ -53,7 +53,7 @@ HIVE_API_NODE_VERSION=1.27.6rc5
# After the sync has finished, do `docker compose down` then move the shared_memory.bin
# file to the shared_memory directory, edit this file to restore original values, and
# `docker compose up -d` to restart HAF.
-# HAF_SHM_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory"
+HAF_SHM_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory"
# HAF_SHM_DIRECTORY="/mnt/haf_shared_mem"
# The docker compose project name, gets prefixed onto each container name
--
GitLab
From c22274a62708605d43d22bb3790a5cb40ebd16f9 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 16 Oct 2024 20:47:32 +0000
Subject: [PATCH 15/78] properly enable healthcheck for hafbe
---
balance_tracker.yaml | 1 +
haf_block_explorer.yaml | 7 +++++++
reputation_tracker.yaml | 1 +
3 files changed, 9 insertions(+)
diff --git a/balance_tracker.yaml b/balance_tracker.yaml
index d1095b4..9163d3f 100644
--- a/balance_tracker.yaml
+++ b/balance_tracker.yaml
@@ -36,6 +36,7 @@ services:
- balance-tracker-standalone
environment:
POSTGRES_HOST: haf
+ POSTGRES_USER: btracker_owner
networks:
haf-network:
command:
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index b068bb5..c173f69 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -54,10 +54,17 @@ services:
- hafbe
environment:
POSTGRES_HOST: haf
+ POSTGRES_USER: hafbe_owner
command:
- process_blocks
networks:
haf-network:
+ healthcheck:
+ test: ["CMD-SHELL","/app/block-processing-healthcheck.sh"]
+ interval: 60s
+ timeout: 10s
+ retries: 3
+ start_period: 48h
depends_on:
block-explorer-install:
condition: service_completed_successfully
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index 87f0794..9824fc1 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -38,6 +38,7 @@ services:
- reputation-tracker-standalone
environment:
POSTGRES_HOST: haf
+ POSTGRES_USER: reptracker_owner
networks:
haf-network:
command:
--
GitLab
From 1bd6fb2538c3bfd3101c6feeb36e4ec8fd1e7f81 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 16 Oct 2024 21:49:21 +0000
Subject: [PATCH 16/78] pass context name to healtcheck for hafbe
---
haf_block_explorer.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index c173f69..470ac11 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -60,7 +60,7 @@ services:
networks:
haf-network:
healthcheck:
- test: ["CMD-SHELL","/app/block-processing-healthcheck.sh"]
+ test: ["CMD-SHELL","/app/block-processing-healthcheck.sh hafbe_app"]
interval: 60s
timeout: 10s
retries: 3
--
GitLab
From 8aed89b17243ed62602e3c16b730242e791f46f1 Mon Sep 17 00:00:00 2001
From: DanEric <dan@syncad.com>
Date: Thu, 17 Oct 2024 20:17:39 +0000
Subject: [PATCH 17/78] Remove dependency: hivemind can replay without
reputation tracker installed.
---
hivemind.yaml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/hivemind.yaml b/hivemind.yaml
index 351b9e2..3391440 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -20,8 +20,8 @@ services:
depends_on:
haf:
condition: service_healthy
- reputation-tracker-install:
- condition: service_completed_successfully
+ # reputation-tracker-install:
+# condition: service_completed_successfully
hivemind-server:
image: ${HIVEMIND_INSTANCE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
--
GitLab
From d93ed1405616b2b93731a2aa48a0be6e0e204823 Mon Sep 17 00:00:00 2001
From: DanEric <dan@syncad.com>
Date: Thu, 17 Oct 2024 21:11:51 +0000
Subject: [PATCH 18/78] switch to using drone by default instead of jussi
---
.env.example | 9 +++++----
assisted_startup.sh | 2 +-
caddy.yaml | 2 +-
compose.yml | 2 +-
exposed/README.md | 4 ++--
exposed/compose.exposed.yml | 2 +-
6 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/.env.example b/.env.example
index c420b27..7e3a76a 100644
--- a/.env.example
+++ b/.env.example
@@ -130,11 +130,12 @@ NETWORK_NAME=haf
# HAF_BLOCK_EXPLORER_VERSION=${HIVE_API_NODE_VERSION}
-# The default setup uses "Jussi" as the API reverse proxy & cache for the old JSON-RPC-style
-# calls. There is an alternate reverse proxy, "Drone", that you can choose to use instead:
+# The default setup uses "Drone" as the API reverse proxy & cache for the old JSON-RPC-style
+# calls. There is the older alternate reverse proxy, "Jussi", that you can choose to use instead.
+# For more info about drone/jussi, see:
# https://hive.blog/hive-139531/@deathwing/announcing-drone-or-leveling-up-hive-api-nodes-and-user-experience
-# To replace Jussi with Drone, uncomment the next line:
-# JSONRPC_API_SERVER_NAME=drone
+# To replace Drone with Jussi, uncomment the next line:
+# JSONRPC_API_SERVER_NAME=jussi
# The default setup will run the recommended version of Jussi
# you can run a custom version by un-commenting and modifying the
diff --git a/assisted_startup.sh b/assisted_startup.sh
index edf7240..2fcbb84 100755
--- a/assisted_startup.sh
+++ b/assisted_startup.sh
@@ -84,7 +84,7 @@ if [ ! -f .env ]; then
echo "core: the minimal HAF system of a database and hived"
echo "admin: useful tools for administrating HAF: pgadmin, pghero"
echo "apps: core HAF apps: hivemind, hafah, hafbe (balance-tracker is a subapp)"
- echo "servers: services for routing/caching API calls: haproxy, jussi (JSON caching), varnish (REST caching)"
+ echo "servers: services for routing/caching API calls: haproxy, jussi/drone (JSON caching), varnish (REST caching)"
read -p "Run admin? (Y or N): " choice
if [[ "$choice" == "Y" || "$choice" == "y" ]]; then
echo "Adding admin to profiles..."
diff --git a/caddy.yaml b/caddy.yaml
index 9016c4d..9a91666 100644
--- a/caddy.yaml
+++ b/caddy.yaml
@@ -10,7 +10,7 @@ services:
- 443:443/udp
environment:
PUBLIC_HOSTNAME: ${PUBLIC_HOSTNAME}
- JSONRPC_API_SERVER_NAME: ${JSONRPC_API_SERVER_NAME:-jussi}
+ JSONRPC_API_SERVER_NAME: ${JSONRPC_API_SERVER_NAME:-drone}
ADMIN_ENDPOINT_PROTOCOL: ${ADMIN_ENDPOINT_PROTOCOL:-https}
volumes:
- type: bind
diff --git a/compose.yml b/compose.yml
index 6b5cfb0..d8bd0de 100644
--- a/compose.yml
+++ b/compose.yml
@@ -9,7 +9,7 @@ include:
- reputation_tracker.yaml
- haf_block_explorer.yaml
- varnish.yaml
- - ${JSONRPC_API_SERVER_NAME:-jussi}.yaml
+ - ${JSONRPC_API_SERVER_NAME:-drone}.yaml
- haproxy.yaml
- caddy.yaml
- monitoring.yaml
diff --git a/exposed/README.md b/exposed/README.md
index 54ad02f..fb69635 100644
--- a/exposed/README.md
+++ b/exposed/README.md
@@ -1,10 +1,10 @@
This "exposed" configuration includes directives that allow you to directly access
the services from other machines -- without this config, the only exposed ports
go to the main webserver, and the only way to access, e.g., hafah's postgrest
-server is by sending it through caddy -> jussi -> haproxy -> hafah-postgrest
+server is by sending it through caddy -> jussi/drone -> haproxy -> hafah-postgrest
Exposing ports like this is useful if you want to split the stack across multiple
-machines (e.g., caddy + jussi + redis + varnish on one machine, everything else
+machines (e.g., caddy + drone + varnish on one machine, everything else
on the others). Or, if you have two redundant servers, and you want to be able
to add backup entries in haproxy that send traffic to the other server.
diff --git a/exposed/compose.exposed.yml b/exposed/compose.exposed.yml
index cc68f8d..49a2ff1 100644
--- a/exposed/compose.exposed.yml
+++ b/exposed/compose.exposed.yml
@@ -2,7 +2,7 @@ include:
- exposed/haproxy.exposed.yaml
- exposed/hafah.exposed.yaml
- exposed/haf_block_explorer.exposed.yaml
- - exposed/${JSONRPC_API_SERVER_NAME:-jussi}.exposed.yaml
+ - exposed/${JSONRPC_API_SERVER_NAME:-drone}.exposed.yaml
- exposed/hivemind.exposed.yaml
- exposed/haf_base.exposed.yaml
- exposed/balance_tracker.exposed.yaml
--
GitLab
From c27dd95351f292769682f83b67630681632b61dd Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Tue, 5 Nov 2024 15:23:54 +0000
Subject: [PATCH 19/78] New comment-search api
---
haf_block_explorer.yaml | 2 ++
haf_block_explorer/rewrite_rules.conf | 7 +++++--
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 470ac11..743de78 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -18,6 +18,8 @@ services:
condition: service_completed_successfully
hafah-install:
condition: service_completed_successfully
+ reputation-tracker-install:
+ condition: service_completed_successfully
block-explorer-uninstall:
image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
diff --git a/haf_block_explorer/rewrite_rules.conf b/haf_block_explorer/rewrite_rules.conf
index d36e206..95b5810 100644
--- a/haf_block_explorer/rewrite_rules.conf
+++ b/haf_block_explorer/rewrite_rules.conf
@@ -13,8 +13,11 @@ rewrite ^/version /rpc/get_hafbe_version break;
rewrite ^/block-numbers /rpc/get_block_by_op?path-filter=$path_filters break;
# endpoint for get /block-numbers
-rewrite ^/accounts/([^/]+)/comment-operations /rpc/get_comment_operations?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/comment-operations
+rewrite ^/accounts/([^/]+)/operations/comments/([^/]+) /rpc/get_comment_operations?account-name=$1&permlink=$2 break;
+# endpoint for get /accounts/{account-name}/operations/comments/{permlink}
+
+rewrite ^/accounts/([^/]+)/comment-permlinks /rpc/get_comment_permlinks?account-name=$1 break;
+# endpoint for get /accounts/{account-name}/comment-permlinks
rewrite ^/accounts/([^/]+)/authority /rpc/get_account_authority?account-name=$1 break;
# endpoint for get /accounts/{account-name}/authority
--
GitLab
From 88711a3bbcf192e7f9f782ba9930ededbc74ce5e Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 7 Nov 2024 16:40:57 +0000
Subject: [PATCH 20/78] Fix up health checks to work after data was migrated
out of hive schema
---
haproxy.yaml | 2 +-
healthchecks/Dockerfile | 4 ++--
healthchecks/checks/hafbe.sh | 11 ++---------
healthchecks/checks/hafbe_btracker.sh | 2 +-
healthchecks/checks/hafbe_reptracker.sh | 2 +-
healthchecks/checks/hivemind.sh | 2 +-
healthchecks/checks/hivemind_rtracker.sh | 2 +-
7 files changed, 9 insertions(+), 16 deletions(-)
diff --git a/haproxy.yaml b/haproxy.yaml
index a1fa963..4ec5a61 100644
--- a/haproxy.yaml
+++ b/haproxy.yaml
@@ -17,7 +17,7 @@ services:
healthcheck:
test: wget -q --spider 127.0.0.1:8001
haproxy-healthchecks:
- image: ${HAPROXY_HEALTHCHECKS_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy-healthchecks}:${HAPROXY_HEALTHCHECKS_VERSION:-latest}
+ image: ${HAPROXY_HEALTHCHECKS_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy-healthchecks}:${HAPROXY_HEALTHCHECKS_VERSION:-${HIVE_API_NODE_VERSION}}
environment:
POSTGRES_URL: postgresql://haf_admin@haf/haf_block_log
POSTGRES_URL_HAFBE: postgresql://hafbe_user@haf/haf_block_log
diff --git a/healthchecks/Dockerfile b/healthchecks/Dockerfile
index f3f4219..a79acc5 100644
--- a/healthchecks/Dockerfile
+++ b/healthchecks/Dockerfile
@@ -3,8 +3,8 @@ FROM alpine:3.18.3
RUN apk --no-cache add postgresql14-client
COPY checks/hived.sh /checks/hived.sh
-COPY checks/hafbe_bal.sh /checks/hafbe_bal.sh
-COPY checks/hafbe_rep.sh /checks/hafbe_rep.sh
+COPY checks/hafbe_btracker.sh /checks/hafbe_btracker.sh
+COPY checks/hafbe_reptracker.sh /checks/hafbe_reptracker.sh
COPY checks/hivemind.sh /checks/hivemind.sh
COPY checks/hafah.sh /checks/hafah.sh
COPY checks/hafbe.sh /checks/hafbe.sh
diff --git a/healthchecks/checks/hafbe.sh b/healthchecks/checks/hafbe.sh
index a066c91..0893d7a 100755
--- a/healthchecks/checks/hafbe.sh
+++ b/healthchecks/checks/hafbe.sh
@@ -8,21 +8,14 @@ trap "trap - 2 15 && kill -- -\$\$" 2 15
check_haf_lib
-REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select hive.app_get_current_block_num('reptracker_app'))")
+REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from hive.get_app_current_block_age('reptracker_app'))::integer")
if [ "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE")
echo "down #reptracker_app block over a minute old ($age_string)"
exit 3
fi
-BTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_BTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select hive.app_get_current_block_num('hafbe_bal'))")
-if [ "$BTRACKER_LAST_PROCESSED_BLOCK_AGE" -gt 60 ]; then
- age_string=$(format_seconds "$BTRACKER_LAST_PROCESSED_BLOCK_AGE")
- echo "down #hafbe_bal block over a minute old ($age_string)"
- exit 3
-fi
-
-HAFBE_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_HAFBE" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select hive.app_get_current_block_num('hafbe_app'))")
+HAFBE_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_HAFBE" --quiet --no-align --tuples-only --command="select extract('epoch' from hive.get_app_current_block_age(ARRAY['hafbe_app', 'hafbe_bal']))::integer")
if [ "$HAFBE_LAST_PROCESSED_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$HAFBE_LAST_PROCESSED_BLOCK_AGE")
echo "down #hafbe block over a minute old ($age_string)"
diff --git a/healthchecks/checks/hafbe_btracker.sh b/healthchecks/checks/hafbe_btracker.sh
index ae48610..fe54048 100755
--- a/healthchecks/checks/hafbe_btracker.sh
+++ b/healthchecks/checks/hafbe_btracker.sh
@@ -8,7 +8,7 @@ trap "trap - 2 15 && kill -- -\$\$" 2 15
check_haf_lib
-BTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_BTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select hive.app_get_current_block_num('btracker_app'))")
+BTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_BTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from hive.get_app_current_block_age('hafbe_bal'))::integer")
if [ "$BTRACKER_LAST_PROCESSED_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$BTRACKER_LAST_PROCESSED_BLOCK_AGE")
echo "down #hafbe_bal block over a minute old ($age_string)"
diff --git a/healthchecks/checks/hafbe_reptracker.sh b/healthchecks/checks/hafbe_reptracker.sh
index 52339b0..1fe1d20 100755
--- a/healthchecks/checks/hafbe_reptracker.sh
+++ b/healthchecks/checks/hafbe_reptracker.sh
@@ -8,7 +8,7 @@ trap "trap - 2 15 && kill -- -\$\$" 2 15
check_haf_lib
-REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select hive.app_get_current_block_num('reptracker_app'))")
+REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from hive.get_app_current_block_age('reptracker_app'))::integer")
if [ "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE")
echo "down #hafbe_rep block over a minute old ($age_string)"
diff --git a/healthchecks/checks/hivemind.sh b/healthchecks/checks/hivemind.sh
index 9bea3fd..e78787f 100755
--- a/healthchecks/checks/hivemind.sh
+++ b/healthchecks/checks/hivemind.sh
@@ -8,7 +8,7 @@ trap "trap - 2 15 && kill -- -\$\$" 2 15
check_haf_lib
-HIVEMIND_LAST_IMPORTED_BLOCK_AGE=$(psql "$POSTGRES_URL" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - (select last_imported_block_date from hivemind_app.hive_state limit 1))::integer")
+HIVEMIND_LAST_IMPORTED_BLOCK_AGE=$(psql "$POSTGRES_URL" --quiet --no-align --tuples-only --command="select extract('epoch' from hive.get_app_current_block_age('hivemind_app'))::integer")
if [ "$HIVEMIND_LAST_IMPORTED_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$HIVEMIND_LAST_IMPORTED_BLOCK_AGE")
echo "down #hivemind block over a minute old ($age_string)"
diff --git a/healthchecks/checks/hivemind_rtracker.sh b/healthchecks/checks/hivemind_rtracker.sh
index d93763e..e816326 100644
--- a/healthchecks/checks/hivemind_rtracker.sh
+++ b/healthchecks/checks/hivemind_rtracker.sh
@@ -8,7 +8,7 @@ trap "trap - 2 15 && kill -- -\$\$" 2 15
check_haf_lib
-REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select hive.app_get_current_block_num('reptracker_app'))")
+REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from hive.get_app_current_block_age('reptracker_app'))::integer")
if [ "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE")
echo "down #reptracker_app block over a minute old ($age_string)"
--
GitLab
From 8b229a3e53578266e7a6fb70d850dea2a946e92a Mon Sep 17 00:00:00 2001
From: SynaptiCAD User <sales@syncad.com>
Date: Mon, 8 Jul 2024 20:06:41 +0000
Subject: [PATCH 21/78] Allow drone log level to be configured in the .env file
---
.env.example | 1 +
drone.yaml | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 7e3a76a..27f15b5 100644
--- a/.env.example
+++ b/.env.example
@@ -149,6 +149,7 @@ NETWORK_NAME=haf
# and modifying the values below
# DRONE_REGISTRY=${HIVE_API_NODE_REGISTRY}/drone
# DRONE_VERSION=latest
+# DRONE_LOG_LEVEL=warn,access_log=info
# In the default configuration, synchronous broadcast_transaction calls are not handled by
# your local stack, but instead are sent to a dedicated hived instance on api.hive.blog.
diff --git a/drone.yaml b/drone.yaml
index 4c7f6da..e044b08 100644
--- a/drone.yaml
+++ b/drone.yaml
@@ -4,7 +4,7 @@ services:
profiles:
- servers
environment:
- RUST_LOG: access_log=info
+ RUST_LOG: ${DRONE_LOG_LEVEL:-warn,access_log=info}
volumes:
- type: bind
source: ./drone/config.yaml
--
GitLab
From a42e98e2694ee7004f4ec042eae541fcfa7ddd75 Mon Sep 17 00:00:00 2001
From: SynaptiCAD User <sales@syncad.com>
Date: Mon, 8 Jul 2024 20:34:35 +0000
Subject: [PATCH 22/78] Add a new custom image for haproxy (just adding socat),
and enable haproxy's management port
---
drone.yaml | 14 ++++++++++++--
haproxy.yaml | 2 +-
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/drone.yaml b/drone.yaml
index e044b08..2a1329b 100644
--- a/drone.yaml
+++ b/drone.yaml
@@ -17,7 +17,7 @@ services:
- haf-network
# to use our log analysis tools, we need to capture drone's traffic using a
# specific nginx logging config. To do this, rename the actual drone
- # service above to 'jussi-real', then uncomment the next section.
+ # service above to 'drone-real', then uncomment the next section.
# drone:
# image: nginx
# profiles:
@@ -33,7 +33,17 @@ services:
# depends_on:
# drone-real:
# condition: service_started
+ # networks:
+ # - haf-network
+ # drone:
+ # image: mitmproxy/mitmproxy
+ # profiles:
+ # - servers
# ports:
- # - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:9001:9000"
+ # - 8081:8081
+ # command: mitmweb --listen-port 9000 --web-host 0.0.0.0 --mode reverse:http://drone-real:9000
+ # depends_on:
+ # drone-real:
+ # condition: service_started
# networks:
# - haf-network
diff --git a/haproxy.yaml b/haproxy.yaml
index 4ec5a61..395f7d3 100644
--- a/haproxy.yaml
+++ b/haproxy.yaml
@@ -1,6 +1,6 @@
services:
haproxy:
- image: ${HAPROXY_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy}:${HAPROXY_VERSION:-latest}
+ image: ${HAPROXY_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy}:${HAPROXY_VERSION:-latest}
profiles:
- servers
environment:
--
GitLab
From b41c9aad95fd567b710ecd480fc24172d410f4d1 Mon Sep 17 00:00:00 2001
From: SynaptiCAD User <sales@syncad.com>
Date: Mon, 8 Jul 2024 20:39:05 +0000
Subject: [PATCH 23/78] Log request ids when intercepting Drone traffic
---
drone/nginx.conf | 2 +-
jussi.yaml | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/drone/nginx.conf b/drone/nginx.conf
index 51d95b0..df0441d 100644
--- a/drone/nginx.conf
+++ b/drone/nginx.conf
@@ -13,7 +13,7 @@ http {
'"$http_user_agent" || "$http_x_forwarded_for" || '
'"$upstream_http_x_jussi_cache_hit" || "$upstream_http_x_jussi_namespace" || '
'"$upstream_http_x_jussi_api" || "$upstream_http_x_jussi_method" || "$upstream_http_x_jussi_params" || '
- '$upstream_connect_time || $upstream_response_time || "$request_body"';
+ '$upstream_connect_time || $upstream_response_time || "$request_body" || "$request_id"';
# we intend to change the log to a structured format (jsonl) to make parsing easier, something like below.
# that work hasn't been done yet
log_format json_log escape=json '{'
diff --git a/jussi.yaml b/jussi.yaml
index f2e178b..1776b9c 100644
--- a/jussi.yaml
+++ b/jussi.yaml
@@ -46,6 +46,8 @@ services:
# service above to 'jussi-real', then uncomment the next section.
# jussi:
# image: nginx
+ # profiles:
+ # - servers
# volumes:
# - type: bind
# source: ./jussi/nginx.conf
--
GitLab
From 51aa0ca81b7423839f069e5a537a4b1c3e9bcfd8 Mon Sep 17 00:00:00 2001
From: SynaptiCAD User <sales@syncad.com>
Date: Mon, 8 Jul 2024 20:41:49 +0000
Subject: [PATCH 24/78] Add config files to restrict the maximum size of docker
log files
---
log_rotation/README.md | 23 +++++++++++++++++++
log_rotation/backend.log_rotation.yaml | 7 ++++++
.../balance_tracker.log_rotation.yaml | 16 +++++++++++++
log_rotation/caddy.log_rotation.yaml | 10 ++++++++
log_rotation/compose.log_rotation.yml | 11 +++++++++
log_rotation/drone.log_rotation.yaml | 7 ++++++
log_rotation/haf_base.log_rotation.yaml | 16 +++++++++++++
.../haf_block_explorer.log_rotation.yaml | 22 ++++++++++++++++++
log_rotation/hafah.log_rotation.yaml | 16 +++++++++++++
log_rotation/haproxy.log_rotation.yaml | 6 +++++
log_rotation/hivemind.log_rotation.yaml | 19 +++++++++++++++
log_rotation/jussi.log_rotation.yaml | 10 ++++++++
log_rotation/varnish.log_rotation.yaml | 7 ++++++
13 files changed, 170 insertions(+)
create mode 100644 log_rotation/README.md
create mode 100644 log_rotation/backend.log_rotation.yaml
create mode 100644 log_rotation/balance_tracker.log_rotation.yaml
create mode 100644 log_rotation/caddy.log_rotation.yaml
create mode 100644 log_rotation/compose.log_rotation.yml
create mode 100644 log_rotation/drone.log_rotation.yaml
create mode 100644 log_rotation/haf_base.log_rotation.yaml
create mode 100644 log_rotation/haf_block_explorer.log_rotation.yaml
create mode 100644 log_rotation/hafah.log_rotation.yaml
create mode 100644 log_rotation/haproxy.log_rotation.yaml
create mode 100644 log_rotation/hivemind.log_rotation.yaml
create mode 100644 log_rotation/jussi.log_rotation.yaml
create mode 100644 log_rotation/varnish.log_rotation.yaml
diff --git a/log_rotation/README.md b/log_rotation/README.md
new file mode 100644
index 0000000..b0cf579
--- /dev/null
+++ b/log_rotation/README.md
@@ -0,0 +1,23 @@
+This _log_rotation_ configuration includes directives that cause docker to
+limit how much data is kept when logging the container's stdout/stderr.
+
+In the default configuration, without these files, docker will log using the
+system's default logging configuration. By default, this uses the json-file
+logging driver, which writes all output to a text file in _JSON Lines_ format.
+By default, logs will be kept forever. This puts you at risk of running out
+of disk space eventually, though if you have large disks, low API traffic,
+or you regularly restart your containers, this may never be an issue for you.
+
+Including this config:
+ - switches the logging driver to the more efficient _local_, and
+ - sets finite limits on how much space the log files can take
+
+At the moment, these limits are high, but should allow a public API node to
+keep at least one day's worth of logs for the most verbose containers.
+
+To use this config, add a line to your .env file telling docker to merge this
+file in:
+
+```
+ COMPOSE_FILE=compose.yml:log_rotation/compose.log_rotation.yml
+```
diff --git a/log_rotation/backend.log_rotation.yaml b/log_rotation/backend.log_rotation.yaml
new file mode 100644
index 0000000..a094fa5
--- /dev/null
+++ b/log_rotation/backend.log_rotation.yaml
@@ -0,0 +1,7 @@
+services:
+ pghero:
+ logging:
+ driver: local
+ pgadmin:
+ logging:
+ driver: local
diff --git a/log_rotation/balance_tracker.log_rotation.yaml b/log_rotation/balance_tracker.log_rotation.yaml
new file mode 100644
index 0000000..c142d55
--- /dev/null
+++ b/log_rotation/balance_tracker.log_rotation.yaml
@@ -0,0 +1,16 @@
+services:
+ balance-tracker-install:
+ logging:
+ driver: local
+ balance-tracker-uninstall:
+ logging:
+ driver: local
+ balance-tracker-block-processing:
+ logging:
+ driver: local
+ balance-tracker-postgrest:
+ logging:
+ driver: local
+ balance-tracker-swagger:
+ logging:
+ driver: local
diff --git a/log_rotation/caddy.log_rotation.yaml b/log_rotation/caddy.log_rotation.yaml
new file mode 100644
index 0000000..8a282f4
--- /dev/null
+++ b/log_rotation/caddy.log_rotation.yaml
@@ -0,0 +1,10 @@
+services:
+ caddy:
+ logging:
+ driver: local
+ options:
+ max-size: "100m"
+ max-file: 10
+ version-display:
+ logging:
+ driver: local
diff --git a/log_rotation/compose.log_rotation.yml b/log_rotation/compose.log_rotation.yml
new file mode 100644
index 0000000..78f7933
--- /dev/null
+++ b/log_rotation/compose.log_rotation.yml
@@ -0,0 +1,11 @@
+include:
+ - log_rotation/backend.log_rotation.yaml
+ - log_rotation/balance_tracker.log_rotation.yaml
+ - log_rotation/caddy.log_rotation.yaml
+ - log_rotation/${JSONRPC_API_SERVER_NAME:-jussi}.log_rotation.yaml
+ - log_rotation/haf_base.log_rotation.yaml
+ - log_rotation/haf_block_explorer.log_rotation.yaml
+ - log_rotation/hafah.log_rotation.yaml
+ - log_rotation/haproxy.log_rotation.yaml
+ - log_rotation/hivemind.log_rotation.yaml
+ - log_rotation/varnish.log_rotation.yaml
diff --git a/log_rotation/drone.log_rotation.yaml b/log_rotation/drone.log_rotation.yaml
new file mode 100644
index 0000000..39872c1
--- /dev/null
+++ b/log_rotation/drone.log_rotation.yaml
@@ -0,0 +1,7 @@
+services:
+ drone:
+ logging:
+ driver: local
+ options:
+ max-size: "1000m"
+ max-file: 50
diff --git a/log_rotation/haf_base.log_rotation.yaml b/log_rotation/haf_base.log_rotation.yaml
new file mode 100644
index 0000000..ec65db8
--- /dev/null
+++ b/log_rotation/haf_base.log_rotation.yaml
@@ -0,0 +1,16 @@
+services:
+ haf:
+ logging:
+ driver: local
+ options:
+ max-size: "100m"
+ max-file: 10
+ logrotate:
+ logging:
+ driver: local
+ pgbadger:
+ logging:
+ driver: local
+ ofelia:
+ logging:
+ driver: local
diff --git a/log_rotation/haf_block_explorer.log_rotation.yaml b/log_rotation/haf_block_explorer.log_rotation.yaml
new file mode 100644
index 0000000..02990f6
--- /dev/null
+++ b/log_rotation/haf_block_explorer.log_rotation.yaml
@@ -0,0 +1,22 @@
+services:
+ block-explorer-block-processing:
+ logging:
+ driver: local
+ options:
+ max-size: "100m"
+ max-file: 10
+ block-explorer-install:
+ logging:
+ driver: local
+ block-explorer-postgrest:
+ logging:
+ driver: local
+ options:
+ max-size: "100m"
+ max-file: 10
+ block-explorer-uninstall:
+ logging:
+ driver: local
+ block-explorer-uninstall-balance-tracker:
+ logging:
+ driver: local
diff --git a/log_rotation/hafah.log_rotation.yaml b/log_rotation/hafah.log_rotation.yaml
new file mode 100644
index 0000000..f049189
--- /dev/null
+++ b/log_rotation/hafah.log_rotation.yaml
@@ -0,0 +1,16 @@
+services:
+ hafah-postgrest:
+ logging:
+ driver: local
+ options:
+ max-size: "100m"
+ max-file: 10
+ hafah-benchmarks-old-style:
+ logging:
+ driver: local
+ hafah-install:
+ logging:
+ driver: local
+ hafah-uninstall:
+ logging:
+ driver: local
diff --git a/log_rotation/haproxy.log_rotation.yaml b/log_rotation/haproxy.log_rotation.yaml
new file mode 100644
index 0000000..c95ba96
--- /dev/null
+++ b/log_rotation/haproxy.log_rotation.yaml
@@ -0,0 +1,6 @@
+services:
+ haproxy-healthchecks:
+ # expose the healthchecks so we can connect to them from an external haproxy
+ # shift the ports by 20000 so they don't conflict with haproxy's (if we want to expose those also)
+ ports:
+ - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_HEALTHCHECK_PORT_PREFIX:-2}7001-${HAF_API_NODE_EXPOSED_HEALTHCHECK_PORT_PREFIX:-2}7005:7001-7005"
diff --git a/log_rotation/hivemind.log_rotation.yaml b/log_rotation/hivemind.log_rotation.yaml
new file mode 100644
index 0000000..8a42173
--- /dev/null
+++ b/log_rotation/hivemind.log_rotation.yaml
@@ -0,0 +1,19 @@
+services:
+ hivemind-block-processing:
+ logging:
+ driver: local
+ options:
+ max-size: "100m"
+ max-file: 10
+ hivemind-server:
+ logging:
+ driver: local
+ options:
+ max-size: "100m"
+ max-file: 10
+ hivemind-install:
+ logging:
+ driver: local
+ hivemind-uninstall:
+ logging:
+ driver: local
diff --git a/log_rotation/jussi.log_rotation.yaml b/log_rotation/jussi.log_rotation.yaml
new file mode 100644
index 0000000..a7ac040
--- /dev/null
+++ b/log_rotation/jussi.log_rotation.yaml
@@ -0,0 +1,10 @@
+services:
+ jussi:
+ logging:
+ driver: local
+ options:
+ max-size: "1000m"
+ max-file: 5
+ redis:
+ logging:
+ driver: local
diff --git a/log_rotation/varnish.log_rotation.yaml b/log_rotation/varnish.log_rotation.yaml
new file mode 100644
index 0000000..d70b165
--- /dev/null
+++ b/log_rotation/varnish.log_rotation.yaml
@@ -0,0 +1,7 @@
+services:
+ varnish:
+ logging:
+ driver: local
+ options:
+ max-size: "100m"
+ max-file: 10
--
GitLab
From 85a292bea5b71283034be44744def81a9b8cf843 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 7 Nov 2024 18:39:04 +0000
Subject: [PATCH 25/78] Add ci to build healthcheck docker images for release
tags
---
.gitlab-ci.yml | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
create mode 100644 .gitlab-ci.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..c24fdf9
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,23 @@
+stages:
+ - publish
+
+build_exchange_flyway_docker_image:
+ stage: publish
+ variables:
+ DOCKER_BUILDKIT: 1
+ DOCKER_DRIVER: overlay2
+ DOCKER_TLS_CERTDIR: "/certs"
+ image: docker:27.3.1
+ services:
+ - docker:27.3.1-dind
+ script:
+ - "docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY"
+ - "docker login -u $BLOG_REGISTRY_USER -p $BLOG_REGISTRY_PASSWORD registry-upload.hive.blog"
+ - "(cd healthchecks && docker build -t $CI_REGISTRY_IMAGE/haproxy-healthchecks:$CI_COMMIT_TAG -t registry-upload.hive.blog/haf_api_node/haproxy-healthchecks:$CI_COMMIT_TAG .)"
+ - "docker push $CI_REGISTRY_IMAGE/haproxy-healthchecks:$CI_COMMIT_TAG"
+ - "docker push registry-upload.hive.blog/haf_api_node/haproxy-healthchecks:$CI_COMMIT_TAG"
+ tags:
+ - public-runner-docker
+ rules:
+ - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG =~ /^1\..+$/
+ when: always
--
GitLab
From d6819d3f5ced94c0dcde83f21e233faae2030777 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 7 Nov 2024 18:51:52 +0000
Subject: [PATCH 26/78] Bump pgadmin
---
backend.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend.yaml b/backend.yaml
index 2e7e706..156558c 100644
--- a/backend.yaml
+++ b/backend.yaml
@@ -14,7 +14,7 @@ services:
condition: service_healthy
pgadmin:
- image: dpage/pgadmin4:${PGADMIN_VERSION:-8.10}
+ image: dpage/pgadmin4:${PGADMIN_VERSION:-8.12}
profiles:
- admin
environment:
--
GitLab
From 4c38e38ec8f6f156d38144137f1b2eacbaa86061 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 24 Sep 2024 21:06:46 +0000
Subject: [PATCH 27/78] Allow setting a different public hostname from the
private hostname, required for swagger to work on, e.g., configurations that
serve plain http
---
.env.example | 28 ++++++++++++++++++----------
caddy.yaml | 1 +
caddy/Caddyfile | 2 +-
3 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/.env.example b/.env.example
index 27f15b5..02fd07f 100644
--- a/.env.example
+++ b/.env.example
@@ -174,23 +174,31 @@ NETWORK_NAME=haf
# CADDY_IMAGE=${HIVE_API_NODE_REGISTRY}/haf_api_node/caddy
# CADDY_VERSION=latest
-# The hostname you'll be running this server on. There are several ways you can configure
-# this. Some examples:
-# - to serve API using HTTPS, with automatic redirect from HTTP -> HTTPS, just give the
-# hostname:
-# PUBLIC_HOSTNAME="your.hostname.com"
+# The hostname you'll be running this server on. This should be a single hostname, the public
+# hostname your server will be accessible from. This is used by the Swagger-UI REST API
+# explorer for generating URLs pointing at your server. If this isn't a public server,
+# this can be a local domain name.
+PUBLIC_HOSTNAME="your.hostname.com"
+
+# There are several ways you can configure serving HTTP/HTTPS. Some examples:
+# - to serve API using HTTPS with automatic redirect from HTTP -> HTTPS (the default),
+# just give the hostname:
+# CADDY_SITES="your.hostname.com"
+# In the normal case, where you want to serve HTTP/HTTPS from the hostname you set in
+# PUBLIC_HOSTNAME above, you don't need to set this variable, it will automatically take
+# the value of PUBLIC_HOSTNAME
# - to serve using only HTTP (if you have nginx or something else handling SSL termination),
# you can use:
-# PUBLIC_HOSTNAME="http://your.hostname.com"
+# CADDY_SITES="http://your.hostname.com"
# or even:
-# PUBLIC_HOSTNAME="http://"
+# CADDY_SITES="http://"
# if you want to respond on any hostname
# - to serve on either HTTP or HTTPS (i.e., respond to HTTP requests in the clear, instead of
# issuing a redirect):
-# PUBLIC_HOSTNAME="http://your.hostname.com, https://your.hostname.com"
+# CADDY_SITES="http://your.hostname.com, https://your.hostname.com"
# - to serve on multiple hostnames, separate them with a comma and space:
-# PUBLIC_HOSTNAME="your.hostname.com, your.other-hostname.net"
-PUBLIC_HOSTNAME="your.hostname.com"
+# CADDY_SITES="your.hostname.com, your.other-hostname.net"
+# CADDY_SITES="your.hostname.com"
# By default, we're configured to use a self-signed SSL certificate (by including the
# file below, which tells Caddy to generate a self-signed certificate). To obtain a real
diff --git a/caddy.yaml b/caddy.yaml
index 9a91666..7462fc0 100644
--- a/caddy.yaml
+++ b/caddy.yaml
@@ -10,6 +10,7 @@ services:
- 443:443/udp
environment:
PUBLIC_HOSTNAME: ${PUBLIC_HOSTNAME}
+ CADDY_SITES: ${CADDY_SITES:-${PUBLIC_HOSTNAME}}
JSONRPC_API_SERVER_NAME: ${JSONRPC_API_SERVER_NAME:-drone}
ADMIN_ENDPOINT_PROTOCOL: ${ADMIN_ENDPOINT_PROTOCOL:-https}
volumes:
diff --git a/caddy/Caddyfile b/caddy/Caddyfile
index 41bb0d3..fa91f4b 100644
--- a/caddy/Caddyfile
+++ b/caddy/Caddyfile
@@ -3,7 +3,7 @@
}
# Simple caddy config, handles SSL and forwards everything to varnish
-{$PUBLIC_HOSTNAME} {
+{$CADDY_SITES} {
# Import a snippet that will generate a self-signed certificate by default.
# To generate a real certificate, bind-mount an empty file here and then
# put your real TLS config in a file in the snippets directory
--
GitLab
From 690afcbbdcab71e923ba9654a4b7cd92931dc2ae Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 7 Nov 2024 22:05:26 +0000
Subject: [PATCH 28/78] Bump version number in example env file
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 02fd07f..eeb7ad1 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.6rc5
+HIVE_API_NODE_VERSION=1.27.6rc8
# Global settings
--
GitLab
From a18e22f1d1200a753044054dc495432e7616b17b Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 11 Nov 2024 18:49:33 +0000
Subject: [PATCH 29/78] Fix reptracker health check
---
healthchecks/checks/check_haf_lib.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/healthchecks/checks/check_haf_lib.sh b/healthchecks/checks/check_haf_lib.sh
index fe8c556..d5058db 100755
--- a/healthchecks/checks/check_haf_lib.sh
+++ b/healthchecks/checks/check_haf_lib.sh
@@ -5,7 +5,7 @@ check_haf_lib() {
echo "down #HAF not in sync"
exit 1
fi
- LAST_IRREVERSIBLE_BLOCK_AGE=$(psql "$POSTGRES_URL" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hive.blocks where num = (select consistent_block from hive.irreversible_data)")
+ LAST_IRREVERSIBLE_BLOCK_AGE=$(psql "$POSTGRES_URL" --quiet --no-align --tuples-only --command="select extract('epoch' from now() - created_at)::integer from hafd.blocks where num = (select consistent_block from hafd.irreversible_data)")
if [ "$LAST_IRREVERSIBLE_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$LAST_IRREVERSIBLE_BLOCK_AGE")
echo "down #HAF LIB over a minute old ($age_string)"
--
GitLab
From 83c389df3dcd2344e29c0c6cc95fee171d02822e Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 14 Nov 2024 23:21:13 +0000
Subject: [PATCH 30/78] Add the ability to get email notifications when haproxy
detects a service going down
---
haproxy.yaml | 40 ++++++++++++++++++++++++++++++++++++++++
haproxy/haproxy.cfg | 25 +++++--------------------
haproxy/mailer.cfg | 28 ++++++++++++++++++++++++++++
haproxy/no-mailer.cfg | 20 ++++++++++++++++++++
4 files changed, 93 insertions(+), 20 deletions(-)
create mode 100644 haproxy/mailer.cfg
create mode 100644 haproxy/no-mailer.cfg
diff --git a/haproxy.yaml b/haproxy.yaml
index 395f7d3..050dc53 100644
--- a/haproxy.yaml
+++ b/haproxy.yaml
@@ -7,15 +7,41 @@ services:
SYNC_BROADCAST_BACKEND_SERVER: "${SYNC_BROADCAST_BACKEND_SERVER:-api.hive.blog}"
SYNC_BROADCAST_BACKEND_PORT: "${SYNC_BROADCAST_BACKEND_PORT:-443}"
SYNC_BROADCAST_BACKEND_SSL: "${SYNC_BROADCAST_BACKEND_SSL:-ssl}"
+ HAPROXY_EMAIL_FROM: "${HAPROXY_EMAIL_FROM:-}"
+ HAPROXY_EMAIL_TO: "${HAPROXY_EMAIL_FROM:-}"
+ HAPROXY_EMAIL_LEVEL: "${HAPROXY_EMAIL_LEVEL:-notice}"
volumes:
- type: bind
source: ./haproxy/haproxy.cfg
target: /usr/local/etc/haproxy/haproxy.cfg
read_only: true
+ - type: bind
+ source: ${HAPROXY_MAILER_CONFIG:-./haproxy/mailer.cfg}
+ target: /usr/local/etc/haproxy/_mailer.cfg
+ read_only: true
networks:
- haf-network
healthcheck:
test: wget -q --spider 127.0.0.1:8001
+ depends_on:
+ smtp-relay:
+ condition: service_started
+ required: false
+ haf:
+ condition: service_started
+ required: false
+ balance-tracker-postgrest:
+ condition: service_started
+ required: false
+ hafah-postgrest:
+ condition: service_started
+ required: false
+ hivemind-server:
+ condition: service_started
+ required: false
+ block-explorer-postgrest:
+ condition: service_started
+ required: false
haproxy-healthchecks:
image: ${HAPROXY_HEALTHCHECKS_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy-healthchecks}:${HAPROXY_HEALTHCHECKS_VERSION:-${HIVE_API_NODE_VERSION}}
environment:
@@ -28,3 +54,17 @@ services:
- haproxy-healthchecks
networks:
- haf-network
+ smtp-relay:
+ profiles:
+ - email
+ image: grafana/smtprelay
+ command:
+ - -hostname=${PUBLIC_HOSTNAME}
+ - -listen=0.0.0.0:25
+ - -allowed_nets=
+ - -remote_host=${SMTP_HOST}
+ - -remote_user=${SMTP_USER}
+ - -remote_pass=${SMTP_PASS}
+ - -remote_auth=${SMTP_AUTH_TYPE:-plain}
+ networks:
+ - haf-network
diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg
index 4e4d0f6..954a3e7 100644
--- a/haproxy/haproxy.cfg
+++ b/haproxy/haproxy.cfg
@@ -7,26 +7,11 @@ global
presetenv SYNC_BROADCAST_BACKEND_PORT 443
presetenv SYNC_BROADCAST_BACKEND_SSL ssl
-defaults
- log global
- mode http
- option httplog
- option dontlognull
- option forwardfor
- option http-server-close
- option log-health-checks
- timeout connect 5s
- timeout client 30s
- timeout server 30s
- timeout tunnel 1h
- default-server init-addr last,libc,none resolvers docker check
- #errorfile 400 /etc/haproxy/errors/400.http
- #errorfile 403 /etc/haproxy/errors/403.http
- #errorfile 408 /etc/haproxy/errors/408.http
- #errorfile 500 /etc/haproxy/errors/500.http
- #errorfile 502 /etc/haproxy/errors/502.http
- #errorfile 503 /etc/haproxy/errors/503.http
- #errorfile 504 /etc/haproxy/errors/504.http
+# DEFAULTS
+# do not put a defaults section here, the defaults section is included
+# from either the `mailer.cfg` or `no-mailer.cfg` files (only one of those
+# files is included, which one depends on your setup). Make any changes
+# to defaults in both of those files.
resolvers docker
parse-resolv-conf
diff --git a/haproxy/mailer.cfg b/haproxy/mailer.cfg
new file mode 100644
index 0000000..8fd04bf
--- /dev/null
+++ b/haproxy/mailer.cfg
@@ -0,0 +1,28 @@
+mailers mta
+ mailer smtp smtp-relay:25
+
+defaults
+ log global
+ mode http
+ option httplog
+ option dontlognull
+ option forwardfor
+ option http-server-close
+ option log-health-checks
+ timeout connect 5s
+ timeout client 30s
+ timeout server 30s
+ timeout tunnel 1h
+ default-server init-addr last,libc,none resolvers docker check
+ #errorfile 400 /etc/haproxy/errors/400.http
+ #errorfile 403 /etc/haproxy/errors/403.http
+ #errorfile 408 /etc/haproxy/errors/408.http
+ #errorfile 500 /etc/haproxy/errors/500.http
+ #errorfile 502 /etc/haproxy/errors/502.http
+ #errorfile 503 /etc/haproxy/errors/503.http
+ #errorfile 504 /etc/haproxy/errors/504.http
+ email-alert from donotreply@syncad.com
+ email-alert to efrias@syncad.com
+ email-alert mailers mta
+ email-alert level notice
+
diff --git a/haproxy/no-mailer.cfg b/haproxy/no-mailer.cfg
new file mode 100644
index 0000000..0583079
--- /dev/null
+++ b/haproxy/no-mailer.cfg
@@ -0,0 +1,20 @@
+defaults
+ log global
+ mode http
+ option httplog
+ option dontlognull
+ option forwardfor
+ option http-server-close
+ option log-health-checks
+ timeout connect 5s
+ timeout client 30s
+ timeout server 30s
+ timeout tunnel 1h
+ default-server init-addr last,libc,none resolvers docker check
+ #errorfile 400 /etc/haproxy/errors/400.http
+ #errorfile 403 /etc/haproxy/errors/403.http
+ #errorfile 408 /etc/haproxy/errors/408.http
+ #errorfile 500 /etc/haproxy/errors/500.http
+ #errorfile 502 /etc/haproxy/errors/502.http
+ #errorfile 503 /etc/haproxy/errors/503.http
+ #errorfile 504 /etc/haproxy/errors/504.http
--
GitLab
From 11825ae1938357746811e4a4f695c7fd8218b143 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 15 Nov 2024 00:26:40 +0000
Subject: [PATCH 31/78] Re-enable a bunch of haproxy healthchecks that were
disabled
---
haproxy.yaml | 17 +++++++++++------
haproxy/haproxy.cfg | 6 +++---
2 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/haproxy.yaml b/haproxy.yaml
index 050dc53..ffcbb5b 100644
--- a/haproxy.yaml
+++ b/haproxy.yaml
@@ -30,16 +30,19 @@ services:
haf:
condition: service_started
required: false
- balance-tracker-postgrest:
+ balance-tracker-postgrest-rewriter:
condition: service_started
required: false
- hafah-postgrest:
+ reputation-tracker-postgrest-rewriter:
+ condition: service_started
+ required: false
+ hafah-postgrest-rewriter:
condition: service_started
required: false
hivemind-server:
condition: service_started
required: false
- block-explorer-postgrest:
+ block-explorer-postgrest-rewriter:
condition: service_started
required: false
haproxy-healthchecks:
@@ -58,13 +61,15 @@ services:
profiles:
- email
image: grafana/smtprelay
+ # When using this service, SMTP_HOST/USER/PASS are required, they're defaulted to empty strings
+ # to keep docker compose from warning when not using this service
command:
- -hostname=${PUBLIC_HOSTNAME}
- -listen=0.0.0.0:25
- -allowed_nets=
- - -remote_host=${SMTP_HOST}
- - -remote_user=${SMTP_USER}
- - -remote_pass=${SMTP_PASS}
+ - -remote_host=${SMTP_HOST:-}
+ - -remote_user=${SMTP_USER:-}
+ - -remote_pass=${SMTP_PASS:-}
- -remote_auth=${SMTP_AUTH_TYPE:-plain}
networks:
- haf-network
diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg
index 954a3e7..2e7b792 100644
--- a/haproxy/haproxy.cfg
+++ b/haproxy/haproxy.cfg
@@ -82,7 +82,7 @@ backend hived-http
backend balance-tracker
balance roundrobin
- server balance-tracker balance-tracker-postgrest-rewriter:80
+ server balance-tracker balance-tracker-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7004 agent-inter 10s
backend reputation-tracker
balance roundrobin
@@ -90,7 +90,7 @@ backend reputation-tracker
backend hafah
balance roundrobin
- server hafah-postgrest hafah-postgrest-rewriter:80
+ server hafah-postgrest hafah-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7003 agent-inter 10s
backend hivemind
balance roundrobin
@@ -98,7 +98,7 @@ backend hivemind
backend block-explorer
balance roundrobin
- server block-explorer block-explorer-postgrest-rewriter:80
+ server block-explorer block-explorer-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7005 agent-inter 10s
backend sync-hived
balance roundrobin
--
GitLab
From 9ba65584df30f0c9ada458e02e5f0d6c1febd887 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 15 Nov 2024 23:08:54 +0000
Subject: [PATCH 32/78] Make email-alerts enable and disable the same way we do
for the "exposed" services.
---
email-alerts/README.md | 37 ++++++++++++++++++++++
email-alerts/compose.email-alerts.yml | 2 ++
email-alerts/haproxy.email-alerts.yaml | 33 +++++++++++++++++++
haproxy.yaml | 44 ++------------------------
4 files changed, 74 insertions(+), 42 deletions(-)
create mode 100644 email-alerts/README.md
create mode 100644 email-alerts/compose.email-alerts.yml
create mode 100644 email-alerts/haproxy.email-alerts.yaml
diff --git a/email-alerts/README.md b/email-alerts/README.md
new file mode 100644
index 0000000..0385550
--- /dev/null
+++ b/email-alerts/README.md
@@ -0,0 +1,37 @@
+This "email-alerts" configuration allows you to have haproxy send you email
+messages to notify you whenever a service goes down.
+
+These alerts are very basic, consisting of a single line that looks like:
+```
+[HAProxy Alert] Server balance-tracker/balance-tracker is DOWN. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue
+```
+
+It's not much, but it's enough to tell you that there's something that needs
+your attention. If you have a more sophisticated monitoring system like
+Zabbix or Nagios, you may want to look into using that instead.
+
+To use this config, add a line to your .env file telling docker to merge this
+file in:
+
+```
+COMPOSE_FILE=compose.yml:email-alerts/compose.email-alerts.yml
+```
+
+In addition, you'll need to set several other settings in your .env file:
+
+First, set the login information for your SMTP server.
+```
+SMTP_HOST="smtp.gmail.com:587"
+SMTP_USER="me@gmail.com"
+SMTP_PASS="myapppassword"
+# Auth defaults to "plain", you can uncomment to use "login" instead
+# SMTP_AUTH_TYPE="login"
+```
+
+You also need to tell it where to send the emails. If you need to, you
+can customize the "from" address and alert threshold.
+```
+HAPROXY_EMAIL_TO="me@gmail.com"
+# HAPROXY_EMAIL_FROM="noreply@${PUBLIC_HOSTNAME}"
+# HAPROXY_EMAIL_LEVEL="notice"
+```
diff --git a/email-alerts/compose.email-alerts.yml b/email-alerts/compose.email-alerts.yml
new file mode 100644
index 0000000..5a0cb18
--- /dev/null
+++ b/email-alerts/compose.email-alerts.yml
@@ -0,0 +1,2 @@
+include:
+ - email-alerts/haproxy.email-alerts.yaml
diff --git a/email-alerts/haproxy.email-alerts.yaml b/email-alerts/haproxy.email-alerts.yaml
new file mode 100644
index 0000000..a78cc16
--- /dev/null
+++ b/email-alerts/haproxy.email-alerts.yaml
@@ -0,0 +1,33 @@
+services:
+ haproxy:
+ environment:
+ HAPROXY_EMAIL_FROM: "${HAPROXY_EMAIL_FROM:-noreply@${PUBLIC_HOSTNAME}}"
+ HAPROXY_EMAIL_TO: "${HAPROXY_EMAIL_TO}"
+ HAPROXY_EMAIL_LEVEL: "${HAPROXY_EMAIL_LEVEL:-notice}"
+ volumes:
+ # override the file bind-mounted by the haproxy.yaml in the upper-level directory
+ - type: bind
+ source: ../haproxy/mailer.cfg
+ target: /usr/local/etc/haproxy/_mailer.cfg
+ read_only: true
+ # if you've enabled email notifications when services go down, haproxy must wait for
+ # the smtp-relay service to be up before launching, otherwise it will fail
+ depends_on:
+ smtp-relay:
+ condition: service_started
+ smtp-relay:
+ profiles:
+ - servers
+ image: grafana/smtprelay
+ # When using this service, SMTP_HOST/USER/PASS are required, they're defaulted to empty strings
+ # to keep docker compose from warning when not using this service
+ command:
+ - -hostname=${PUBLIC_HOSTNAME}
+ - -listen=0.0.0.0:25
+ - -allowed_nets=
+ - -remote_host=${SMTP_HOST}
+ - -remote_user=${SMTP_USER}
+ - -remote_pass=${SMTP_PASS}
+ - -remote_auth=${SMTP_AUTH_TYPE:-plain}
+ networks:
+ - haf-network
diff --git a/haproxy.yaml b/haproxy.yaml
index ffcbb5b..f63d3a5 100644
--- a/haproxy.yaml
+++ b/haproxy.yaml
@@ -7,44 +7,20 @@ services:
SYNC_BROADCAST_BACKEND_SERVER: "${SYNC_BROADCAST_BACKEND_SERVER:-api.hive.blog}"
SYNC_BROADCAST_BACKEND_PORT: "${SYNC_BROADCAST_BACKEND_PORT:-443}"
SYNC_BROADCAST_BACKEND_SSL: "${SYNC_BROADCAST_BACKEND_SSL:-ssl}"
- HAPROXY_EMAIL_FROM: "${HAPROXY_EMAIL_FROM:-}"
- HAPROXY_EMAIL_TO: "${HAPROXY_EMAIL_FROM:-}"
- HAPROXY_EMAIL_LEVEL: "${HAPROXY_EMAIL_LEVEL:-notice}"
volumes:
- type: bind
source: ./haproxy/haproxy.cfg
target: /usr/local/etc/haproxy/haproxy.cfg
read_only: true
- type: bind
- source: ${HAPROXY_MAILER_CONFIG:-./haproxy/mailer.cfg}
+ source: ./haproxy/no-mailer.cfg
target: /usr/local/etc/haproxy/_mailer.cfg
read_only: true
networks:
- haf-network
healthcheck:
test: wget -q --spider 127.0.0.1:8001
- depends_on:
- smtp-relay:
- condition: service_started
- required: false
- haf:
- condition: service_started
- required: false
- balance-tracker-postgrest-rewriter:
- condition: service_started
- required: false
- reputation-tracker-postgrest-rewriter:
- condition: service_started
- required: false
- hafah-postgrest-rewriter:
- condition: service_started
- required: false
- hivemind-server:
- condition: service_started
- required: false
- block-explorer-postgrest-rewriter:
- condition: service_started
- required: false
+ command: -W -db -f /usr/local/etc/haproxy
haproxy-healthchecks:
image: ${HAPROXY_HEALTHCHECKS_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy-healthchecks}:${HAPROXY_HEALTHCHECKS_VERSION:-${HIVE_API_NODE_VERSION}}
environment:
@@ -57,19 +33,3 @@ services:
- haproxy-healthchecks
networks:
- haf-network
- smtp-relay:
- profiles:
- - email
- image: grafana/smtprelay
- # When using this service, SMTP_HOST/USER/PASS are required, they're defaulted to empty strings
- # to keep docker compose from warning when not using this service
- command:
- - -hostname=${PUBLIC_HOSTNAME}
- - -listen=0.0.0.0:25
- - -allowed_nets=
- - -remote_host=${SMTP_HOST:-}
- - -remote_user=${SMTP_USER:-}
- - -remote_pass=${SMTP_PASS:-}
- - -remote_auth=${SMTP_AUTH_TYPE:-plain}
- networks:
- - haf-network
--
GitLab
From babb158ea5674a6efacc46544d35cafcbc13402e Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 15 Nov 2024 23:10:23 +0000
Subject: [PATCH 33/78] Make hivemind's healthcheck require reptracker to be in
sync also
---
healthchecks/checks/hivemind.sh | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/healthchecks/checks/hivemind.sh b/healthchecks/checks/hivemind.sh
index e78787f..4f98ff8 100755
--- a/healthchecks/checks/hivemind.sh
+++ b/healthchecks/checks/hivemind.sh
@@ -8,6 +8,13 @@ trap "trap - 2 15 && kill -- -\$\$" 2 15
check_haf_lib
+REPTRACKER_LAST_PROCESSED_BLOCK_AGE=$(psql "$POSTGRES_URL_REPTRACKER" --quiet --no-align --tuples-only --command="select extract('epoch' from hive.get_app_current_block_age('reptracker_app'))::integer")
+if [ "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE" -gt 60 ]; then
+ age_string=$(format_seconds "$REPTRACKER_LAST_PROCESSED_BLOCK_AGE")
+ echo "down #reptracker_app block over a minute old ($age_string)"
+ exit 3
+fi
+
HIVEMIND_LAST_IMPORTED_BLOCK_AGE=$(psql "$POSTGRES_URL" --quiet --no-align --tuples-only --command="select extract('epoch' from hive.get_app_current_block_age('hivemind_app'))::integer")
if [ "$HIVEMIND_LAST_IMPORTED_BLOCK_AGE" -gt 60 ]; then
age_string=$(format_seconds "$HIVEMIND_LAST_IMPORTED_BLOCK_AGE")
--
GitLab
From 4fac68e0aaf3872cc28fbd964cd285e0ade5c033 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sun, 17 Nov 2024 20:20:44 +0000
Subject: [PATCH 34/78] Fix haproxy non-agent healthchecks to test something
that requires both the postgrest-rewriter and the postgrest instance to be
working. Our previous checks just tested whether the rewriter was working,
and it could still succeed even when the postgrest instance it was proxying
was down.
---
haproxy/haproxy.cfg | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg
index 2e7b792..0768183 100644
--- a/haproxy/haproxy.cfg
+++ b/haproxy/haproxy.cfg
@@ -82,14 +82,29 @@ backend hived-http
backend balance-tracker
balance roundrobin
+ # balance tracker doesn't have a cheap API method we can call, so test whether we can get
+ # the OpenAPI JSON
+ option httpchk
+ http-check connect
+ http-check send meth HEAD uri /
+ http-check expect status 200
+ timeout check 100
server balance-tracker balance-tracker-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7004 agent-inter 10s
backend reputation-tracker
balance roundrobin
+ option httpchk
+ http-check connect
+ http-check send meth GET uri /last-synced-block
+ http-check expect status 200
server reputation-tracker reputation-tracker-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7009 agent-inter 10s
backend hafah
balance roundrobin
+ option httpchk
+ http-check connect
+ http-check send meth GET uri /version
+ http-check expect status 200
server hafah-postgrest hafah-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7003 agent-inter 10s
backend hivemind
@@ -98,6 +113,10 @@ backend hivemind
backend block-explorer
balance roundrobin
+ option httpchk
+ http-check connect
+ http-check send meth GET uri /last-synced-block
+ http-check expect status 200
server block-explorer block-explorer-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7005 agent-inter 10s
backend sync-hived
--
GitLab
From bf81e0a42d8888f11e71cb9557e685ee31599cd8 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sat, 9 Nov 2024 22:46:32 +0000
Subject: [PATCH 35/78] Remove rewrite rules from haf_api_node, use docker
containers built in the other repositories instead
Resolves #15
---
.env.example | 2 +-
balance_tracker.yaml | 11 +--
balance_tracker/balance_tracker_nginx.conf | 32 --------
balance_tracker/rewrite_rules.conf | 11 ---
haf_block_explorer.yaml | 11 +--
.../haf_block_explorer_nginx.conf | 82 -------------------
haf_block_explorer/rewrite_rules.conf | 47 -----------
hafah.yaml | 11 +--
hafah_rest/hafah_rest_nginx.conf | 82 -------------------
hafah_rest/rewrite_rules.conf | 47 -----------
reputation_tracker.yaml | 11 +--
.../reputation_tracker_nginx.conf | 32 --------
reputation_tracker/rewrite_rules.conf | 11 ---
13 files changed, 5 insertions(+), 385 deletions(-)
delete mode 100644 balance_tracker/balance_tracker_nginx.conf
delete mode 100644 balance_tracker/rewrite_rules.conf
delete mode 100644 haf_block_explorer/haf_block_explorer_nginx.conf
delete mode 100644 haf_block_explorer/rewrite_rules.conf
delete mode 100644 hafah_rest/hafah_rest_nginx.conf
delete mode 100644 hafah_rest/rewrite_rules.conf
delete mode 100644 reputation_tracker/reputation_tracker_nginx.conf
delete mode 100644 reputation_tracker/rewrite_rules.conf
diff --git a/.env.example b/.env.example
index eeb7ad1..9745cc5 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.6rc8
+HIVE_API_NODE_VERSION=1.27.6rc9
# Global settings
diff --git a/balance_tracker.yaml b/balance_tracker.yaml
index 9163d3f..b9882d0 100644
--- a/balance_tracker.yaml
+++ b/balance_tracker.yaml
@@ -83,20 +83,11 @@ services:
haf:
condition: service_healthy
balance-tracker-postgrest-rewriter:
- image: nginx
+ image: ${BALANCE_TRACKER_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker/postgrest-rewriter}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- balance-tracker-standalone
- hafbe
- volumes:
- - type: bind
- source: balance_tracker/balance_tracker_nginx.conf
- target: /etc/nginx/nginx.conf
- read_only: true
- - type: bind
- source: balance_tracker/rewrite_rules.conf
- target: /etc/nginx/rewrite_rules.conf
- read_only: true
depends_on:
balance-tracker-postgrest:
condition: service_healthy
diff --git a/balance_tracker/balance_tracker_nginx.conf b/balance_tracker/balance_tracker_nginx.conf
deleted file mode 100644
index e4e44ea..0000000
--- a/balance_tracker/balance_tracker_nginx.conf
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Homepage and endpoints of the API "HAF Block Explorer".
-#
-worker_processes 5;
-error_log /dev/stdout info;
-worker_rlimit_nofile 8192;
-
-events {
- worker_connections 4096;
-}
-http {
- access_log /dev/stdout;
- server {
- listen 0.0.0.0:80 default_server;
- server_name _;
-
- location / {
- include rewrite_rules.conf;
- rewrite_log on;
-
- proxy_pass http://balance-tracker-postgrest:3000; # my PostREST is here!
-
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- #default_type;
- proxy_hide_header Content-Location;
- proxy_set_header Connection "";
- proxy_http_version 1.1;
- }
- }
-}
diff --git a/balance_tracker/rewrite_rules.conf b/balance_tracker/rewrite_rules.conf
deleted file mode 100644
index ff134bf..0000000
--- a/balance_tracker/rewrite_rules.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-rewrite ^/accounts/([^/]+)/balance-history /rpc/get_balance_history?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/balance-history
-
-rewrite ^/accounts/([^/]+)/balances /rpc/get_account_balances?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/balances
-
-rewrite ^/$ / break;
-# endpoint for openapi spec itself
-
-rewrite ^/(.*)$ /rpc/$1 break;
-# default endpoint for everything else
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 743de78..f79157b 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -73,19 +73,10 @@ services:
haf:
condition: service_healthy
block-explorer-postgrest-rewriter:
- image: openresty/openresty:alpine
+ image: ${HAF_BLOCK_EXPLORER_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer/postgrest-rewriter}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
- volumes:
- - type: bind
- source: haf_block_explorer/haf_block_explorer_nginx.conf
- target: /usr/local/openresty/nginx/conf/nginx.conf
- read_only: true
- - type: bind
- source: haf_block_explorer/rewrite_rules.conf
- target: /usr/local/openresty/nginx/conf/rewrite_rules.conf
- read_only: true
depends_on:
block-explorer-postgrest:
condition: service_healthy
diff --git a/haf_block_explorer/haf_block_explorer_nginx.conf b/haf_block_explorer/haf_block_explorer_nginx.conf
deleted file mode 100644
index 20c2f4d..0000000
--- a/haf_block_explorer/haf_block_explorer_nginx.conf
+++ /dev/null
@@ -1,82 +0,0 @@
-#
-# Homepage and endpoints of the API "HAF Block Explorer".
-#
-worker_processes 5;
-error_log /dev/stdout info;
-worker_rlimit_nofile 8192;
-
-events {
- worker_connections 4096;
-}
-
-http {
- access_log /dev/stdout;
- # Initialize Lua and load functions here
- init_by_lua_block {
- -- Base64 encoding function
- local b = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
-
- function enc(data)
- return ((data:gsub('.', function(x)
- local r, b = '', x:byte()
- for i = 8, 1, -1 do
- r = r .. (b % 2^i - b % 2^(i - 1) > 0 and '1' or '0')
- end
- return r
- end)..'0000'):gsub('%d%d%d?%d?%d?%d?', function(x)
- if (#x < 6) then return '' end
- local c = 0
- for i = 1, 6 do
- c = c + (x:sub(i, i) == '1' and 2^(6 - i) or 0)
- end
- return b:sub(c + 1, c + 1)
- end)..({ '', '==', '=' })[#data % 3 + 1])
- end
-
- -- Process path filters
- function process_path_filters(args)
- local path_filters = {}
-
- for path_filter in args:gmatch("path%-filter=([^&]*)") do
- table.insert(path_filters, enc(ngx.unescape_uri(path_filter)))
- end
- if #path_filters > 0 then
- return "{" .. table.concat(path_filters, ",") .. "}"
- else
- return "{}"
- end
- end
- }
-
- server {
- listen 0.0.0.0:80 default_server;
- server_name _;
-
- location / {
- # Set the path_filters variable
- set_by_lua_block $path_filters {
- local args = ngx.var.args or ""
- return process_path_filters(args)
- }
-
- # Modify args to remove path-filters
- set_by_lua_block $args {
- local args = ngx.var.args or ""
- return args:gsub("&?path%-filter=[^&]*", "")
- }
-
- # Include rewrite rules after setting variables
- include rewrite_rules.conf;
- rewrite_log on;
-
- proxy_pass http://block-explorer-postgrest:3000;
-
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_hide_header Content-Location;
- proxy_set_header Connection "";
- proxy_http_version 1.1;
- }
- }
-}
diff --git a/haf_block_explorer/rewrite_rules.conf b/haf_block_explorer/rewrite_rules.conf
deleted file mode 100644
index 95b5810..0000000
--- a/haf_block_explorer/rewrite_rules.conf
+++ /dev/null
@@ -1,47 +0,0 @@
-rewrite ^/operation-type-counts /rpc/get_latest_blocks break;
-# endpoint for get /operation-type-counts
-
-rewrite ^/input-type/([^/]+) /rpc/get_input_type?input-value=$1 break;
-# endpoint for get /input-type/{input-value}
-
-rewrite ^/last-synced-block /rpc/get_hafbe_last_synced_block break;
-# endpoint for get /last-synced-block
-
-rewrite ^/version /rpc/get_hafbe_version break;
-# endpoint for get /version
-
-rewrite ^/block-numbers /rpc/get_block_by_op?path-filter=$path_filters break;
-# endpoint for get /block-numbers
-
-rewrite ^/accounts/([^/]+)/operations/comments/([^/]+) /rpc/get_comment_operations?account-name=$1&permlink=$2 break;
-# endpoint for get /accounts/{account-name}/operations/comments/{permlink}
-
-rewrite ^/accounts/([^/]+)/comment-permlinks /rpc/get_comment_permlinks?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/comment-permlinks
-
-rewrite ^/accounts/([^/]+)/authority /rpc/get_account_authority?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/authority
-
-rewrite ^/accounts/([^/]+) /rpc/get_account?account-name=$1 break;
-# endpoint for get /accounts/{account-name}
-
-rewrite ^/witnesses/([^/]+)/votes/history /rpc/get_witness_votes_history?account-name=$1 break;
-# endpoint for get /witnesses/{account-name}/votes/history
-
-rewrite ^/witnesses/([^/]+)/voters/count /rpc/get_witness_voters_num?account-name=$1 break;
-# endpoint for get /witnesses/{account-name}/voters/count
-
-rewrite ^/witnesses/([^/]+)/voters /rpc/get_witness_voters?account-name=$1 break;
-# endpoint for get /witnesses/{account-name}/voters
-
-rewrite ^/witnesses/([^/]+) /rpc/get_witness?account-name=$1 break;
-# endpoint for get /witnesses/{account-name}
-
-rewrite ^/witnesses /rpc/get_witnesses break;
-# endpoint for get /witnesses
-
-rewrite ^/$ / break;
-# endpoint for openapi spec itself
-
-rewrite ^/(.*)$ /rpc/$1 break;
-# default endpoint for everything else
diff --git a/hafah.yaml b/hafah.yaml
index 55f9041..c0f6608 100644
--- a/hafah.yaml
+++ b/hafah.yaml
@@ -58,20 +58,11 @@ services:
haf:
condition: service_healthy
hafah-postgrest-rewriter:
- image: openresty/openresty:alpine
+ image: ${HAFAH_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hafah/postgrest-rewriter}:${HAFAH_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafah
- hafbe
- volumes:
- - type: bind
- source: hafah_rest/hafah_rest_nginx.conf
- target: /usr/local/openresty/nginx/conf/nginx.conf
- read_only: true
- - type: bind
- source: hafah_rest/rewrite_rules.conf
- target: /usr/local/openresty/nginx/conf/rewrite_rules.conf
- read_only: true
depends_on:
hafah-postgrest:
condition: service_healthy
diff --git a/hafah_rest/hafah_rest_nginx.conf b/hafah_rest/hafah_rest_nginx.conf
deleted file mode 100644
index de51c4b..0000000
--- a/hafah_rest/hafah_rest_nginx.conf
+++ /dev/null
@@ -1,82 +0,0 @@
-#
-# Homepage and endpoints of the API "HAF Block Explorer".
-#
-worker_processes 5;
-error_log /dev/stdout info;
-worker_rlimit_nofile 8192;
-
-events {
- worker_connections 4096;
-}
-
-http {
- access_log /dev/stdout;
- # Initialize Lua and load functions here
- init_by_lua_block {
- -- Base64 encoding function
- local b = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
-
- function enc(data)
- return ((data:gsub('.', function(x)
- local r, b = '', x:byte()
- for i = 8, 1, -1 do
- r = r .. (b % 2^i - b % 2^(i - 1) > 0 and '1' or '0')
- end
- return r
- end)..'0000'):gsub('%d%d%d?%d?%d?%d?', function(x)
- if (#x < 6) then return '' end
- local c = 0
- for i = 1, 6 do
- c = c + (x:sub(i, i) == '1' and 2^(6 - i) or 0)
- end
- return b:sub(c + 1, c + 1)
- end)..({ '', '==', '=' })[#data % 3 + 1])
- end
-
- -- Process path filters
- function process_path_filters(args)
- local path_filters = {}
-
- for path_filter in args:gmatch("path%-filter=([^&]*)") do
- table.insert(path_filters, enc(ngx.unescape_uri(path_filter)))
- end
- if #path_filters > 0 then
- return "{" .. table.concat(path_filters, ",") .. "}"
- else
- return "{}"
- end
- end
- }
-
- server {
- listen 0.0.0.0:80 default_server;
- server_name _;
-
- location / {
- # Set the path_filters variable
- set_by_lua_block $path_filters {
- local args = ngx.var.args or ""
- return process_path_filters(args)
- }
-
- # Modify args to remove path-filters
- set_by_lua_block $args {
- local args = ngx.var.args or ""
- return args:gsub("&?path%-filter=[^&]*", "")
- }
-
- # Include rewrite rules after setting variables
- include rewrite_rules.conf;
- rewrite_log on;
-
- proxy_pass http://hafah-postgrest:3000; # my PostREST is here!
-
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_hide_header Content-Location;
- proxy_set_header Connection "";
- proxy_http_version 1.1;
- }
- }
-}
diff --git a/hafah_rest/rewrite_rules.conf b/hafah_rest/rewrite_rules.conf
deleted file mode 100644
index 6b6b8a6..0000000
--- a/hafah_rest/rewrite_rules.conf
+++ /dev/null
@@ -1,47 +0,0 @@
-rewrite ^/global-state /rpc/get_global_state break;
-# endpoint for get /global-state
-
-rewrite ^/headblock /rpc/get_head_block_num break;
-# endpoint for get /headblock
-
-rewrite ^/version /rpc/get_version break;
-# endpoint for get /version
-
-rewrite ^/accounts/([^/]+)/operation-types /rpc/get_acc_op_types?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/operation-types
-
-rewrite ^/accounts/([^/]+)/operations /rpc/get_ops_by_account?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/operations
-
-rewrite ^/transactions/([^/]+) /rpc/get_transaction?transaction-id=$1 break;
-# endpoint for get /transactions/{transaction-id}
-
-rewrite ^/operation-types/([^/]+)/keys /rpc/get_operation_keys?type-id=$1 break;
-# endpoint for get /operation-types/{type-id}/keys
-
-rewrite ^/operation-types /rpc/get_op_types break;
-# endpoint for get /operation-types
-
-rewrite ^/operations/([^/]+) /rpc/get_operation?operation-id=$1 break;
-# endpoint for get /operations/{operation-id}
-
-rewrite ^/operations /rpc/get_operations break;
-# endpoint for get /operations
-
-rewrite ^/blocks/([^/]+)/operations /rpc/get_ops_by_block_paging?block-num=$1&path-filter=$path_filters break;
-# endpoint for get /blocks/{block-num}/operations
-
-rewrite ^/blocks/([^/]+)/header /rpc/get_block_header?block-num=$1 break;
-# endpoint for get /blocks/{block-num}/header
-
-rewrite ^/blocks/([^/]+) /rpc/get_block?block-num=$1 break;
-# endpoint for get /blocks/{block-num}
-
-rewrite ^/blocks /rpc/get_block_range break;
-# endpoint for get /blocks
-
-rewrite ^/$ / break;
-# endpoint for openapi spec itself
-
-rewrite ^/(.*)$ /rpc/$1 break;
-# default endpoint for everything else
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index 9824fc1..a1bfc3e 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -85,21 +85,12 @@ services:
haf:
condition: service_healthy
reputation-tracker-postgrest-rewriter:
- image: nginx
+ image: ${REPUTATION_TRACKER_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer/postgrest-rewriter}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- reputation-tracker-standalone
- hafbe
- hivemind
- volumes:
- - type: bind
- source: reputation_tracker/reputation_tracker_nginx.conf
- target: /etc/nginx/nginx.conf
- read_only: true
- - type: bind
- source: reputation_tracker/rewrite_rules.conf
- target: /etc/nginx/rewrite_rules.conf
- read_only: true
depends_on:
reputation-tracker-postgrest:
condition: service_healthy
diff --git a/reputation_tracker/reputation_tracker_nginx.conf b/reputation_tracker/reputation_tracker_nginx.conf
deleted file mode 100644
index 5001753..0000000
--- a/reputation_tracker/reputation_tracker_nginx.conf
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Homepage and endpoints of the API "HAF Block Explorer".
-#
-worker_processes 5;
-error_log /dev/stdout info;
-worker_rlimit_nofile 8192;
-
-events {
- worker_connections 4096;
-}
-http {
- access_log /dev/stdout;
- server {
- listen 0.0.0.0:80 default_server;
- server_name _;
-
- location / {
- include rewrite_rules.conf;
- rewrite_log on;
-
- proxy_pass http://reputation-tracker-postgrest:3000; # my PostREST is here!
-
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- #default_type;
- proxy_hide_header Content-Location;
- proxy_set_header Connection "";
- proxy_http_version 1.1;
- }
- }
-}
diff --git a/reputation_tracker/rewrite_rules.conf b/reputation_tracker/rewrite_rules.conf
deleted file mode 100644
index e9c56f2..0000000
--- a/reputation_tracker/rewrite_rules.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-rewrite ^/last-synced-block /rpc/get_rep_last_synced_block break;
-# endpoint for get /last-synced-block
-
-rewrite ^/accounts/([^/]+)/reputation /rpc/get_account_reputation?account-name=$1 break;
-# endpoint for get /accounts/{account-name}/reputation
-
-rewrite ^/$ / break;
-# endpoint for openapi spec itself
-
-rewrite ^/(.*)$ /rpc/$1 break;
-# default endpoint for everything else
--
GitLab
From c507b9fd95050c5cb1092fdcc5d808f02fe4a1e3 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 12 Nov 2024 23:49:59 +0000
Subject: [PATCH 36/78] Split reptracker installer to allow hivemind to start
processing blocks before the slow index-creation process begins
---
.env.example | 4 ++--
haf_base.yaml | 3 ++-
haf_block_explorer.yaml | 2 +-
hivemind.yaml | 6 +++---
reputation_tracker.yaml | 27 ++++++++++++++++++++++++---
5 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/.env.example b/.env.example
index 9745cc5..ec1a042 100644
--- a/.env.example
+++ b/.env.example
@@ -117,8 +117,8 @@ NETWORK_NAME=haf
# REPUTATION_TRACKER_ADDON
-# HAF_REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
-# HAF_REPUTATION_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
+# REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
+# REPUTATION_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
# REPTRACKER_SCHEMA="hafbe-rt"
diff --git a/haf_base.yaml b/haf_base.yaml
index 8d17cb0..4c5ecc2 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -27,7 +27,8 @@ services:
host haf_block_log btracker_owner ${COMPOSE_PROJECT_NAME}-balance-tracker-block-processing-1.${NETWORK_NAME} trust\n
host haf_block_log btracker_user ${COMPOSE_PROJECT_NAME}-balance-tracker-postgrest-1.${NETWORK_NAME} trust\n
- host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-install-1.${NETWORK_NAME} trust\n
+ host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-install-schema-1.${NETWORK_NAME} trust\n
+ host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-install-indexes-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log reptracker_owner ${COMPOSE_PROJECT_NAME}-reputation-tracker-block-processing-1.${NETWORK_NAME} trust\n
host haf_block_log reptracker_user ${COMPOSE_PROJECT_NAME}-reputation-tracker-postgrest-1.${NETWORK_NAME} trust\n
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index f79157b..14a91b1 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -18,7 +18,7 @@ services:
condition: service_completed_successfully
hafah-install:
condition: service_completed_successfully
- reputation-tracker-install:
+ reputation-tracker-install-indexes:
condition: service_completed_successfully
block-explorer-uninstall:
image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
diff --git a/hivemind.yaml b/hivemind.yaml
index 3391440..9f0283f 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -20,8 +20,8 @@ services:
depends_on:
haf:
condition: service_healthy
- # reputation-tracker-install:
-# condition: service_completed_successfully
+ reputation-tracker-install-schema:
+ condition: service_completed_successfully
hivemind-server:
image: ${HIVEMIND_INSTANCE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
@@ -55,7 +55,7 @@ services:
depends_on:
haf:
condition: service_healthy
- reputation-tracker-install:
+ reputation-tracker-install-schema:
condition: service_completed_successfully
hivemind-uninstall:
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index a1bfc3e..f14926c 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -1,5 +1,5 @@
services:
- reputation-tracker-install:
+ reputation-tracker-install-schema:
image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
@@ -13,9 +13,30 @@ services:
command:
- install_app
- --swagger-url=${PUBLIC_HOSTNAME}
+ - --schema-only
depends_on:
haf:
condition: service_healthy
+ reputation-tracker-install-indexes:
+ image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ profiles:
+ - apps
+ - hafbe
+ - hivemind
+ - reputation-tracker-setup
+ environment:
+ POSTGRES_HOST: haf
+ networks:
+ haf-network:
+ command:
+ - install_app
+ - --swagger-url=${PUBLIC_HOSTNAME}
+ - --indexes-only
+ depends_on:
+ haf:
+ condition: service_healthy
+ reputation-tracker-install-schema:
+ condition: service_completed_successfully
reputation-tracker-uninstall:
image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
@@ -50,7 +71,7 @@ services:
retries: 3
start_period: 48h
depends_on:
- reputation-tracker-install:
+ reputation-tracker-install-indexes:
condition: service_completed_successfully
haf:
condition: service_healthy
@@ -80,7 +101,7 @@ services:
networks:
haf-network:
depends_on:
- reputation-tracker-install:
+ reputation-tracker-install-schema:
condition: service_completed_successfully
haf:
condition: service_healthy
--
GitLab
From 7ca952903f67880b59e7b0de97247a8f7358dd92 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 14 Nov 2024 18:29:59 -0500
Subject: [PATCH 37/78] Change block-explorer install so it only requires
reputation-reputation-tracker's schema (it doesn't need the indexes to
install itself).
---
haf_block_explorer.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 14a91b1..51da194 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -18,7 +18,7 @@ services:
condition: service_completed_successfully
hafah-install:
condition: service_completed_successfully
- reputation-tracker-install-indexes:
+ reputation-tracker-install-schema:
condition: service_completed_successfully
block-explorer-uninstall:
image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
--
GitLab
From 1a92f421ef706f22786d0c4482bce0b253b759f0 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 22 Nov 2024 14:27:14 -0500
Subject: [PATCH 38/78] Add script to update .env file with git hashes
---
use_develop_env.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 68 insertions(+)
create mode 100644 use_develop_env.py
diff --git a/use_develop_env.py b/use_develop_env.py
new file mode 100644
index 0000000..d71f590
--- /dev/null
+++ b/use_develop_env.py
@@ -0,0 +1,68 @@
+import os
+import subprocess
+
+def get_git_revision(repo_path):
+ try:
+ repo_url = subprocess.check_output(
+ ["git", "-C", repo_path, "config", "--get", "remote.origin.url"],
+ stderr=subprocess.STDOUT
+ ).strip().decode('utf-8')
+
+ git_hash = subprocess.check_output(
+ ["git", "-C", repo_path, "rev-parse", "HEAD"],
+ stderr=subprocess.STDOUT
+ ).strip().decode('utf-8')[:8] # Use only the first 8 characters of the hash
+
+ return repo_url, git_hash
+ except subprocess.CalledProcessError:
+ return None, None
+
+def update_env_file(env_file_path, repo_name, git_hash):
+ updated = False
+ with open(env_file_path, "r") as file:
+ lines = file.readlines()
+
+ with open(env_file_path, "w") as file:
+ for line in lines:
+ if line.startswith(f"{repo_name.upper()}_VERSION="):
+ file.write(f"{repo_name.upper()}_VERSION={git_hash}\n")
+ updated = True
+ print(f"Updated {repo_name.upper()}_VERSION to {git_hash} in .env file")
+ else:
+ file.write(line)
+
+ if not updated:
+ with open(env_file_path, "a") as file:
+ file.write(f"{repo_name.upper()}_VERSION={git_hash}\n")
+ print(f"Added {repo_name.upper()}_VERSION={git_hash} to .env file")
+
+def main(directory):
+ env_file_path = None
+ for item in os.listdir(directory):
+ item_path = os.path.join(directory, item)
+ if os.path.isdir(item_path) and item == "haf_api_node":
+ env_file_path = os.path.join(item_path, ".env")
+ break
+
+ if not env_file_path or not os.path.exists(env_file_path):
+ print("Error: .env file not found in haf_api_node repo.")
+ return
+
+ with open("repo_versions.txt", "w") as output_file:
+ for item in os.listdir(directory):
+ item_path = os.path.join(directory, item)
+ if os.path.isdir(item_path) and ".git" in os.listdir(item_path):
+ repo_url, git_hash = get_git_revision(item_path)
+ if repo_url and git_hash:
+ output_file.write(f"{repo_url}, {git_hash}\n")
+ print(f"Found repo: {repo_url} at {item_path}")
+ repo_name = os.path.basename(item_path)
+ if repo_name != "hive":
+ update_env_file(env_file_path, repo_name, git_hash)
+
+if __name__ == "__main__":
+ import sys
+ if len(sys.argv) != 2:
+ print("Usage: python use_develop_env.py <directory>")
+ else:
+ main(sys.argv[1])
--
GitLab
From 46f088e6f197c8f564120f6e4aba51937f74e02e Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 19 Nov 2024 19:57:22 +0000
Subject: [PATCH 39/78] Re-sync the log-rotation files with the main service
definitions
---
.../balance_tracker.log_rotation.yaml | 2 +-
log_rotation/caddy.log_rotation.yaml | 2 +-
log_rotation/compose.log_rotation.yml | 5 +++-
log_rotation/drone.log_rotation.yaml | 2 +-
log_rotation/haf_base.log_rotation.yaml | 2 +-
.../haf_block_explorer.log_rotation.yaml | 7 +++--
log_rotation/hafah.log_rotation.yaml | 5 +++-
log_rotation/haproxy.log_rotation.yaml | 9 +++---
log_rotation/hivemind.log_rotation.yaml | 4 +--
log_rotation/jussi.log_rotation.yaml | 2 +-
log_rotation/monitoring.log_rotation.yaml | 28 +++++++++++++++++++
.../reputation_tracker.log_rotation.yaml | 19 +++++++++++++
log_rotation/swagger.log_rotation.yaml | 4 +++
log_rotation/varnish.log_rotation.yaml | 2 +-
14 files changed, 77 insertions(+), 16 deletions(-)
create mode 100644 log_rotation/monitoring.log_rotation.yaml
create mode 100644 log_rotation/reputation_tracker.log_rotation.yaml
create mode 100644 log_rotation/swagger.log_rotation.yaml
diff --git a/log_rotation/balance_tracker.log_rotation.yaml b/log_rotation/balance_tracker.log_rotation.yaml
index c142d55..6acdddb 100644
--- a/log_rotation/balance_tracker.log_rotation.yaml
+++ b/log_rotation/balance_tracker.log_rotation.yaml
@@ -11,6 +11,6 @@ services:
balance-tracker-postgrest:
logging:
driver: local
- balance-tracker-swagger:
+ balance-tracker-postgrest-rewriter:
logging:
driver: local
diff --git a/log_rotation/caddy.log_rotation.yaml b/log_rotation/caddy.log_rotation.yaml
index 8a282f4..8998b07 100644
--- a/log_rotation/caddy.log_rotation.yaml
+++ b/log_rotation/caddy.log_rotation.yaml
@@ -3,7 +3,7 @@ services:
logging:
driver: local
options:
- max-size: "100m"
+ max-size: 100m
max-file: 10
version-display:
logging:
diff --git a/log_rotation/compose.log_rotation.yml b/log_rotation/compose.log_rotation.yml
index 78f7933..0f5628d 100644
--- a/log_rotation/compose.log_rotation.yml
+++ b/log_rotation/compose.log_rotation.yml
@@ -2,10 +2,13 @@ include:
- log_rotation/backend.log_rotation.yaml
- log_rotation/balance_tracker.log_rotation.yaml
- log_rotation/caddy.log_rotation.yaml
- - log_rotation/${JSONRPC_API_SERVER_NAME:-jussi}.log_rotation.yaml
+ - log_rotation/${JSONRPC_API_SERVER_NAME:-drone}.log_rotation.yaml
- log_rotation/haf_base.log_rotation.yaml
- log_rotation/haf_block_explorer.log_rotation.yaml
- log_rotation/hafah.log_rotation.yaml
- log_rotation/haproxy.log_rotation.yaml
- log_rotation/hivemind.log_rotation.yaml
- log_rotation/varnish.log_rotation.yaml
+ - log_rotation/monitoring.log_rotation.yaml
+ - log_rotation/reputation_tracker.log_rotation.yaml
+ - log_rotation/swagger.log_rotation.yaml
diff --git a/log_rotation/drone.log_rotation.yaml b/log_rotation/drone.log_rotation.yaml
index 39872c1..95df8b0 100644
--- a/log_rotation/drone.log_rotation.yaml
+++ b/log_rotation/drone.log_rotation.yaml
@@ -3,5 +3,5 @@ services:
logging:
driver: local
options:
- max-size: "1000m"
+ max-size: 1000m
max-file: 50
diff --git a/log_rotation/haf_base.log_rotation.yaml b/log_rotation/haf_base.log_rotation.yaml
index ec65db8..b7b9d80 100644
--- a/log_rotation/haf_base.log_rotation.yaml
+++ b/log_rotation/haf_base.log_rotation.yaml
@@ -3,7 +3,7 @@ services:
logging:
driver: local
options:
- max-size: "100m"
+ max-size: 100m
max-file: 10
logrotate:
logging:
diff --git a/log_rotation/haf_block_explorer.log_rotation.yaml b/log_rotation/haf_block_explorer.log_rotation.yaml
index 02990f6..b827921 100644
--- a/log_rotation/haf_block_explorer.log_rotation.yaml
+++ b/log_rotation/haf_block_explorer.log_rotation.yaml
@@ -3,7 +3,7 @@ services:
logging:
driver: local
options:
- max-size: "100m"
+ max-size: 100m
max-file: 10
block-explorer-install:
logging:
@@ -12,7 +12,7 @@ services:
logging:
driver: local
options:
- max-size: "100m"
+ max-size: 100m
max-file: 10
block-explorer-uninstall:
logging:
@@ -20,3 +20,6 @@ services:
block-explorer-uninstall-balance-tracker:
logging:
driver: local
+ block-explorer-postgrest-rewriter:
+ logging:
+ driver: local
diff --git a/log_rotation/hafah.log_rotation.yaml b/log_rotation/hafah.log_rotation.yaml
index f049189..f8e60b7 100644
--- a/log_rotation/hafah.log_rotation.yaml
+++ b/log_rotation/hafah.log_rotation.yaml
@@ -3,7 +3,7 @@ services:
logging:
driver: local
options:
- max-size: "100m"
+ max-size: 100m
max-file: 10
hafah-benchmarks-old-style:
logging:
@@ -14,3 +14,6 @@ services:
hafah-uninstall:
logging:
driver: local
+ hafah-postgrest-rewriter:
+ logging:
+ driver: local
diff --git a/log_rotation/haproxy.log_rotation.yaml b/log_rotation/haproxy.log_rotation.yaml
index c95ba96..509e3ee 100644
--- a/log_rotation/haproxy.log_rotation.yaml
+++ b/log_rotation/haproxy.log_rotation.yaml
@@ -1,6 +1,7 @@
services:
+ haproxy:
+ logging:
+ driver: local
haproxy-healthchecks:
- # expose the healthchecks so we can connect to them from an external haproxy
- # shift the ports by 20000 so they don't conflict with haproxy's (if we want to expose those also)
- ports:
- - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_HEALTHCHECK_PORT_PREFIX:-2}7001-${HAF_API_NODE_EXPOSED_HEALTHCHECK_PORT_PREFIX:-2}7005:7001-7005"
+ logging:
+ driver: local
diff --git a/log_rotation/hivemind.log_rotation.yaml b/log_rotation/hivemind.log_rotation.yaml
index 8a42173..9c37aa0 100644
--- a/log_rotation/hivemind.log_rotation.yaml
+++ b/log_rotation/hivemind.log_rotation.yaml
@@ -3,13 +3,13 @@ services:
logging:
driver: local
options:
- max-size: "100m"
+ max-size: 100m
max-file: 10
hivemind-server:
logging:
driver: local
options:
- max-size: "100m"
+ max-size: 100m
max-file: 10
hivemind-install:
logging:
diff --git a/log_rotation/jussi.log_rotation.yaml b/log_rotation/jussi.log_rotation.yaml
index a7ac040..fdebbbe 100644
--- a/log_rotation/jussi.log_rotation.yaml
+++ b/log_rotation/jussi.log_rotation.yaml
@@ -3,7 +3,7 @@ services:
logging:
driver: local
options:
- max-size: "1000m"
+ max-size: 1000m
max-file: 5
redis:
logging:
diff --git a/log_rotation/monitoring.log_rotation.yaml b/log_rotation/monitoring.log_rotation.yaml
new file mode 100644
index 0000000..2f80dca
--- /dev/null
+++ b/log_rotation/monitoring.log_rotation.yaml
@@ -0,0 +1,28 @@
+services:
+ prometheus:
+ logging:
+ driver: local
+ grafana:
+ logging:
+ driver: local
+ cadvisor:
+ logging:
+ driver: local
+ blackboxexporter:
+ logging:
+ driver: local
+ postgresexporter:
+ logging:
+ driver: local
+ promtail:
+ logging:
+ driver: local
+ hived-pme:
+ logging:
+ driver: local
+ nodeexporter:
+ logging:
+ driver: local
+ loki:
+ logging:
+ driver: local
diff --git a/log_rotation/reputation_tracker.log_rotation.yaml b/log_rotation/reputation_tracker.log_rotation.yaml
new file mode 100644
index 0000000..8a0286f
--- /dev/null
+++ b/log_rotation/reputation_tracker.log_rotation.yaml
@@ -0,0 +1,19 @@
+services:
+ reputation-tracker-uninstall:
+ logging:
+ driver: local
+ reputation-tracker-block-processing:
+ logging:
+ driver: local
+ reputation-tracker-postgrest:
+ logging:
+ driver: local
+ reputation-tracker-postgrest-rewriter:
+ logging:
+ driver: local
+ reputation-tracker-install-schema:
+ logging:
+ driver: local
+ reputation-tracker-install-indexes:
+ logging:
+ driver: local
diff --git a/log_rotation/swagger.log_rotation.yaml b/log_rotation/swagger.log_rotation.yaml
new file mode 100644
index 0000000..87678d8
--- /dev/null
+++ b/log_rotation/swagger.log_rotation.yaml
@@ -0,0 +1,4 @@
+services:
+ swagger:
+ logging:
+ driver: local
diff --git a/log_rotation/varnish.log_rotation.yaml b/log_rotation/varnish.log_rotation.yaml
index d70b165..aefdd9c 100644
--- a/log_rotation/varnish.log_rotation.yaml
+++ b/log_rotation/varnish.log_rotation.yaml
@@ -3,5 +3,5 @@ services:
logging:
driver: local
options:
- max-size: "100m"
+ max-size: 100m
max-file: 10
--
GitLab
From a675ce29f22af7b07a1b086d1e4bc0c73b1e796d Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 20 Nov 2024 16:38:27 +0000
Subject: [PATCH 40/78] Bump versions of pgadmin, pghero, varnish
---
backend.yaml | 4 ++--
varnish.yaml | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/backend.yaml b/backend.yaml
index 156558c..a17e69d 100644
--- a/backend.yaml
+++ b/backend.yaml
@@ -1,6 +1,6 @@
services:
pghero:
- image: ankane/pghero:v3.3.3
+ image: ankane/pghero:v3.6.1
profiles:
- admin
environment:
@@ -14,7 +14,7 @@ services:
condition: service_healthy
pgadmin:
- image: dpage/pgadmin4:${PGADMIN_VERSION:-8.12}
+ image: dpage/pgadmin4:${PGADMIN_VERSION:-8.13}
profiles:
- admin
environment:
diff --git a/varnish.yaml b/varnish.yaml
index 2ce101e..348f4ab 100644
--- a/varnish.yaml
+++ b/varnish.yaml
@@ -1,6 +1,6 @@
services:
varnish:
- image: varnish:7.3.0-alpine
+ image: varnish:7.6.1-alpine
profiles:
- servers
volumes:
--
GitLab
From 32b0e72077cbabd5eec95c9f550f35b43fdfe16d Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 20 Nov 2024 20:16:39 +0000
Subject: [PATCH 41/78] Split block explorer install to separate schema and
index creation. This allows us to prevent reptracker and hafbe index creation
from causing deadlock while still beginning hafbe block processing as soon as
possible. hive/haf_api_node#21
---
.gitlab-ci.yml | 2 +-
haf_base.yaml | 6 ++--
haf_block_explorer.yaml | 32 ++++++++++++++++---
.../haf_block_explorer.log_rotation.yaml | 9 ++++--
reputation_tracker.yaml | 2 +-
5 files changed, 40 insertions(+), 11 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c24fdf9..6df8d2f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
stages:
- publish
-build_exchange_flyway_docker_image:
+build_haproxy_healthchecks_docker_image:
stage: publish
variables:
DOCKER_BUILDKIT: 1
diff --git a/haf_base.yaml b/haf_base.yaml
index 4c5ecc2..986c27f 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -38,8 +38,10 @@ services:
host haf_block_log hafah_user ${COMPOSE_PROJECT_NAME}-hafah-postgrest-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-hafah-benchmarks-1.${NETWORK_NAME} trust\n
- host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-install-1.${NETWORK_NAME} trust\n
- host haf_block_log hafbe_owner ${COMPOSE_PROJECT_NAME}-block-explorer-install-1.${NETWORK_NAME} trust\n
+ host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-install-schema-1.${NETWORK_NAME} trust\n
+ host haf_block_log hafbe_owner ${COMPOSE_PROJECT_NAME}-block-explorer-install-schema-1.${NETWORK_NAME} trust\n
+ host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-install-indexes-1.${NETWORK_NAME} trust\n
+ host haf_block_log hafbe_owner ${COMPOSE_PROJECT_NAME}-block-explorer-install-indexes-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log hafbe_owner ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-balance-tracker-1.${NETWORK_NAME} trust\n
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 51da194..9ecbc4c 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -1,5 +1,5 @@
services:
- block-explorer-install:
+ block-explorer-install-schema:
image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
@@ -9,6 +9,7 @@ services:
command:
- install_app
- --swagger-url=${PUBLIC_HOSTNAME}
+ - --schema-only
networks:
haf-network:
depends_on:
@@ -20,6 +21,30 @@ services:
condition: service_completed_successfully
reputation-tracker-install-schema:
condition: service_completed_successfully
+ block-explorer-install-indexes:
+ image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
+ profiles:
+ - apps
+ - hafbe
+ environment:
+ POSTGRES_HOST: haf
+ command:
+ - install_app
+ - --swagger-url=${PUBLIC_HOSTNAME}
+ - --indexes-only
+ networks:
+ haf-network:
+ depends_on:
+ haf:
+ condition: service_healthy
+ block-explorer-install-schema:
+ condition: service_completed_successfully
+ balance-tracker-install:
+ condition: service_completed_successfully
+ hafah-install:
+ condition: service_completed_successfully
+ reputation-tracker-install-indexes:
+ condition: service_completed_successfully
block-explorer-uninstall:
image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
@@ -68,7 +93,7 @@ services:
retries: 3
start_period: 48h
depends_on:
- block-explorer-install:
+ block-explorer-install-schema:
condition: service_completed_successfully
haf:
condition: service_healthy
@@ -82,7 +107,6 @@ services:
condition: service_healthy
networks:
- haf-network
-
block-explorer-postgrest:
image: ${POSTGREST_REGISTRY:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
@@ -107,7 +131,7 @@ services:
retries: 10
start_period: 1m
depends_on:
- block-explorer-install:
+ block-explorer-install-schema:
condition: service_completed_successfully
haf:
condition: service_healthy
diff --git a/log_rotation/haf_block_explorer.log_rotation.yaml b/log_rotation/haf_block_explorer.log_rotation.yaml
index b827921..a6301b4 100644
--- a/log_rotation/haf_block_explorer.log_rotation.yaml
+++ b/log_rotation/haf_block_explorer.log_rotation.yaml
@@ -5,9 +5,6 @@ services:
options:
max-size: 100m
max-file: 10
- block-explorer-install:
- logging:
- driver: local
block-explorer-postgrest:
logging:
driver: local
@@ -23,3 +20,9 @@ services:
block-explorer-postgrest-rewriter:
logging:
driver: local
+ block-explorer-install-schema:
+ logging:
+ driver: local
+ block-explorer-install-indexes:
+ logging:
+ driver: local
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index f14926c..49981c9 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -106,7 +106,7 @@ services:
haf:
condition: service_healthy
reputation-tracker-postgrest-rewriter:
- image: ${REPUTATION_TRACKER_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer/postgrest-rewriter}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${REPUTATION_TRACKER_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker/postgrest-rewriter}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- reputation-tracker-standalone
--
GitLab
From 63cdb467f79242cc0152019167dd0a9d4fd1c521 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 20 Nov 2024 21:28:36 +0000
Subject: [PATCH 42/78] Add application names to PostgREST instances
---
balance_tracker.yaml | 2 +-
haf_block_explorer.yaml | 2 +-
hafah.yaml | 2 +-
reputation_tracker.yaml | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/balance_tracker.yaml b/balance_tracker.yaml
index b9882d0..05f38de 100644
--- a/balance_tracker.yaml
+++ b/balance_tracker.yaml
@@ -61,7 +61,7 @@ services:
- hafbe
environment:
PGRST_ADMIN_SERVER_PORT: 3001
- PGRST_DB_URI: postgresql://btracker_user@haf/haf_block_log
+ PGRST_DB_URI: postgresql://btracker_user@haf/haf_block_log?application_name=balance_tracker_postgrest
PGRST_DB_SCHEMA: btracker_endpoints
PGRST_DB_ANON_ROLE: btracker_user
PGRST_DB_POOL: 20
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 9ecbc4c..8b036a3 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -114,7 +114,7 @@ services:
- hafbe
environment:
PGRST_ADMIN_SERVER_PORT: 3001
- PGRST_DB_URI: postgresql://hafbe_user@haf/haf_block_log
+ PGRST_DB_URI: postgresql://hafbe_user@haf/haf_block_log?application_name=block_explorer_postgrest
PGRST_DB_SCHEMA: hafbe_endpoints
PGRST_DB_ANON_ROLE: hafbe_user
PGRST_DB_POOL: 20
diff --git a/hafah.yaml b/hafah.yaml
index c0f6608..c2ffc59 100644
--- a/hafah.yaml
+++ b/hafah.yaml
@@ -37,7 +37,7 @@ services:
- hafbe
environment:
PGRST_ADMIN_SERVER_PORT: 3001
- PGRST_DB_URI: postgresql://hafah_user@haf/haf_block_log
+ PGRST_DB_URI: postgresql://hafah_user@haf/haf_block_log?application_name=hafah_postgrest
PGRST_DB_SCHEMA: hafah_endpoints, hafah_api_v1, hafah_api_v2
PGRST_DB_ROOT_SPEC: home
PGRST_DB_ANON_ROLE: hafah_user
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index 49981c9..ae92d09 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -84,7 +84,7 @@ services:
- hafbe
environment:
PGRST_ADMIN_SERVER_PORT: 3001
- PGRST_DB_URI: postgresql://reptracker_user@haf/haf_block_log
+ PGRST_DB_URI: postgresql://reptracker_user@haf/haf_block_log?application_name=reputation_tracker_postgrest
PGRST_DB_SCHEMA: reptracker_endpoints
PGRST_DB_ANON_ROLE: reptracker_user
PGRST_DB_POOL: 20
--
GitLab
From e0cc8c94f36c6768407f47eb2e8392a46fdb1251 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 20 Nov 2024 22:45:59 +0000
Subject: [PATCH 43/78] Add healthchecks for rewriters, haproxy-healthchecks
---
backend.yaml | 12 ++++++++++++
balance_tracker.yaml | 6 ++++++
haf_block_explorer.yaml | 6 ++++++
hafah.yaml | 6 ++++++
haproxy.yaml | 6 ++++++
healthchecks/Dockerfile | 1 +
healthchecks/healthcheck_healthcheck.sh | 2 ++
reputation_tracker.yaml | 6 ++++++
8 files changed, 45 insertions(+)
create mode 100755 healthchecks/healthcheck_healthcheck.sh
diff --git a/backend.yaml b/backend.yaml
index a17e69d..c3560bc 100644
--- a/backend.yaml
+++ b/backend.yaml
@@ -6,6 +6,12 @@ services:
environment:
DATABASE_URL: postgres://pghero@haf:5432/haf_block_log
RAILS_RELATIVE_URL_ROOT: /admin/pghero
+ healthcheck:
+ test: ["CMD-SHELL", "nc -z 127.0.0.1 8080 || exit 1"]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ start_period: 30s
networks:
- haf-network
init: true
@@ -20,6 +26,12 @@ services:
environment:
PGADMIN_DEFAULT_EMAIL: "admin@haf.world"
PGADMIN_DEFAULT_PASSWORD: "admin"
+ healthcheck:
+ test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1/misc/ping || exit 1"]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 1m
networks:
- haf-network
init: true
diff --git a/balance_tracker.yaml b/balance_tracker.yaml
index 05f38de..bbad1dd 100644
--- a/balance_tracker.yaml
+++ b/balance_tracker.yaml
@@ -88,6 +88,12 @@ services:
- apps
- balance-tracker-standalone
- hafbe
+ healthcheck:
+ test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:81/health || exit 1"]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 1m
depends_on:
balance-tracker-postgrest:
condition: service_healthy
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 8b036a3..fab1234 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -102,6 +102,12 @@ services:
profiles:
- apps
- hafbe
+ healthcheck:
+ test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:81/health || exit 1"]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 1m
depends_on:
block-explorer-postgrest:
condition: service_healthy
diff --git a/hafah.yaml b/hafah.yaml
index c2ffc59..d8bccb3 100644
--- a/hafah.yaml
+++ b/hafah.yaml
@@ -63,6 +63,12 @@ services:
- apps
- hafah
- hafbe
+ healthcheck:
+ test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:81/health || exit 1"]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 1m
depends_on:
hafah-postgrest:
condition: service_healthy
diff --git a/haproxy.yaml b/haproxy.yaml
index f63d3a5..cdfb940 100644
--- a/haproxy.yaml
+++ b/haproxy.yaml
@@ -28,6 +28,12 @@ services:
POSTGRES_URL_HAFBE: postgresql://hafbe_user@haf/haf_block_log
POSTGRES_URL_BTRACKER: postgresql://btracker_user@haf/haf_block_log
POSTGRES_URL_REPTRACKER: postgresql://reptracker_user@haf/haf_block_log
+ healthcheck:
+ test: ["CMD-SHELL", "/healthcheck_healthcheck.sh || exit 1"]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 1m
profiles:
- servers
- haproxy-healthchecks
diff --git a/healthchecks/Dockerfile b/healthchecks/Dockerfile
index a79acc5..58a7f9d 100644
--- a/healthchecks/Dockerfile
+++ b/healthchecks/Dockerfile
@@ -10,6 +10,7 @@ COPY checks/hafah.sh /checks/hafah.sh
COPY checks/hafbe.sh /checks/hafbe.sh
COPY checks/format_seconds.sh /checks/format_seconds.sh
COPY checks/check_haf_lib.sh /checks/check_haf_lib.sh
+COPY healthcheck_healthcheck.sh /
COPY docker_entrypoint.sh /
ENTRYPOINT ["/docker_entrypoint.sh"]
diff --git a/healthchecks/healthcheck_healthcheck.sh b/healthchecks/healthcheck_healthcheck.sh
new file mode 100755
index 0000000..cb60479
--- /dev/null
+++ b/healthchecks/healthcheck_healthcheck.sh
@@ -0,0 +1,2 @@
+#! /bin/sh
+#exec netstat -tln | grep -E ':(7001|7002|7003|7004|7005|7009)\b' | wc -l | grep -q '^6$'
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index ae92d09..ceb2357 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -112,6 +112,12 @@ services:
- reputation-tracker-standalone
- hafbe
- hivemind
+ healthcheck:
+ test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:81/health || exit 1"]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 1m
depends_on:
reputation-tracker-postgrest:
condition: service_healthy
--
GitLab
From 45432e138ca41706843e1298b01e238a8e9b35bf Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 21 Nov 2024 00:19:23 +0000
Subject: [PATCH 44/78] Adjust HAfAH image path to account for newly-fixed
image names in CI. Bump the version number to the yet-to-be-tagged rc10
---
.env.example | 13 +++----------
1 file changed, 3 insertions(+), 10 deletions(-)
diff --git a/.env.example b/.env.example
index ec1a042..52353fd 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.6rc9
+HIVE_API_NODE_VERSION=1.27.6rc10
# Global settings
@@ -93,15 +93,8 @@ NETWORK_NAME=haf
# The default setup will run the recommended version of HAfAH,
# you can run a custom version by un-commenting and modifying the
# values below
-
-# when using rc tag
-#HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah
-
-# when using commit hash
-#HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah/setup
-
-# HAFAH_VERSION=9556cd0f
-# HAFAH_VERSION=a85fc0b4
+# HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah
+# HAFAH_VERSION=${HIVE_API_NODE_VERSION}
# The default setup will run the recommended version of Hivemind using the values
# below. You can override them here to run a custom version of Hivemind
--
GitLab
From 2b6db54b07f58e81903a88a7803cb8f9cfa56e21 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 21 Nov 2024 17:14:51 +0000
Subject: [PATCH 45/78] We don't need to pass the app name to
haf_block_explorer's health check anymore
---
haf_block_explorer.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index fab1234..331f5bc 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -87,7 +87,7 @@ services:
networks:
haf-network:
healthcheck:
- test: ["CMD-SHELL","/app/block-processing-healthcheck.sh hafbe_app"]
+ test: ["CMD-SHELL","/app/block-processing-healthcheck.sh"]
interval: 60s
timeout: 10s
retries: 3
--
GitLab
From 68d8071461db8ff522a91ba2c65bcb5b57931f67 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 22 Nov 2024 22:44:16 +0000
Subject: [PATCH 46/78] Add use_develop_env.py script and update README.md with
descriptions of more scripts.
---
README.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++
use_develop_env.py | 1 +
2 files changed, 66 insertions(+)
diff --git a/README.md b/README.md
index a2c71d6..a6e1d19 100644
--- a/README.md
+++ b/README.md
@@ -275,3 +275,68 @@ If you're upgrading to a new version of hivemind:
- run `docker compose pull` to grab the new version
- run `docker compose up -d` to bring up all services. This should run hivemind's install, then launch the block processing container.
- you can monitor Hivemind's sync process by watching the logs from `docker compose logs -f hivemind-block-processing`. In a few short days, your Hivemind app should be fully synced and ready to handle API requests.
+
+# Scripts in the haf_api_node Directory
+
+## use_develop_env.py
+This script updates the `.env` file in the `haf_api_node` repository with the short git hashes of other repositories in the specified directory. It scans the given directory for git repositories, retrieves their remote URLs and short git hashes, and updates the `.env` file accordingly.
+
+Usage:
+```
+python3 use_develop_env.py <path_to_directory>
+```
+
+## make_ramdisk.sh
+This script creates a ramdisk and mounts it to the `/mnt/haf_shared_mem` directory. It sets the size of the ramdisk to 26GB and changes the permissions to allow read/write access for all users.
+
+Usage:
+```
+sudo ./make_ramdisk.sh
+```
+
+## clone_zfs_datasets.sh
+This script clones an existing ZFS dataset to create a new dataset. It is useful for creating backups or duplicating datasets for testing purposes. The script takes the source dataset and the target dataset as arguments and performs the cloning operation.
+
+Usage:
+```
+sudo ./clone_zfs_datasets.sh <source_dataset> <target_dataset>
+```
+Example:
+```markdown
+sudo ./clone_zfs_datasets.sh haf-pool/haf-datadir haf-pool/haf-datadir-test-upgrade
+```
+
+## snapshot_zfs_datasets.sh
+This script creates a ZFS snapshot of the HAF datasets. It unmounts the datasets, takes a snapshot, and then remounts them. It also provides options for handling log files during the snapshot process.
+
+Usage:
+```
+sudo ./snapshot_zfs_datasets.sh [--env-file=filename] [--public-snapshot] [--temp-dir=dir] [--swap-logs-with-dataset=dataset] snapshot-name
+```
+Options:
+- `--env-file=filename`: Specify the environment file to use.
+- `--public-snapshot`: Move log files to /tmp before taking the snapshot, then restore them afterwards.
+- `--temp-dir=dir`: Use a different temp directory (use if /tmp isn't big enough).
+- `--swap-logs-with-dataset=dataset`: Swap the logs dataset with an empty dataset before taking the snapshot, then swap back afterwards.
+
+Example:
+```
+sudo ./snapshot_zfs_datasets.sh 20231023T1831Z-haf-only
+```
+
+## rollback_zfs_datasets.sh
+This script rolls back ZFS datasets to a specified snapshot. It unmounts the datasets, rolls them back to the named snapshot, and then remounts them. This process will result in the loss of all data on those datasets since the snapshot.
+
+Usage:
+```
+sudo ./rollback_zfs_datasets.sh [--env-file=filename] [--zpool=zpool_name] [--top-level-dataset=dataset_name] snapshot-name
+```
+Options:
+- `--env-file=filename`: Specify the environment file to use.
+- `--zpool=zpool_name`: Specify the ZFS pool name.
+- `--top-level-dataset=dataset_name`: Specify the top-level dataset name.
+
+Example:
+```
+sudo ./rollback_zfs_datasets.sh --env-file=.env --zpool=haf-pool --top-level-dataset=haf-datadir snapshot_name
+```
\ No newline at end of file
diff --git a/use_develop_env.py b/use_develop_env.py
index d71f590..5be0a03 100644
--- a/use_develop_env.py
+++ b/use_develop_env.py
@@ -64,5 +64,6 @@ if __name__ == "__main__":
import sys
if len(sys.argv) != 2:
print("Usage: python use_develop_env.py <directory>")
+ print("This script updates the .env file in the haf_api_node repository with the short git hashes of other repositories in the specified directory.")
else:
main(sys.argv[1])
--
GitLab
From 05490f76f5418c8c44b8ea2e60a935d93b0b5c14 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 22 Nov 2024 17:57:20 -0500
Subject: [PATCH 47/78] Refactor image variable names in YAML configuration
files for consistency
---
.env.example | 18 +++++++++---------
README.md | 2 +-
balance_tracker.yaml | 10 +++++-----
drone.yaml | 2 +-
haf_base.yaml | 4 ++--
haf_block_explorer.yaml | 14 +++++++-------
hafah.yaml | 8 ++++----
haproxy.yaml | 4 ++--
hivemind.yaml | 8 ++++----
jussi.yaml | 2 +-
reputation_tracker.yaml | 12 ++++++------
11 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/.env.example b/.env.example
index 52353fd..909c49c 100644
--- a/.env.example
+++ b/.env.example
@@ -40,7 +40,7 @@ HIVE_API_NODE_VERSION=1.27.6rc10
# Global settings
# override the HAF core image's version and registry image here:
-# HAF_REGISTRY=${HIVE_API_NODE_REGISTRY}/haf
+# HAF_IMAGE=${HIVE_API_NODE_REGISTRY}/haf
# HAF_VERSION=${HIVE_API_NODE_VERSION}
# HAF_DATA_DIRECTORY="${TOP_LEVEL_DATASET_MOUNTPOINT}"
@@ -93,24 +93,24 @@ NETWORK_NAME=haf
# The default setup will run the recommended version of HAfAH,
# you can run a custom version by un-commenting and modifying the
# values below
-# HAFAH_REGISTRY=${HIVE_API_NODE_REGISTRY}/hafah
+# HAFAH_IMAGE=${HIVE_API_NODE_REGISTRY}/hafah
# HAFAH_VERSION=${HIVE_API_NODE_VERSION}
# The default setup will run the recommended version of Hivemind using the values
# below. You can override them here to run a custom version of Hivemind
-# HIVEMIND_INSTANCE_IMAGE=${HIVE_API_NODE_REGISTRY}/hivemind
-# HIVEMIND_INSTANCE_VERSION=${HIVE_API_NODE_VERSION}
+# HIVEMIND_IMAGE=${HIVE_API_NODE_REGISTRY}/hivemind
+# HIVEMIND_VERSION=${HIVE_API_NODE_VERSION}
# The default setup will run the recommended version of balance tracker,
# you can run a custom version by un-commenting and modifying the
# values below
-# BALANCE_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/balance_tracker
+# BALANCE_TRACKER_IMAGE=${HIVE_API_NODE_REGISTRY}/balance_tracker
# BALANCE_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
# REPUTATION_TRACKER_ADDON
-# REPUTATION_TRACKER_REGISTRY=${HIVE_API_NODE_REGISTRY}/reputation_tracker
+# REPUTATION_TRACKER_IMAGE=${HIVE_API_NODE_REGISTRY}/reputation_tracker
# REPUTATION_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
@@ -119,7 +119,7 @@ NETWORK_NAME=haf
# The default setup will run the recommended version of HAF block explorer,
# you can run a custom version by un-commenting and modifying the
# values below
-# HAF_BLOCK_EXPLORER_REGISTRY=${HIVE_API_NODE_REGISTRY}/haf_block_explorer
+# HAF_BLOCK_EXPLORER_IMAGE=${HIVE_API_NODE_REGISTRY}/haf_block_explorer
# HAF_BLOCK_EXPLORER_VERSION=${HIVE_API_NODE_VERSION}
@@ -133,14 +133,14 @@ NETWORK_NAME=haf
# The default setup will run the recommended version of Jussi
# you can run a custom version by un-commenting and modifying the
# values below
-# JUSSI_REGISTRY=${HIVE_API_NODE_REGISTRY}/jussi
+# JUSSI_IMAGE=${HIVE_API_NODE_REGISTRY}/jussi
# JUSSI_VERSION=latest
# JUSSI_REDIS_MAX_MEMORY=8G
# If you have chosen to run Drone instead of Jussi, it will run the
# this version by default. You can run a custom version by un-commenting
# and modifying the values below
-# DRONE_REGISTRY=${HIVE_API_NODE_REGISTRY}/drone
+# DRONE_IMAGE=${HIVE_API_NODE_REGISTRY}/drone
# DRONE_VERSION=latest
# DRONE_LOG_LEVEL=warn,access_log=info
diff --git a/README.md b/README.md
index a6e1d19..b8d9b93 100644
--- a/README.md
+++ b/README.md
@@ -271,7 +271,7 @@ The Hivemind data is now gone.
If you're uninstalling Hivemind permanently, then remember to remove the `hivemind` profile from your `.env` file's `COMPOSE_PROFILES` line so it doesn't start automatically next time you do a `docker compose up -d`.
If you're upgrading to a new version of hivemind:
-- if you're upgrading to a pre-release version, you'll need to set `HIVEMIND_INSTANCE_VERSION` in your `.env` file to the correct tag for the version you want to run. If you're just upgrading to a new release version (the ones tagged `haf_api_node`), you can leave this alone.
+- if you're upgrading to a pre-release version, you'll need to set `HIVEMIND_VERSION` in your `.env` file to the correct tag for the version you want to run. If you're just upgrading to a new release version (the ones tagged `haf_api_node`), you can leave this alone.
- run `docker compose pull` to grab the new version
- run `docker compose up -d` to bring up all services. This should run hivemind's install, then launch the block processing container.
- you can monitor Hivemind's sync process by watching the logs from `docker compose logs -f hivemind-block-processing`. In a few short days, your Hivemind app should be fully synced and ready to handle API requests.
diff --git a/balance_tracker.yaml b/balance_tracker.yaml
index bbad1dd..f5a61dc 100644
--- a/balance_tracker.yaml
+++ b/balance_tracker.yaml
@@ -1,6 +1,6 @@
services:
balance-tracker-install:
- image: ${BALANCE_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${BALANCE_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
@@ -17,7 +17,7 @@ services:
haf:
condition: service_healthy
balance-tracker-uninstall:
- image: ${BALANCE_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${BALANCE_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- balance-tracker-uninstall
environment:
@@ -31,7 +31,7 @@ services:
haf:
condition: service_healthy
balance-tracker-block-processing:
- image: ${BALANCE_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${BALANCE_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- balance-tracker-standalone
environment:
@@ -54,7 +54,7 @@ services:
haf:
condition: service_healthy
balance-tracker-postgrest:
- image: ${POSTGREST_REGISTRY:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
+ image: ${POSTGREST_IMAGE:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
- apps
- balance-tracker-standalone
@@ -83,7 +83,7 @@ services:
haf:
condition: service_healthy
balance-tracker-postgrest-rewriter:
- image: ${BALANCE_TRACKER_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker/postgrest-rewriter}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${BALANCE_TRACKER_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker/postgrest-rewriter}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- balance-tracker-standalone
diff --git a/drone.yaml b/drone.yaml
index 2a1329b..734799f 100644
--- a/drone.yaml
+++ b/drone.yaml
@@ -1,6 +1,6 @@
services:
drone:
- image: ${DRONE_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/drone}:${DRONE_VERSION:-latest}
+ image: ${DRONE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/drone}:${DRONE_VERSION:-latest}
profiles:
- servers
environment:
diff --git a/haf_base.yaml b/haf_base.yaml
index 986c27f..d5e8ae6 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -3,7 +3,7 @@ services:
profiles:
- core
- block-explorer-uninstall
- image: ${HAF_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf}:${HAF_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAF_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf}:${HAF_VERSION:-${HIVE_API_NODE_VERSION}}
networks:
- haf-network
tty: true
@@ -109,7 +109,7 @@ services:
retries: 10
start_period: 72h
logrotate:
- image: ${LOGROTATE_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/logrotate}:${LOGROTATE_VERSION:-latest}
+ image: ${LOGROTATE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/logrotate}:${LOGROTATE_VERSION:-latest}
profiles:
- logrotate
init: true
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 331f5bc..f320216 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -1,6 +1,6 @@
services:
block-explorer-install-schema:
- image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAF_BLOCK_EXPLORER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
@@ -22,7 +22,7 @@ services:
reputation-tracker-install-schema:
condition: service_completed_successfully
block-explorer-install-indexes:
- image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAF_BLOCK_EXPLORER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
@@ -46,7 +46,7 @@ services:
reputation-tracker-install-indexes:
condition: service_completed_successfully
block-explorer-uninstall:
- image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAF_BLOCK_EXPLORER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- block-explorer-uninstall
environment:
@@ -60,7 +60,7 @@ services:
haf:
condition: service_healthy
block-explorer-uninstall-balance-tracker:
- image: ${BALANCE_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${BALANCE_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- block-explorer-uninstall
environment:
@@ -75,7 +75,7 @@ services:
haf:
condition: service_healthy
block-explorer-block-processing:
- image: ${HAF_BLOCK_EXPLORER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAF_BLOCK_EXPLORER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
@@ -98,7 +98,7 @@ services:
haf:
condition: service_healthy
block-explorer-postgrest-rewriter:
- image: ${HAF_BLOCK_EXPLORER_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer/postgrest-rewriter}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAF_BLOCK_EXPLORER_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer/postgrest-rewriter}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
@@ -114,7 +114,7 @@ services:
networks:
- haf-network
block-explorer-postgrest:
- image: ${POSTGREST_REGISTRY:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
+ image: ${POSTGREST_IMAGE:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
- apps
- hafbe
diff --git a/hafah.yaml b/hafah.yaml
index d8bccb3..3ceabd2 100644
--- a/hafah.yaml
+++ b/hafah.yaml
@@ -1,6 +1,6 @@
services:
hafah-install:
- image: ${HAFAH_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hafah}:${HAFAH_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAFAH_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hafah}:${HAFAH_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafah
@@ -17,7 +17,7 @@ services:
haf:
condition: service_healthy
hafah-uninstall:
- image: ${HAFAH_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hafah}:${HAFAH_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAFAH_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hafah}:${HAFAH_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- hafah-uninstall
environment:
@@ -30,7 +30,7 @@ services:
haf:
condition: service_healthy
hafah-postgrest:
- image: ${POSTGREST_REGISTRY:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
+ image: ${POSTGREST_IMAGE:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
- apps
- hafah
@@ -58,7 +58,7 @@ services:
haf:
condition: service_healthy
hafah-postgrest-rewriter:
- image: ${HAFAH_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hafah/postgrest-rewriter}:${HAFAH_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAFAH_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hafah/postgrest-rewriter}:${HAFAH_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafah
diff --git a/haproxy.yaml b/haproxy.yaml
index cdfb940..1f2875a 100644
--- a/haproxy.yaml
+++ b/haproxy.yaml
@@ -1,6 +1,6 @@
services:
haproxy:
- image: ${HAPROXY_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy}:${HAPROXY_VERSION:-latest}
+ image: ${HAPROXY_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy}:${HAPROXY_VERSION:-latest}
profiles:
- servers
environment:
@@ -22,7 +22,7 @@ services:
test: wget -q --spider 127.0.0.1:8001
command: -W -db -f /usr/local/etc/haproxy
haproxy-healthchecks:
- image: ${HAPROXY_HEALTHCHECKS_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy-healthchecks}:${HAPROXY_HEALTHCHECKS_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HAPROXY_HEALTHCHECKS_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_api_node/haproxy-healthchecks}:${HAPROXY_HEALTHCHECKS_VERSION:-${HIVE_API_NODE_VERSION}}
environment:
POSTGRES_URL: postgresql://haf_admin@haf/haf_block_log
POSTGRES_URL_HAFBE: postgresql://hafbe_user@haf/haf_block_log
diff --git a/hivemind.yaml b/hivemind.yaml
index 9f0283f..464d70a 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -1,6 +1,6 @@
services:
hivemind-block-processing:
- image: ${HIVEMIND_INSTANCE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hivemind
@@ -24,7 +24,7 @@ services:
condition: service_completed_successfully
hivemind-server:
- image: ${HIVEMIND_INSTANCE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hivemind
@@ -44,7 +44,7 @@ services:
# be sure that the hivemind block-processor is not running at the same time as this install script, as doing so may corrupt your
# database
hivemind-install:
- image: ${HIVEMIND_INSTANCE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- hivemind-install
networks:
@@ -59,7 +59,7 @@ services:
condition: service_completed_successfully
hivemind-uninstall:
- image: ${HIVEMIND_INSTANCE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- hivemind-uninstall
networks:
diff --git a/jussi.yaml b/jussi.yaml
index 1776b9c..1ca19c7 100644
--- a/jussi.yaml
+++ b/jussi.yaml
@@ -13,7 +13,7 @@ services:
networks:
- haf-network
jussi:
- image: ${JUSSI_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/jussi}:${JUSSI_VERSION:-latest}
+ image: ${JUSSI_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/jussi}:${JUSSI_VERSION:-latest}
profiles:
- servers
environment:
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index ceb2357..0b44ac2 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -1,6 +1,6 @@
services:
reputation-tracker-install-schema:
- image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${REPUTATION_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
@@ -18,7 +18,7 @@ services:
haf:
condition: service_healthy
reputation-tracker-install-indexes:
- image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${REPUTATION_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
@@ -38,7 +38,7 @@ services:
reputation-tracker-install-schema:
condition: service_completed_successfully
reputation-tracker-uninstall:
- image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${REPUTATION_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- reputation-tracker-uninstall
environment:
@@ -51,7 +51,7 @@ services:
haf:
condition: service_healthy
reputation-tracker-block-processing:
- image: ${REPUTATION_TRACKER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${REPUTATION_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
@@ -76,7 +76,7 @@ services:
haf:
condition: service_healthy
reputation-tracker-postgrest:
- image: ${POSTGREST_REGISTRY:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
+ image: ${POSTGREST_IMAGE:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
- apps
- hivemind
@@ -106,7 +106,7 @@ services:
haf:
condition: service_healthy
reputation-tracker-postgrest-rewriter:
- image: ${REPUTATION_TRACKER_REWRITER_REGISTRY:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker/postgrest-rewriter}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${REPUTATION_TRACKER_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker/postgrest-rewriter}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- reputation-tracker-standalone
--
GitLab
From c40a18d271464673cdfb2c8e7314392102e3f422 Mon Sep 17 00:00:00 2001
From: Grzegorz Mirek <gmirek@syncad.com>
Date: Wed, 27 Nov 2024 14:47:18 +0100
Subject: [PATCH 48/78] Add fix(issue#22) and version variable for
postgresexporter
---
monitoring.yaml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/monitoring.yaml b/monitoring.yaml
index cd45ea1..b746652 100644
--- a/monitoring.yaml
+++ b/monitoring.yaml
@@ -142,10 +142,12 @@ services:
org.label-schema.group: "monitoring"
postgresexporter:
- image: quay.io/prometheuscommunity/postgres-exporter
+ image: quay.io/prometheuscommunity/postgres-exporter:${POSTGRESEXPORTER_VERSION:-v0.16.0}
profiles:
- monitoring
restart: unless-stopped
+ command:
+ - --no-collector.stat_bgwriter
expose:
- 9187
depends_on:
--
GitLab
From 553fa1fd2b1fb278e9aba709d53dd8de3d30902a Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 29 Nov 2024 18:24:52 -0500
Subject: [PATCH 49/78] Enhance use_develop_env.py to include commit date and
description in .env file updates
---
repo_versions.txt | 8 ++++++++
use_develop_env.py | 28 +++++++++++++++++++---------
2 files changed, 27 insertions(+), 9 deletions(-)
create mode 100644 repo_versions.txt
diff --git a/repo_versions.txt b/repo_versions.txt
new file mode 100644
index 0000000..be1561b
--- /dev/null
+++ b/repo_versions.txt
@@ -0,0 +1,8 @@
+git@gitlab.syncad.com:hive/balance_tracker, 93238167, 2024-11-27 13:39:48 -0500, change raise from warning to info level
+git@gitlab.syncad.com:/hive/haf.git, 23c9c614, 2024-11-25 18:24:43 +0000, Index operator class specific to hafd.operation definition also shall be located in hafd schema.
+git@gitlab.syncad.com:hive/HAfAH, 624d76c1, 2024-11-21 01:03:55 +0000, set GET_DEPTH to 1
+git@gitlab.syncad.com:hive/haf_api_node.git, c40a18d2, 2024-11-27 14:47:18 +0100, Add fix(issue#22) and version variable for postgresexporter
+git@gitlab.syncad.com:hive/haf_block_explorer, d6399952, 2024-11-27 13:44:24 -0500, change raise from warning to info level
+git@gitlab.syncad.com:/hive/hive.git, 5fe83988, 2024-11-26 09:01:59 +0100, Add test for options `exit-at-block` and `stop-at-block`
+git@gitlab.syncad.com:hive/hivemind, ca020dc0, 2024-11-27 20:30:16 -0500, Fix error in get_trending_ranked_posts_for_communities
+git@gitlab.syncad.com:/hive/reputation_tracker.git, 78a60d25, 2024-11-27 13:44:04 -0500, change raise from warning to info level
diff --git a/use_develop_env.py b/use_develop_env.py
index 5be0a03..1bafcb7 100644
--- a/use_develop_env.py
+++ b/use_develop_env.py
@@ -13,11 +13,21 @@ def get_git_revision(repo_path):
stderr=subprocess.STDOUT
).strip().decode('utf-8')[:8] # Use only the first 8 characters of the hash
- return repo_url, git_hash
+ commit_date = subprocess.check_output(
+ ["git", "-C", repo_path, "show", "-s", "--format=%ci", git_hash],
+ stderr=subprocess.STDOUT
+ ).strip().decode('utf-8')
+
+ commit_description = subprocess.check_output(
+ ["git", "-C", repo_path, "show", "-s", "--format=%s", git_hash],
+ stderr=subprocess.STDOUT
+ ).strip().decode('utf-8')
+
+ return repo_url, git_hash, commit_date, commit_description
except subprocess.CalledProcessError:
- return None, None
+ return None, None, None, None
-def update_env_file(env_file_path, repo_name, git_hash):
+def update_env_file(env_file_path, repo_name, git_hash, commit_date, commit_description):
updated = False
with open(env_file_path, "r") as file:
lines = file.readlines()
@@ -25,7 +35,7 @@ def update_env_file(env_file_path, repo_name, git_hash):
with open(env_file_path, "w") as file:
for line in lines:
if line.startswith(f"{repo_name.upper()}_VERSION="):
- file.write(f"{repo_name.upper()}_VERSION={git_hash}\n")
+ file.write(f"{repo_name.upper()}_VERSION={git_hash} # {commit_date} - {commit_description}\n")
updated = True
print(f"Updated {repo_name.upper()}_VERSION to {git_hash} in .env file")
else:
@@ -33,7 +43,7 @@ def update_env_file(env_file_path, repo_name, git_hash):
if not updated:
with open(env_file_path, "a") as file:
- file.write(f"{repo_name.upper()}_VERSION={git_hash}\n")
+ file.write(f"{repo_name.upper()}_VERSION={git_hash} # {commit_date} - {commit_description}\n")
print(f"Added {repo_name.upper()}_VERSION={git_hash} to .env file")
def main(directory):
@@ -52,13 +62,13 @@ def main(directory):
for item in os.listdir(directory):
item_path = os.path.join(directory, item)
if os.path.isdir(item_path) and ".git" in os.listdir(item_path):
- repo_url, git_hash = get_git_revision(item_path)
- if repo_url and git_hash:
- output_file.write(f"{repo_url}, {git_hash}\n")
+ repo_url, git_hash, commit_date, commit_description = get_git_revision(item_path)
+ if repo_url and git_hash and commit_date and commit_description:
+ output_file.write(f"{repo_url}, {git_hash}, {commit_date}, {commit_description}\n")
print(f"Found repo: {repo_url} at {item_path}")
repo_name = os.path.basename(item_path)
if repo_name != "hive":
- update_env_file(env_file_path, repo_name, git_hash)
+ update_env_file(env_file_path, repo_name, git_hash, commit_date, commit_description)
if __name__ == "__main__":
import sys
--
GitLab
From 1207c4ea9b9b5b05aefd2ee591bbc42c7a9fc058 Mon Sep 17 00:00:00 2001
From: Michal Zander <mzander@syncad.com>
Date: Tue, 10 Dec 2024 15:14:48 +0000
Subject: [PATCH 50/78] Remove index installation stage
---
haf_base.yaml | 4 +--
haf_block_explorer.yaml | 25 -------------------
.../haf_block_explorer.log_rotation.yaml | 3 ---
.../reputation_tracker.log_rotation.yaml | 3 ---
reputation_tracker.yaml | 23 +----------------
5 files changed, 2 insertions(+), 56 deletions(-)
diff --git a/haf_base.yaml b/haf_base.yaml
index d5e8ae6..cbb7bea 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -28,7 +28,7 @@ services:
host haf_block_log btracker_user ${COMPOSE_PROJECT_NAME}-balance-tracker-postgrest-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-install-schema-1.${NETWORK_NAME} trust\n
- host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-install-indexes-1.${NETWORK_NAME} trust\n
+
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log reptracker_owner ${COMPOSE_PROJECT_NAME}-reputation-tracker-block-processing-1.${NETWORK_NAME} trust\n
host haf_block_log reptracker_user ${COMPOSE_PROJECT_NAME}-reputation-tracker-postgrest-1.${NETWORK_NAME} trust\n
@@ -40,8 +40,6 @@ services:
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-install-schema-1.${NETWORK_NAME} trust\n
host haf_block_log hafbe_owner ${COMPOSE_PROJECT_NAME}-block-explorer-install-schema-1.${NETWORK_NAME} trust\n
- host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-install-indexes-1.${NETWORK_NAME} trust\n
- host haf_block_log hafbe_owner ${COMPOSE_PROJECT_NAME}-block-explorer-install-indexes-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log hafbe_owner ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-balance-tracker-1.${NETWORK_NAME} trust\n
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index f320216..6275614 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -9,7 +9,6 @@ services:
command:
- install_app
- --swagger-url=${PUBLIC_HOSTNAME}
- - --schema-only
networks:
haf-network:
depends_on:
@@ -21,30 +20,6 @@ services:
condition: service_completed_successfully
reputation-tracker-install-schema:
condition: service_completed_successfully
- block-explorer-install-indexes:
- image: ${HAF_BLOCK_EXPLORER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
- profiles:
- - apps
- - hafbe
- environment:
- POSTGRES_HOST: haf
- command:
- - install_app
- - --swagger-url=${PUBLIC_HOSTNAME}
- - --indexes-only
- networks:
- haf-network:
- depends_on:
- haf:
- condition: service_healthy
- block-explorer-install-schema:
- condition: service_completed_successfully
- balance-tracker-install:
- condition: service_completed_successfully
- hafah-install:
- condition: service_completed_successfully
- reputation-tracker-install-indexes:
- condition: service_completed_successfully
block-explorer-uninstall:
image: ${HAF_BLOCK_EXPLORER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
diff --git a/log_rotation/haf_block_explorer.log_rotation.yaml b/log_rotation/haf_block_explorer.log_rotation.yaml
index a6301b4..cb6a505 100644
--- a/log_rotation/haf_block_explorer.log_rotation.yaml
+++ b/log_rotation/haf_block_explorer.log_rotation.yaml
@@ -23,6 +23,3 @@ services:
block-explorer-install-schema:
logging:
driver: local
- block-explorer-install-indexes:
- logging:
- driver: local
diff --git a/log_rotation/reputation_tracker.log_rotation.yaml b/log_rotation/reputation_tracker.log_rotation.yaml
index 8a0286f..b667bc2 100644
--- a/log_rotation/reputation_tracker.log_rotation.yaml
+++ b/log_rotation/reputation_tracker.log_rotation.yaml
@@ -14,6 +14,3 @@ services:
reputation-tracker-install-schema:
logging:
driver: local
- reputation-tracker-install-indexes:
- logging:
- driver: local
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index 0b44ac2..8edf7d3 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -13,30 +13,9 @@ services:
command:
- install_app
- --swagger-url=${PUBLIC_HOSTNAME}
- - --schema-only
depends_on:
haf:
condition: service_healthy
- reputation-tracker-install-indexes:
- image: ${REPUTATION_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
- profiles:
- - apps
- - hafbe
- - hivemind
- - reputation-tracker-setup
- environment:
- POSTGRES_HOST: haf
- networks:
- haf-network:
- command:
- - install_app
- - --swagger-url=${PUBLIC_HOSTNAME}
- - --indexes-only
- depends_on:
- haf:
- condition: service_healthy
- reputation-tracker-install-schema:
- condition: service_completed_successfully
reputation-tracker-uninstall:
image: ${REPUTATION_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
@@ -71,7 +50,7 @@ services:
retries: 3
start_period: 48h
depends_on:
- reputation-tracker-install-indexes:
+ reputation-tracker-install-schema:
condition: service_completed_successfully
haf:
condition: service_healthy
--
GitLab
From dcaf29a3fad6995a98554d6bea8f8f8a63e2071b Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 14:44:18 -0500
Subject: [PATCH 51/78] fix reptracker by itself, and rename my helper script
---
.gitignore | 3 ++-
haf_base.yaml | 2 +-
haf_block_explorer.yaml | 3 +--
hivemind.yaml | 4 ++--
use_develop_env.py => local_to_env.py | 14 +++++++-------
repo_versions.txt | 8 --------
reputation_tracker.yaml | 14 +++++++-------
7 files changed, 20 insertions(+), 28 deletions(-)
rename use_develop_env.py => local_to_env.py (96%)
delete mode 100644 repo_versions.txt
diff --git a/.gitignore b/.gitignore
index 13d1d46..376f62f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
.env
-*.sw?
\ No newline at end of file
+repo_versions.txt
+*.sw?
diff --git a/haf_base.yaml b/haf_base.yaml
index cbb7bea..3fcaab1 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -27,7 +27,7 @@ services:
host haf_block_log btracker_owner ${COMPOSE_PROJECT_NAME}-balance-tracker-block-processing-1.${NETWORK_NAME} trust\n
host haf_block_log btracker_user ${COMPOSE_PROJECT_NAME}-balance-tracker-postgrest-1.${NETWORK_NAME} trust\n
- host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-install-schema-1.${NETWORK_NAME} trust\n
+ host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-install-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-reputation-tracker-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log reptracker_owner ${COMPOSE_PROJECT_NAME}-reputation-tracker-block-processing-1.${NETWORK_NAME} trust\n
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 6275614..09a2ea6 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -18,7 +18,7 @@ services:
condition: service_completed_successfully
hafah-install:
condition: service_completed_successfully
- reputation-tracker-install-schema:
+ reputation-tracker-install:
condition: service_completed_successfully
block-explorer-uninstall:
image: ${HAF_BLOCK_EXPLORER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
@@ -28,7 +28,6 @@ services:
POSTGRES_HOST: haf
command:
- uninstall_app
- - --skip-btracker
networks:
haf-network:
depends_on:
diff --git a/hivemind.yaml b/hivemind.yaml
index 464d70a..8a486db 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -20,7 +20,7 @@ services:
depends_on:
haf:
condition: service_healthy
- reputation-tracker-install-schema:
+ reputation-tracker-install:
condition: service_completed_successfully
hivemind-server:
@@ -55,7 +55,7 @@ services:
depends_on:
haf:
condition: service_healthy
- reputation-tracker-install-schema:
+ reputation-tracker-install:
condition: service_completed_successfully
hivemind-uninstall:
diff --git a/use_develop_env.py b/local_to_env.py
similarity index 96%
rename from use_develop_env.py
rename to local_to_env.py
index 1bafcb7..4cf1072 100644
--- a/use_develop_env.py
+++ b/local_to_env.py
@@ -7,22 +7,22 @@ def get_git_revision(repo_path):
["git", "-C", repo_path, "config", "--get", "remote.origin.url"],
stderr=subprocess.STDOUT
).strip().decode('utf-8')
-
+
git_hash = subprocess.check_output(
["git", "-C", repo_path, "rev-parse", "HEAD"],
stderr=subprocess.STDOUT
).strip().decode('utf-8')[:8] # Use only the first 8 characters of the hash
-
+
commit_date = subprocess.check_output(
["git", "-C", repo_path, "show", "-s", "--format=%ci", git_hash],
stderr=subprocess.STDOUT
).strip().decode('utf-8')
-
+
commit_description = subprocess.check_output(
["git", "-C", repo_path, "show", "-s", "--format=%s", git_hash],
stderr=subprocess.STDOUT
).strip().decode('utf-8')
-
+
return repo_url, git_hash, commit_date, commit_description
except subprocess.CalledProcessError:
return None, None, None, None
@@ -31,7 +31,7 @@ def update_env_file(env_file_path, repo_name, git_hash, commit_date, commit_desc
updated = False
with open(env_file_path, "r") as file:
lines = file.readlines()
-
+
with open(env_file_path, "w") as file:
for line in lines:
if line.startswith(f"{repo_name.upper()}_VERSION="):
@@ -40,7 +40,7 @@ def update_env_file(env_file_path, repo_name, git_hash, commit_date, commit_desc
print(f"Updated {repo_name.upper()}_VERSION to {git_hash} in .env file")
else:
file.write(line)
-
+
if not updated:
with open(env_file_path, "a") as file:
file.write(f"{repo_name.upper()}_VERSION={git_hash} # {commit_date} - {commit_description}\n")
@@ -73,7 +73,7 @@ def main(directory):
if __name__ == "__main__":
import sys
if len(sys.argv) != 2:
- print("Usage: python use_develop_env.py <directory>")
+ print("Usage: python local_to_env.py <directory>")
print("This script updates the .env file in the haf_api_node repository with the short git hashes of other repositories in the specified directory.")
else:
main(sys.argv[1])
diff --git a/repo_versions.txt b/repo_versions.txt
deleted file mode 100644
index be1561b..0000000
--- a/repo_versions.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-git@gitlab.syncad.com:hive/balance_tracker, 93238167, 2024-11-27 13:39:48 -0500, change raise from warning to info level
-git@gitlab.syncad.com:/hive/haf.git, 23c9c614, 2024-11-25 18:24:43 +0000, Index operator class specific to hafd.operation definition also shall be located in hafd schema.
-git@gitlab.syncad.com:hive/HAfAH, 624d76c1, 2024-11-21 01:03:55 +0000, set GET_DEPTH to 1
-git@gitlab.syncad.com:hive/haf_api_node.git, c40a18d2, 2024-11-27 14:47:18 +0100, Add fix(issue#22) and version variable for postgresexporter
-git@gitlab.syncad.com:hive/haf_block_explorer, d6399952, 2024-11-27 13:44:24 -0500, change raise from warning to info level
-git@gitlab.syncad.com:/hive/hive.git, 5fe83988, 2024-11-26 09:01:59 +0100, Add test for options `exit-at-block` and `stop-at-block`
-git@gitlab.syncad.com:hive/hivemind, ca020dc0, 2024-11-27 20:30:16 -0500, Fix error in get_trending_ranked_posts_for_communities
-git@gitlab.syncad.com:/hive/reputation_tracker.git, 78a60d25, 2024-11-27 13:44:04 -0500, change raise from warning to info level
diff --git a/reputation_tracker.yaml b/reputation_tracker.yaml
index 8edf7d3..bab526b 100644
--- a/reputation_tracker.yaml
+++ b/reputation_tracker.yaml
@@ -1,11 +1,11 @@
services:
- reputation-tracker-install-schema:
+ reputation-tracker-install:
image: ${REPUTATION_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hafbe
- hivemind
- - reputation-tracker-setup
+ - reputation-tracker
environment:
POSTGRES_HOST: haf
networks:
@@ -35,7 +35,7 @@ services:
- apps
- hafbe
- hivemind
- - reputation-tracker-standalone
+ - reputation-tracker
environment:
POSTGRES_HOST: haf
POSTGRES_USER: reptracker_owner
@@ -50,7 +50,7 @@ services:
retries: 3
start_period: 48h
depends_on:
- reputation-tracker-install-schema:
+ reputation-tracker-install:
condition: service_completed_successfully
haf:
condition: service_healthy
@@ -59,7 +59,7 @@ services:
profiles:
- apps
- hivemind
- - reputation-tracker-standalone
+ - reputation-tracker
- hafbe
environment:
PGRST_ADMIN_SERVER_PORT: 3001
@@ -80,7 +80,7 @@ services:
networks:
haf-network:
depends_on:
- reputation-tracker-install-schema:
+ reputation-tracker-install:
condition: service_completed_successfully
haf:
condition: service_healthy
@@ -88,7 +88,7 @@ services:
image: ${REPUTATION_TRACKER_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/reputation_tracker/postgrest-rewriter}:${REPUTATION_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- - reputation-tracker-standalone
+ - reputation-tracker
- hafbe
- hivemind
healthcheck:
--
GitLab
From 4cdb0c845e00883bdb64b2fad4ebcc9e40713ae6 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 12 Dec 2024 23:17:57 +0000
Subject: [PATCH 52/78] Fix balance tracker so that it can be run standalone in
addition to just as part of hafbe
---
.env.example | 15 +++++++++++++--
balance_tracker.yaml | 23 ++++++++++++++++-------
haf_base.yaml | 1 +
haf_block_explorer.yaml | 23 +++++++++++++++++++++--
4 files changed, 51 insertions(+), 11 deletions(-)
diff --git a/.env.example b/.env.example
index 909c49c..23b506c 100644
--- a/.env.example
+++ b/.env.example
@@ -114,8 +114,19 @@ NETWORK_NAME=haf
# REPUTATION_TRACKER_VERSION=${HIVE_API_NODE_VERSION}
-# REPTRACKER_SCHEMA="hafbe-rt"
-# BTRACKER_SCHEMA="hafbe-bt"
+# There are two ways of running Balance Tracker: as a standalone app, or
+# integrated with HAF Block Explorer. While you can technically run both,
+# there's no good reason to do so--you'll just waste disk space and processing
+# power maintaining two copies of the data.
+# Regardless of which way you decide to run Balance Tracker, you will need
+# to run a single API server, and it needs to know which schema the data is
+# stored in. It will be in "hafbe_bal" if you're running HAF Block Explorer,
+# and "btracker_app" if you're running Balance Tracker standalone.
+# The default behavior is to serve data from the HAF Block Explorer, but
+# if you're only running the standalone Balance Tracker, uncomment the next
+# line:
+# BTRACKER_SCHEMA="btracker_app"
+
# The default setup will run the recommended version of HAF block explorer,
# you can run a custom version by un-commenting and modifying the
# values below
diff --git a/balance_tracker.yaml b/balance_tracker.yaml
index f5a61dc..e50f12b 100644
--- a/balance_tracker.yaml
+++ b/balance_tracker.yaml
@@ -1,32 +1,37 @@
+# Note, most of the services in this file are only used in the uncommon case where
+# the node is running Balance Tracker standalone. Balance Tracker is typically
+# run as part of HAF Block Explorer, and the services related to that usage are
+# in haf_block_explorer.yaml.
+# The exception is the postgrest server & rewriter. Those are shared between the two
services:
balance-tracker-install:
image: ${BALANCE_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- - apps
- - hafbe
- - balance-tracker-setup
+ - balance-tracker-standalone
environment:
POSTGRES_HOST: haf
networks:
haf-network:
command:
- install_app
- - --schema=${BTRACKER_SCHEMA:-hafbe_bal}
+ - --schema=btracker_app
- --swagger-url=${PUBLIC_HOSTNAME}
depends_on:
haf:
condition: service_healthy
+ # to uninstall Balance Tracker (standalone), shut down the stack and run a command like:
+ # docker compose --profile core --profile balance-tracker-standalone-uninstall up -d
balance-tracker-uninstall:
image: ${BALANCE_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- - balance-tracker-uninstall
+ - balance-tracker-standalone-uninstall
environment:
POSTGRES_HOST: haf
networks:
haf-network:
command:
- uninstall_app
- - --schema=${BTRACKER_SCHEMA:-hafbe_bal}
+ - --schema=btracker_app
depends_on:
haf:
condition: service_healthy
@@ -41,7 +46,7 @@ services:
haf-network:
command:
- process_blocks
- - --schema=${BTRACKER_SCHEMA:-hafbe_bal}
+ - --schema=btracker_app
healthcheck:
test: ["CMD-SHELL","/app/block-processing-healthcheck.sh"]
interval: 60s
@@ -80,6 +85,10 @@ services:
depends_on:
balance-tracker-install:
condition: service_completed_successfully
+ required: false # allow this service to be missing (when only running hafbe)
+ block-explorer-install-balance-tracker:
+ condition: service_completed_successfully
+ required: false # allow this service to be missing (when only running standalone)
haf:
condition: service_healthy
balance-tracker-postgrest-rewriter:
diff --git a/haf_base.yaml b/haf_base.yaml
index 3fcaab1..44ccbb5 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -43,6 +43,7 @@ services:
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log hafbe_owner ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-balance-tracker-1.${NETWORK_NAME} trust\n
+ host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-install-balance-tracker-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-block-explorer-uninstall-reputation-tracker-1.${NETWORK_NAME} trust\n
diff --git a/haf_block_explorer.yaml b/haf_block_explorer.yaml
index 09a2ea6..ff20cc3 100644
--- a/haf_block_explorer.yaml
+++ b/haf_block_explorer.yaml
@@ -14,12 +14,14 @@ services:
depends_on:
haf:
condition: service_healthy
- balance-tracker-install:
+ block-explorer-install-balance-tracker:
condition: service_completed_successfully
hafah-install:
condition: service_completed_successfully
reputation-tracker-install:
condition: service_completed_successfully
+ # to uninstall HAF Block Explorer, shut down the stack and run a command like:
+ # docker compose --profile core --profile block-explorer-uninstall up -d
block-explorer-uninstall:
image: ${HAF_BLOCK_EXPLORER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/haf_block_explorer}:${HAF_BLOCK_EXPLORER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
@@ -33,6 +35,22 @@ services:
depends_on:
haf:
condition: service_healthy
+ block-explorer-install-balance-tracker:
+ image: ${BALANCE_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
+ profiles:
+ - apps
+ - hafbe
+ environment:
+ POSTGRES_HOST: haf
+ networks:
+ haf-network:
+ command:
+ - install_app
+ - --schema=hafbe_bal
+ - --swagger-url=${PUBLIC_HOSTNAME}
+ depends_on:
+ haf:
+ condition: service_healthy
block-explorer-uninstall-balance-tracker:
image: ${BALANCE_TRACKER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/balance_tracker}:${BALANCE_TRACKER_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
@@ -43,6 +61,7 @@ services:
haf-network:
command:
- uninstall_app
+ - --schema=hafbe_bal
depends_on:
block-explorer-uninstall:
condition: service_completed_successfully
@@ -100,7 +119,7 @@ services:
PGRST_DB_POOL: 20
PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
PGRST_DB_ROOT_SPEC: root
- PGRST_DB_EXTRA_SEARCH_PATH: ${BTRACKER_SCHEMA:-hafbe_bal}, ${REPTRACKER_SCHEMA:-reptracker_app}
+ PGRST_DB_EXTRA_SEARCH_PATH: hafbe_bal, reptracker_app
# PGRST_OPENAPI_SERVER_PROXY_URI: http://${PUBLIC_HOSTNAME}/hafbe_api/
networks:
haf-network:
--
GitLab
From 981b76b4d2d243c50a7f4531bdf730118307b748 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Fri, 29 Nov 2024 13:50:03 +0100
Subject: [PATCH 53/78] Updated PG_ACCESS definition to allow trusted access
from Hivemind PostgREST server instance
---
haf_base.yaml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/haf_base.yaml b/haf_base.yaml
index 44ccbb5..e203b57 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -57,6 +57,8 @@ services:
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-hivemind-block-processing-1.${NETWORK_NAME} trust\n
host haf_block_log hivemind ${COMPOSE_PROJECT_NAME}-hivemind-block-processing-1.${NETWORK_NAME} trust\n
host haf_block_log hivemind ${COMPOSE_PROJECT_NAME}-hivemind-server-1.${NETWORK_NAME} trust\n
+ host haf_block_log hivemind ${COMPOSE_PROJECT_NAME}-hivemind-postgrest-server-1.${NETWORK_NAME} trust\n
+
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-hivemind-uninstall-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-pgadmin-1.${NETWORK_NAME} trust\n
--
GitLab
From a6d1f872ef57f1a4ab5eb2a6f74de7f3e240b454 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Fri, 29 Nov 2024 13:55:29 +0100
Subject: [PATCH 54/78] WIP: preliminary definition of hivemind postgrest
server integration and dedicated rewriter container.
---
hivemind.yaml | 40 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 40 insertions(+)
diff --git a/hivemind.yaml b/hivemind.yaml
index 8a486db..27ba605 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -39,6 +39,29 @@ services:
haf:
condition: service_healthy
+
+ hivemind-postgrest-server:
+ image: ${HIVEMIND_INSTANCE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
+ profiles:
+ - apps
+ - hivemind
+ networks:
+ haf-network:
+ environment:
+ PGRST_DB_PLAN_ENABLED: true
+ #PGRST_DB_ROOT_SPEC: home
+ PGRST_OPENAPI_MODE: "disabled"
+ PGRST_DB_POOL: 20
+ command:
+ - "postgrest-server"
+ - "--postgres-url=postgresql://hivemind@haf/haf_block_log?application_name=hive-mind-postgrest" #warning hivemind improperly matches all connections starting from `hivemind` and crashes if they are made externally (not from python)
+ - "--webserver-port=8080"
+ depends_on:
+ hivemind-block-processing:
+ condition: service_healthy
+ haf:
+ condition: service_healthy
+
# note: the hivemind-install steps, which create/update the hivemind database schema, are also run automatically at the beginning
# of the hivemind-block-processing image's startup. There is normally no reason to run this step separately. If you do,
# be sure that the hivemind block-processor is not running at the same time as this install script, as doing so may corrupt your
@@ -58,6 +81,23 @@ services:
reputation-tracker-install:
condition: service_completed_successfully
+ hivemind-postgrest-rewriter:
+ image: ${HIVEMIND_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind/postgrest-rewriter}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
+ profiles:
+ - apps
+ - hivemind
+ healthcheck:
+ test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:81/health || exit 1"]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 1m
+ depends_on:
+ hivemind-postgrest-server:
+ condition: service_started # TODO ADD HC TO POSTGREST-SERVER
+ networks:
+ - haf-network
+
hivemind-uninstall:
image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
--
GitLab
From 1a29c00cc94833c4f8ef71a7f75010286a69a15d Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Fri, 29 Nov 2024 14:13:58 +0100
Subject: [PATCH 55/78] Disabled bg_writer collection in postgres-exporter to
avoid errors in Postgres 17 Changed default port mapped on host for Grafana
to avoid conflicts with default Denser deployment (it works on port 3000)
---
monitoring.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/monitoring.yaml b/monitoring.yaml
index b746652..d12b80a 100644
--- a/monitoring.yaml
+++ b/monitoring.yaml
@@ -82,7 +82,7 @@ services:
- GF_USERS_ALLOW_SIGN_UP=false
restart: unless-stopped
ports:
- - "3000:3000"
+ - "3003:3000"
networks:
- haf-network
labels:
--
GitLab
From eb31c87be8666b61096b697180f6460e23d9425e Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Fri, 29 Nov 2024 14:17:01 +0100
Subject: [PATCH 56/78] Hivemind Postgrest rewriter container directly exposed
from stack
---
exposed/hivemind.exposed.yaml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/exposed/hivemind.exposed.yaml b/exposed/hivemind.exposed.yaml
index 0870d9f..08b7020 100644
--- a/exposed/hivemind.exposed.yaml
+++ b/exposed/hivemind.exposed.yaml
@@ -2,3 +2,6 @@ services:
hivemind-server:
ports:
- "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7002:8080"
+ hivemind-postgrest-rewriter:
+ ports:
+ - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-3}7002:80"
--
GitLab
From 50663f7597238d7b9b4f6b4f13cd8acd17f7acb7 Mon Sep 17 00:00:00 2001
From: Bartek Wrona <wrona@syncad.com>
Date: Fri, 29 Nov 2024 14:22:46 +0100
Subject: [PATCH 57/78] Supplemented example environment definition (added
HIVEMIND_REWRITER_IMAGE env. variable)
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 23b506c..500302f 100644
--- a/.env.example
+++ b/.env.example
@@ -100,7 +100,7 @@ NETWORK_NAME=haf
# below. You can override them here to run a custom version of Hivemind
# HIVEMIND_IMAGE=${HIVE_API_NODE_REGISTRY}/hivemind
# HIVEMIND_VERSION=${HIVE_API_NODE_VERSION}
-
+# HIVEMIND_REWRITER_IMAGE=${HIVE_API_NODE_REGISTRY}/hivemind/postgrest-rewriter
# The default setup will run the recommended version of balance tracker,
# you can run a custom version by un-commenting and modifying the
--
GitLab
From f537467d606044169132f6838cecb0334396ac6d Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sun, 1 Dec 2024 22:19:24 +0000
Subject: [PATCH 58/78] Remove python hivemind, switch hivemind postgrest
server to use thesame image as all of our other postgrest servers. Still
need to rename postgrest and switch it to the standard port 3000
---
exposed/hivemind.exposed.yaml | 5 +---
haproxy/haproxy.cfg | 2 +-
hivemind.yaml | 44 ++++++++++++++---------------------
3 files changed, 20 insertions(+), 31 deletions(-)
diff --git a/exposed/hivemind.exposed.yaml b/exposed/hivemind.exposed.yaml
index 08b7020..9ad23db 100644
--- a/exposed/hivemind.exposed.yaml
+++ b/exposed/hivemind.exposed.yaml
@@ -1,7 +1,4 @@
services:
- hivemind-server:
- ports:
- - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7002:8080"
hivemind-postgrest-rewriter:
ports:
- - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-3}7002:80"
+ - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7002:80"
diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg
index 0768183..47310e5 100644
--- a/haproxy/haproxy.cfg
+++ b/haproxy/haproxy.cfg
@@ -109,7 +109,7 @@ backend hafah
backend hivemind
balance roundrobin
- server hivemind hivemind-server:8080 check agent-check agent-addr haproxy-healthchecks agent-port 7002 agent-inter 10s
+ server hivemind hivemind-postgrest-rewriter:80 check agent-check agent-addr haproxy-healthchecks agent-port 7002 agent-inter 10s
backend block-explorer
balance roundrobin
diff --git a/hivemind.yaml b/hivemind.yaml
index 27ba605..dc1dc87 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -23,39 +23,31 @@ services:
reputation-tracker-install:
condition: service_completed_successfully
- hivemind-server:
- image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
- profiles:
- - apps
- - hivemind
- networks:
- haf-network:
- command:
- - "--database-url=postgresql://hivemind@haf/haf_block_log"
- - "server"
- depends_on:
- hivemind-block-processing:
- condition: service_healthy
- haf:
- condition: service_healthy
-
-
hivemind-postgrest-server:
- image: ${HIVEMIND_INSTANCE_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${POSTGREST_IMAGE:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
- apps
- hivemind
networks:
haf-network:
environment:
- PGRST_DB_PLAN_ENABLED: true
+ PGRST_ADMIN_SERVER_PORT: 3001
+ PGRST_SERVER_PORT: 8080
+ PGRST_DB_URI: "postgresql://hivemind@haf/haf_block_log?application_name=hive-mind-postgrest" #warning hivemind improperly matches all connections starting from `hivemind` and crashes if they are made externally (not from python)
+ PGRST_DB_SCHEMA: hivemind_endpoints
#PGRST_DB_ROOT_SPEC: home
- PGRST_OPENAPI_MODE: "disabled"
+ PGRST_DB_ANON_ROLE: hivemind
PGRST_DB_POOL: 20
- command:
- - "postgrest-server"
- - "--postgres-url=postgresql://hivemind@haf/haf_block_log?application_name=hive-mind-postgrest" #warning hivemind improperly matches all connections starting from `hivemind` and crashes if they are made externally (not from python)
- - "--webserver-port=8080"
+ PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
+ PGRST_OPENAPI_MODE: "disabled" # unclear why this is set, I guess because we currently only support json-rpc?
+ # when debugging, you can enable this
+ PGRST_DB_PLAN_ENABLED: true
+ healthcheck:
+ test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:3001/ready || exit 1"]
+ interval: 10s
+ timeout: 3s
+ retries: 10
+ start_period: 1m
depends_on:
hivemind-block-processing:
condition: service_healthy
@@ -82,7 +74,7 @@ services:
condition: service_completed_successfully
hivemind-postgrest-rewriter:
- image: ${HIVEMIND_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind/postgrest-rewriter}:${HIVEMIND_INSTANCE_VERSION:-${HIVE_API_NODE_VERSION}}
+ image: ${HIVEMIND_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind/postgrest-rewriter}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- apps
- hivemind
@@ -94,7 +86,7 @@ services:
start_period: 1m
depends_on:
hivemind-postgrest-server:
- condition: service_started # TODO ADD HC TO POSTGREST-SERVER
+ condition: service_healthy
networks:
- haf-network
--
GitLab
From 54f67697b400198b7c05bba880e7e019684a108e Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 5 Dec 2024 03:56:24 +0000
Subject: [PATCH 59/78] add option to run legcy python server for now, and
increase postgrest connection pool (seems to help loading)
---
exposed/hivemind.exposed.yaml | 8 ++++++++
haf_base.yaml | 1 +
hivemind.yaml | 24 ++++++++++++++++++++----
pgtune.conf | 2 +-
postgrest/Dockerfile | 2 +-
5 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/exposed/hivemind.exposed.yaml b/exposed/hivemind.exposed.yaml
index 9ad23db..51657c3 100644
--- a/exposed/hivemind.exposed.yaml
+++ b/exposed/hivemind.exposed.yaml
@@ -1,4 +1,12 @@
services:
+ # When we want to benchmark the old hivemind server against the new, you can use this
+ # to expose it on a different port from the postgrest version. For everyone else,
+ # it's not needed at all. This should be removed once we remove the config for the
+ # python version of the server.
+ #
+ # hivemind-server:
+ # ports:
+ # - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-3}7002:8080"
hivemind-postgrest-rewriter:
ports:
- "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7002:80"
diff --git a/haf_base.yaml b/haf_base.yaml
index e203b57..2f9d54f 100644
--- a/haf_base.yaml
+++ b/haf_base.yaml
@@ -56,6 +56,7 @@ services:
host haf_block_log hivemind ${COMPOSE_PROJECT_NAME}-hivemind-install-1.${NETWORK_NAME} trust\n
host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-hivemind-block-processing-1.${NETWORK_NAME} trust\n
host haf_block_log hivemind ${COMPOSE_PROJECT_NAME}-hivemind-block-processing-1.${NETWORK_NAME} trust\n
+ host haf_block_log haf_admin ${COMPOSE_PROJECT_NAME}-hivemind-server-1.${NETWORK_NAME} trust\n
host haf_block_log hivemind ${COMPOSE_PROJECT_NAME}-hivemind-server-1.${NETWORK_NAME} trust\n
host haf_block_log hivemind ${COMPOSE_PROJECT_NAME}-hivemind-postgrest-server-1.${NETWORK_NAME} trust\n
diff --git a/hivemind.yaml b/hivemind.yaml
index dc1dc87..1c4f0ab 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -23,6 +23,22 @@ services:
reputation-tracker-install:
condition: service_completed_successfully
+ hivemind-server:
+ image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
+ profiles:
+ - apps
+ - hivemind
+ networks:
+ haf-network:
+ command:
+ - "--database-url=postgresql://hivemind@haf/haf_block_log"
+ - "server"
+ depends_on:
+ hivemind-block-processing:
+ condition: service_healthy
+ haf:
+ condition: service_healthy
+
hivemind-postgrest-server:
image: ${POSTGREST_IMAGE:-registry.gitlab.syncad.com/hive/haf_api_node/postgrest}:${POSTGREST_VERSION:-latest}
profiles:
@@ -33,15 +49,15 @@ services:
environment:
PGRST_ADMIN_SERVER_PORT: 3001
PGRST_SERVER_PORT: 8080
- PGRST_DB_URI: "postgresql://hivemind@haf/haf_block_log?application_name=hive-mind-postgrest" #warning hivemind improperly matches all connections starting from `hivemind` and crashes if they are made externally (not from python)
+ PGRST_DB_URI: "postgresql://hivemind@haf/haf_block_log?application_name=hive-mind-postgrest" # warning hivemind improperly matches all connections starting from `hivemind` and crashes if they are made externally (not from python)
PGRST_DB_SCHEMA: hivemind_endpoints
- #PGRST_DB_ROOT_SPEC: home
+ # PGRST_DB_ROOT_SPEC: home
PGRST_DB_ANON_ROLE: hivemind
- PGRST_DB_POOL: 20
+ PGRST_DB_POOL: 40
PGRST_DB_POOL_ACQUISITION_TIMEOUT: 10
PGRST_OPENAPI_MODE: "disabled" # unclear why this is set, I guess because we currently only support json-rpc?
# when debugging, you can enable this
- PGRST_DB_PLAN_ENABLED: true
+ # PGRST_DB_PLAN_ENABLED: true
healthcheck:
test: ["CMD-SHELL", "wget --timeout=2 -nv -t1 --spider 127.0.0.1:3001/ready || exit 1"]
interval: 10s
diff --git a/pgtune.conf b/pgtune.conf
index aec54b7..b80b4fc 100644
--- a/pgtune.conf
+++ b/pgtune.conf
@@ -1,5 +1,5 @@
shared_buffers = 16GB
-effective_cache_size = 8GB
+effective_cache_size = 48GB
maintenance_work_mem = 4GB
work_mem = 1024MB
diff --git a/postgrest/Dockerfile b/postgrest/Dockerfile
index c74b1a7..0b2f087 100644
--- a/postgrest/Dockerfile
+++ b/postgrest/Dockerfile
@@ -1,4 +1,4 @@
-ARG POSTGREST_VERSION=v12.0.2
+ARG POSTGREST_VERSION=v12.2.3
FROM postgrest/postgrest:${POSTGREST_VERSION} AS postgrest-base
FROM alpine:latest AS base
COPY --from=postgrest-base /bin/postgrest /bin
--
GitLab
From 8293bd0702440c282a5ab677f3196340c1c69fd7 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sat, 14 Dec 2024 15:55:58 -0500
Subject: [PATCH 60/78] Separate the hivemind installer so that it's not
integrated into the block-processing container. That allows us to fire up
the postgrest server as soon as the installer is finished, instead of having
to wait until the block processor starts processing blocks after sync/replay
has finished
---
hivemind.yaml | 45 +++++++++++++++++++++------------------------
1 file changed, 21 insertions(+), 24 deletions(-)
diff --git a/hivemind.yaml b/hivemind.yaml
index 1c4f0ab..6aa8a1b 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -1,4 +1,20 @@
services:
+ hivemind-install:
+ image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
+ profiles:
+ - apps
+ - hivemind
+ networks:
+ haf-network:
+ command:
+ - "setup"
+ - "--database-admin-url=postgresql://haf_admin@haf/haf_block_log"
+ depends_on:
+ haf:
+ condition: service_healthy
+ reputation-tracker-install:
+ condition: service_completed_successfully
+
hivemind-block-processing:
image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
@@ -10,7 +26,6 @@ services:
- "sync"
- "--database-url=postgresql://hivemind@haf/haf_block_log"
- "--database-admin-url=postgresql://haf_admin@haf/haf_block_log"
- - "--install-app"
healthcheck:
test: ["CMD-SHELL","/home/hivemind/block-processing-healthcheck.sh || exit 1"]
interval: 10s
@@ -22,12 +37,13 @@ services:
condition: service_healthy
reputation-tracker-install:
condition: service_completed_successfully
+ hivemind-install:
+ condition: service_completed_successfully
hivemind-server:
image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
profiles:
- - apps
- - hivemind
+ - legacy-hivemind-server
networks:
haf-network:
command:
@@ -65,29 +81,10 @@ services:
retries: 10
start_period: 1m
depends_on:
- hivemind-block-processing:
- condition: service_healthy
- haf:
- condition: service_healthy
-
- # note: the hivemind-install steps, which create/update the hivemind database schema, are also run automatically at the beginning
- # of the hivemind-block-processing image's startup. There is normally no reason to run this step separately. If you do,
- # be sure that the hivemind block-processor is not running at the same time as this install script, as doing so may corrupt your
- # database
- hivemind-install:
- image: ${HIVEMIND_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
- profiles:
- - hivemind-install
- networks:
- haf-network:
- command:
- - "setup"
- - "--database-admin-url=postgresql://haf_admin@haf/haf_block_log"
- depends_on:
+ hivemind-install:
+ condition: service_completed_successfully
haf:
condition: service_healthy
- reputation-tracker-install:
- condition: service_completed_successfully
hivemind-postgrest-rewriter:
image: ${HIVEMIND_REWRITER_IMAGE:-${HIVE_API_NODE_REGISTRY:-registry.hive.blog}/hivemind/postgrest-rewriter}:${HIVEMIND_VERSION:-${HIVE_API_NODE_VERSION}}
--
GitLab
From dd425e7995640e0b0cc404f10e6f557aec563f43 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sun, 15 Dec 2024 19:26:45 +0000
Subject: [PATCH 61/78] create_zfs_dataset creates an empty snapshot,
snapshot_zfs_datasets checks if...
---
create_zfs_datasets.sh | 3 +++
snapshot_zfs_datasets.sh | 27 +++++++++++++++++++++++++++
2 files changed, 30 insertions(+)
diff --git a/create_zfs_datasets.sh b/create_zfs_datasets.sh
index 7441b05..c6b70dd 100755
--- a/create_zfs_datasets.sh
+++ b/create_zfs_datasets.sh
@@ -133,3 +133,6 @@ mkdir -p "$TOP_LEVEL_DATASET_MOUNTPOINT/logs/caddy"
chown -R 1000:100 "$TOP_LEVEL_DATASET_MOUNTPOINT/logs"
# 105:109 is postgres:postgres inside the container
chown -R 105:109 "$TOP_LEVEL_DATASET_MOUNTPOINT/logs/postgresql" "$TOP_LEVEL_DATASET_MOUNTPOINT/logs/pgbadger"
+
+# Create a snapshot called 'empty'
+./snapshot_zfs_datasets.sh empty
diff --git a/snapshot_zfs_datasets.sh b/snapshot_zfs_datasets.sh
index 62dc339..960f697 100755
--- a/snapshot_zfs_datasets.sh
+++ b/snapshot_zfs_datasets.sh
@@ -134,6 +134,33 @@ fi
echo "All datasets appear unmountable"
+if [ "$SNAPSHOT_NAME" != "empty" ]; then
+ if [ ! -e "${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory/shared_memory.bin" ]; then
+ echo "Warning: No shared memory file found in the shared_memory directory"
+ exit 1
+ fi
+
+ last_shared_memory_write=$(stat -c %Y "${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory/shared_memory.bin")
+ last_blockchain_write=$(find "${TOP_LEVEL_DATASET_MOUNTPOINT}/blockchain" -type f -printf '%T@\n' | sort -n | tail -1)
+
+ if [ -z "$last_blockchain_write" ]; then
+ echo "Warning: No files found in the blockchain directory"
+ exit 1
+ fi
+
+ time_diff=$((last_blockchain_write - last_shared_memory_write))
+
+ if [ $time_diff -gt 300 ] || [ $time_diff -lt -300 ]; then
+ echo "Warning: The shared_memory.bin file was not written to within 5 minutes of the last write to a file in the blockchain directory."
+ read -p "Do you want to continue? (y/n): " choice
+ case "$choice" in
+ y|Y ) echo "Continuing...";;
+ n|N ) echo "Aborting."; exit 1;;
+ * ) echo "Invalid input. Aborting."; exit 1;;
+ esac
+ fi
+fi
+
if [ $PUBLIC_SNAPSHOT -eq 1 ]; then
stdbuf -o0 echo ""
stdbuf -o0 echo "Moving log files out of the dataset because this is a public snapshot... "
--
GitLab
From 1c0690a237250a52bdeb75f98cf55f91a66c0a86 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sun, 15 Dec 2024 19:40:24 +0000
Subject: [PATCH 62/78] fix rc version in example env
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 500302f..9b348f3 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.6rc10
+HIVE_API_NODE_VERSION=1.27.7rc10
# Global settings
--
GitLab
From 7454bf793c2550ae9bc218bcea86a4c4c29fe16a Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sun, 15 Dec 2024 20:52:46 +0000
Subject: [PATCH 63/78] Update .env.example
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 9b348f3..9672e42 100644
--- a/.env.example
+++ b/.env.example
@@ -71,7 +71,7 @@ NETWORK_NAME=haf
#
# Example how to use monitoring services
#
-# ARGUMENTS="--replay-blockchain --stop-replay-at-block 5000000 --exit-before-sync --block-stats-report-output=NOTIFY --block-stats-report-type=FULL --notifications-endpoint=hived-pme:9185"
+# ARGUMENTS="--replay-blockchain --stop-at-block 5000000 --block-stats-report-output=NOTIFY --block-stats-report-type=FULL --notifications-endpoint=hived-pme:9185"
#
# Mandatory options are:
# --block-stats-report-output=NOTIFY --block-stats-report-type=FULL --notifications-endpoint=hived-pme:9185
--
GitLab
From 602dcedcdd5ec51e01ea8dd8f113485e2553ff9f Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 16 Dec 2024 15:52:43 -0500
Subject: [PATCH 64/78] Fix the check in zfs snapshot script
---
create_zfs_datasets.sh | 15 ++++++++++++---
snapshot_zfs_datasets.sh | 26 ++++++++++++++++++--------
2 files changed, 30 insertions(+), 11 deletions(-)
diff --git a/create_zfs_datasets.sh b/create_zfs_datasets.sh
index c6b70dd..1e46050 100755
--- a/create_zfs_datasets.sh
+++ b/create_zfs_datasets.sh
@@ -6,7 +6,7 @@ print_help() {
echo "Usage: $0 --env-file=filename"
}
-OPTIONS=$(getopt -o he: --long env-file:,help,zpool:,top-level-dataset: -n "$0" -- "$@")
+OPTIONS=$(getopt -o he:s --long env-file:,help,zpool:,top-level-dataset:,skip-empty-snapshot -n "$0" -- "$@")
if [ $? -ne 0 ]; then
print_help
@@ -17,6 +17,7 @@ ZPOOL=""
TOP_LEVEL_DATASET=""
ZPOOL_MOUNT_POINT=""
TOP_LEVEL_DATASET_MOUNTPOINT=""
+SKIP_EMPTY_SNAPSHOT=false
eval set -- "$OPTIONS"
@@ -34,6 +35,10 @@ while true; do
TOP_LEVEL_DATASET="$2"
shift 2
;;
+ --skip-empty-snapshot|-s)
+ SKIP_EMPTY_SNAPSHOT=true
+ shift
+ ;;
--help|-h)
print_help
exit 0
@@ -134,5 +139,9 @@ chown -R 1000:100 "$TOP_LEVEL_DATASET_MOUNTPOINT/logs"
# 105:109 is postgres:postgres inside the container
chown -R 105:109 "$TOP_LEVEL_DATASET_MOUNTPOINT/logs/postgresql" "$TOP_LEVEL_DATASET_MOUNTPOINT/logs/pgbadger"
-# Create a snapshot called 'empty'
-./snapshot_zfs_datasets.sh empty
+if [ "$SKIP_EMPTY_SNAPSHOT" = false ]; then
+ # Create a snapshot called 'empty'
+ ./snapshot_zfs_datasets.sh empty
+else
+ echo "Skipping creation of 'empty' snapshot."
+fi
diff --git a/snapshot_zfs_datasets.sh b/snapshot_zfs_datasets.sh
index 960f697..1e0791c 100755
--- a/snapshot_zfs_datasets.sh
+++ b/snapshot_zfs_datasets.sh
@@ -13,9 +13,10 @@ print_help() {
echo " dataset, then swap back afterwards. That way the large logs files aren't"
echo " of the snapshots. This is a lot faster, but makes managing datasets more"
echo " complicated, so only use it if you really need to"
+ echo " --force, -f continue without prompting, even if warnings are detected"
}
-OPTIONS=$(getopt -o he:pt:l: --long env-file:,help,zpool:,top-level-dataset:,public-snapshot,temp-dir:,swap-logs-with-dataset: -n "$0" -- "$@")
+OPTIONS=$(getopt -o he:pt:l:f --long env-file:,help,zpool:,top-level-dataset:,public-snapshot,temp-dir:,swap-logs-with-dataset:,force -n "$0" -- "$@")
if [ $? -ne 0 ]; then
print_help
@@ -29,6 +30,7 @@ TOP_LEVEL_DATASET_MOUNTPOINT=""
PUBLIC_SNAPSHOT=0
SWAP_LOGS_DATASET=""
TMPDIR=/tmp
+FORCE=0
eval set -- "$OPTIONS"
@@ -58,6 +60,10 @@ while true; do
SWAP_LOGS_DATASET="$2"
shift 2
;;
+ --force|-f)
+ FORCE=1
+ shift 2
+ ;;
--help|-h)
print_help
exit 0
@@ -141,7 +147,7 @@ if [ "$SNAPSHOT_NAME" != "empty" ]; then
fi
last_shared_memory_write=$(stat -c %Y "${TOP_LEVEL_DATASET_MOUNTPOINT}/shared_memory/shared_memory.bin")
- last_blockchain_write=$(find "${TOP_LEVEL_DATASET_MOUNTPOINT}/blockchain" -type f -printf '%T@\n' | sort -n | tail -1)
+ last_blockchain_write=$(find "${TOP_LEVEL_DATASET_MOUNTPOINT}/blockchain" -type f -printf '%T@\n' | sort -n | tail -1 | cut -d. -f1)
if [ -z "$last_blockchain_write" ]; then
echo "Warning: No files found in the blockchain directory"
@@ -152,12 +158,16 @@ if [ "$SNAPSHOT_NAME" != "empty" ]; then
if [ $time_diff -gt 300 ] || [ $time_diff -lt -300 ]; then
echo "Warning: The shared_memory.bin file was not written to within 5 minutes of the last write to a file in the blockchain directory."
- read -p "Do you want to continue? (y/n): " choice
- case "$choice" in
- y|Y ) echo "Continuing...";;
- n|N ) echo "Aborting."; exit 1;;
- * ) echo "Invalid input. Aborting."; exit 1;;
- esac
+ if [ "$FORCE" -eq 1 ]; then
+ echo "Continuing due to --force option."
+ else
+ read -p "Do you want to continue? (y/n): " choice
+ case "$choice" in
+ y|Y ) echo "Continuing...";;
+ n|N ) echo "Aborting."; exit 1;;
+ * ) echo "Invalid input. Aborting."; exit 1;;
+ esac
+ fi
fi
fi
--
GitLab
From 0e7a514e4ff8e0caa929794e21a6989ac38d49b4 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 16 Dec 2024 21:44:06 +0000
Subject: [PATCH 65/78] Update log rotation scripts to match changed services
---
.../haf_block_explorer.log_rotation.yaml | 3 +
log_rotation/hivemind.log_rotation.yaml | 6 +
.../reputation_tracker.log_rotation.yaml | 2 +-
log_rotation/sync_log_rotation.py | 103 ++++++++++++++++++
4 files changed, 113 insertions(+), 1 deletion(-)
create mode 100755 log_rotation/sync_log_rotation.py
diff --git a/log_rotation/haf_block_explorer.log_rotation.yaml b/log_rotation/haf_block_explorer.log_rotation.yaml
index cb6a505..df3a05f 100644
--- a/log_rotation/haf_block_explorer.log_rotation.yaml
+++ b/log_rotation/haf_block_explorer.log_rotation.yaml
@@ -23,3 +23,6 @@ services:
block-explorer-install-schema:
logging:
driver: local
+ block-explorer-install-balance-tracker:
+ logging:
+ driver: local
diff --git a/log_rotation/hivemind.log_rotation.yaml b/log_rotation/hivemind.log_rotation.yaml
index 9c37aa0..9fc78cd 100644
--- a/log_rotation/hivemind.log_rotation.yaml
+++ b/log_rotation/hivemind.log_rotation.yaml
@@ -17,3 +17,9 @@ services:
hivemind-uninstall:
logging:
driver: local
+ hivemind-postgrest-server:
+ logging:
+ driver: local
+ hivemind-postgrest-rewriter:
+ logging:
+ driver: local
diff --git a/log_rotation/reputation_tracker.log_rotation.yaml b/log_rotation/reputation_tracker.log_rotation.yaml
index b667bc2..e48a86a 100644
--- a/log_rotation/reputation_tracker.log_rotation.yaml
+++ b/log_rotation/reputation_tracker.log_rotation.yaml
@@ -11,6 +11,6 @@ services:
reputation-tracker-postgrest-rewriter:
logging:
driver: local
- reputation-tracker-install-schema:
+ reputation-tracker-install:
logging:
driver: local
diff --git a/log_rotation/sync_log_rotation.py b/log_rotation/sync_log_rotation.py
new file mode 100755
index 0000000..7b20d79
--- /dev/null
+++ b/log_rotation/sync_log_rotation.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+# This helper script can be run after editing the main .yaml files in the top-level
+# haf_api_node directory. It will detect when new services have been added or removed,
+# and it will update the corresponding yaml files in this directory by adding or removing
+# services. Newly-detected services will be added with a default config, edit by hand
+# if you want to add rotation.
+#
+# You'll need to edit compose.yml yourself if you've added/removed new yaml files
+#
+# Run this script from the top-level haf_api_node directory (..)
+
+import os
+import yaml
+import sys
+
+def load_yaml(file_path):
+ """Load YAML content from a file."""
+ if not os.path.exists(file_path):
+ return {}
+ with open(file_path, 'r') as f:
+ try:
+ return yaml.safe_load(f) or {}
+ except yaml.YAMLError as exc:
+ print(f"Error parsing YAML file {file_path}: {exc}")
+ sys.exit(1)
+
+def save_yaml(data, file_path):
+ """Save YAML content to a file."""
+ with open(file_path, 'w') as f:
+ yaml.dump(data, f, default_flow_style=False, sort_keys=False)
+
+def ensure_log_rotation_directory(log_rotation_dir):
+ """Ensure the log_rotation directory exists."""
+ if not os.path.exists(log_rotation_dir):
+ os.makedirs(log_rotation_dir)
+ print(f"Created directory: {log_rotation_dir}")
+
+def get_top_level_yaml_files(top_level_dir, exclude_files=None):
+ """Get all .yaml files in the top-level directory, excluding specified files."""
+ if exclude_files is None:
+ exclude_files = []
+ yaml_files = []
+ for file in os.listdir(top_level_dir):
+ if (
+ file.endswith('.yaml') or file.endswith('.yml')
+ ) and os.path.isfile(os.path.join(top_level_dir, file)) and file not in exclude_files:
+ yaml_files.append(file)
+ return yaml_files
+
+def sync_log_rotation(top_level_dir, log_rotation_dir, exclude_files=None):
+ """Synchronize log rotation configurations with top-level Docker Compose files."""
+ if exclude_files is None:
+ exclude_files = []
+ ensure_log_rotation_directory(log_rotation_dir)
+ yaml_files = get_top_level_yaml_files(top_level_dir, exclude_files)
+
+ for yaml_file in yaml_files:
+ top_yaml_path = os.path.join(top_level_dir, yaml_file)
+ top_yaml = load_yaml(top_yaml_path)
+ services = top_yaml.get('services', {}).keys()
+
+ log_rotation_file = os.path.splitext(yaml_file)[0] + '.log_rotation.yaml'
+ log_rotation_path = os.path.join(log_rotation_dir, log_rotation_file)
+
+ log_yaml = load_yaml(log_rotation_path)
+ log_services = log_yaml.get('services', {})
+
+ # Remove services that no longer exist
+ removed_services = set(log_services.keys()) - set(services)
+ if removed_services:
+ for svc in removed_services:
+ del log_services[svc]
+ print(f"Removed services from {log_rotation_file}: {', '.join(removed_services)}")
+
+ # Add new services with default logging config
+ added_services = set(services) - set(log_services.keys())
+ if added_services:
+ for svc in added_services:
+ log_services[svc] = {
+ 'logging': {
+ 'driver': 'local'
+ }
+ }
+ print(f"Added services to {log_rotation_file}: {', '.join(added_services)}")
+
+ # Update the log_yaml structure
+ log_yaml['services'] = log_services
+
+ # Save the updated log rotation file
+ save_yaml(log_yaml, log_rotation_path)
+ print(f"Updated log rotation file: {log_rotation_file}")
+
+def main():
+ """Main function to execute the synchronization."""
+ top_level_dir = os.getcwd() # Current working directory
+ log_rotation_dir = os.path.join(top_level_dir, 'log_rotation')
+ exclude_files = ['compose.yml', 'compose.yaml', '.gitlab-ci.yaml', '.gitlab-ci.yml'] # Add any other files to exclude if necessary
+
+ sync_log_rotation(top_level_dir, log_rotation_dir, exclude_files)
+
+if __name__ == "__main__":
+ main()
--
GitLab
From 475c051007d18e605d2a6003d55a866e1ff15131 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Tue, 17 Dec 2024 16:34:51 +0000
Subject: [PATCH 66/78] Bump default version number to rc12
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 9672e42..f5b8c65 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.7rc10
+HIVE_API_NODE_VERSION=1.27.7rc12
# Global settings
--
GitLab
From 16e7e5ccab109c44a96e71b41b0e63afccfdc4e0 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 18 Dec 2024 15:54:35 +0000
Subject: [PATCH 67/78] Bump pgadmin to 8.14
---
backend.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend.yaml b/backend.yaml
index c3560bc..0331513 100644
--- a/backend.yaml
+++ b/backend.yaml
@@ -20,7 +20,7 @@ services:
condition: service_healthy
pgadmin:
- image: dpage/pgadmin4:${PGADMIN_VERSION:-8.13}
+ image: dpage/pgadmin4:${PGADMIN_VERSION:-8.14}
profiles:
- admin
environment:
--
GitLab
From 84e6c0ff7f7b73b657771092a75e041190a48b83 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 19 Dec 2024 17:18:43 +0000
Subject: [PATCH 68/78] Allow hivemind's block processor 30s to shut down
cleanly before killing it
---
hivemind.yaml | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/hivemind.yaml b/hivemind.yaml
index 6aa8a1b..ac96fde 100644
--- a/hivemind.yaml
+++ b/hivemind.yaml
@@ -32,6 +32,14 @@ services:
timeout: 2s
retries: 10
start_period: 72h
+ # after requesting a shutdown with SIGTERM, allow the container 30 seconds to exit
+ # before killing it.
+ # Normally it will exit cleanly in a second or two, but it can take longer if it's
+ # in the middle of massive sync. When in massive sync, a forced shutdown can leave
+ # the hivemind database in an inconsistent state, requiring an uninstall/reinstall
+ # of hivemind to correct it.
+ stop_grace_period: 30s
+ stop_signal: SIGINT
depends_on:
haf:
condition: service_healthy
--
GitLab
From be29f1228c0ba862fbb572862678b99d2600bb62 Mon Sep 17 00:00:00 2001
From: Eric Frias <efrias@syncad.com>
Date: Sat, 21 Dec 2024 17:49:18 -0500
Subject: [PATCH 69/78] Force nginx to parse and store the request body in a
way that it can be logged
---
drone/nginx.conf | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/drone/nginx.conf b/drone/nginx.conf
index df0441d..d37b4fe 100644
--- a/drone/nginx.conf
+++ b/drone/nginx.conf
@@ -49,13 +49,16 @@ http {
server_name _;
location / {
- access_log /var/log/drone/access_log api_log buffer=32k flush=5s;
+ #access_log /var/log/drone/access_log api_log buffer=32k flush=5s;
# switch to this to log in jsonl format instead
access_log /var/log/drone/access_log.json json_log buffer=32k flush=5s;
proxy_pass http://drone;
# Allow fast streaming HTTP/1.1 pipes (keep-alive, unbuffered)
proxy_http_version 1.1;
- proxy_request_buffering off;
+
+ client_body_buffer_size 128k; # Adjust size as needed
+ proxy_request_buffering on;
+ client_body_in_single_buffer on;
proxy_buffering off;
#proxy_set_header forwarded 'by=\"_$hostname\";$for_addr;proto=$scheme;host=\"$http_host\"';
}
--
GitLab
From eb395cb884c61ae2c8ee3fd1b3fea28042568972 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 20 Dec 2024 15:58:29 -0500
Subject: [PATCH 70/78] default to using lz4 for toast compression
---
create_zfs_datasets.sh | 1 +
1 file changed, 1 insertion(+)
diff --git a/create_zfs_datasets.sh b/create_zfs_datasets.sh
index 1e46050..c06cf81 100755
--- a/create_zfs_datasets.sh
+++ b/create_zfs_datasets.sh
@@ -127,6 +127,7 @@ chown -R 105:109 "$TOP_LEVEL_DATASET_MOUNTPOINT/haf_db_store"
mkdir -p "$TOP_LEVEL_DATASET_MOUNTPOINT/haf_postgresql_conf.d"
cp pgtune.conf "$TOP_LEVEL_DATASET_MOUNTPOINT/haf_postgresql_conf.d"
cp zfs.conf "$TOP_LEVEL_DATASET_MOUNTPOINT/haf_postgresql_conf.d"
+cp compression.conf "$TOP_LEVEL_DATASET_MOUNTPOINT/haf_postgresql_conf.d"
cp logging.conf "$TOP_LEVEL_DATASET_MOUNTPOINT/haf_postgresql_conf.d"
# 105:109 is postgres:postgres inside the container
chown -R 105:109 "$TOP_LEVEL_DATASET_MOUNTPOINT/haf_postgresql_conf.d"
--
GitLab
From 178fa60aa6d15687666600663e4037a63c7357f5 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 20 Dec 2024 16:00:34 -0500
Subject: [PATCH 71/78] add compression.conf
---
compression.conf | 1 +
1 file changed, 1 insertion(+)
create mode 100644 compression.conf
diff --git a/compression.conf b/compression.conf
new file mode 100644
index 0000000..16916e8
--- /dev/null
+++ b/compression.conf
@@ -0,0 +1 @@
+default_toast_compress = 'lz4'
--
GitLab
From f94b1d09149b4a5ed2d2f5077cb83e253d692ae4 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 20 Dec 2024 19:11:12 -0500
Subject: [PATCH 72/78] up to rc13 and switch to lz4 for toast
---
.env.example | 2 +-
compression.conf | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.env.example b/.env.example
index f5b8c65..240c1c0 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.7rc12
+HIVE_API_NODE_VERSION=1.27.7rc13
# Global settings
diff --git a/compression.conf b/compression.conf
index 16916e8..1f83380 100644
--- a/compression.conf
+++ b/compression.conf
@@ -1 +1 @@
-default_toast_compress = 'lz4'
+default_toast_compression = 'lz4'
--
GitLab
From fe8fdf23bfc8623b9bddd0054401da81447a1880 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Sat, 21 Dec 2024 18:20:15 -0500
Subject: [PATCH 73/78] up to rc14
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 240c1c0..d19551e 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.7rc13
+HIVE_API_NODE_VERSION=1.27.7rc14
# Global settings
--
GitLab
From 1b8d71cbdcb62eae12edcb7b2ff8f4bb28067dc6 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Mon, 23 Dec 2024 17:15:56 +0000
Subject: [PATCH 74/78] Update .env.example to rc15
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index d19551e..8a63bfe 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.7rc14
+HIVE_API_NODE_VERSION=1.27.7rc15
# Global settings
--
GitLab
From cc16fcc71fd37ab8e1cc418968a028789006d1c0 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 26 Dec 2024 13:46:46 -0500
Subject: [PATCH 75/78] increase cache times for some api calls
---
drone/config.yaml | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/drone/config.yaml b/drone/config.yaml
index f93b676..403664f 100644
--- a/drone/config.yaml
+++ b/drone/config.yaml
@@ -120,17 +120,20 @@ ttls:
appbase.condenser_api.get_account_history: 6
appbase.condenser_api.get_content: 6
appbase.condenser_api.get_profile: 6
+ appbase.condenser_api.get_follow_count: 6
appbase.database_api.find_accounts: 3
appbase.condenser_api.get_dynamic_global_properties: 1
hive: NO_CACHE
bridge: NO_CACHE
bridge.get_discussion: 6
bridge.get_account_posts: 12
- bridge.get_ranked_posts: 6
bridge.get_profile: 6
- bridge.get_community: 6
bridge.get_post: 6
- bridge.get_trending_topics: 3
+ bridge.unread_notifications: 3
+ bridge.get_ranked_posts: 6
+ bridge.get_community: 12
+ bridge.get_trending_topics: 30
+ bridge.does_user_follow_any_lists: 3
hafsql: NO_CACHE
# how long to wait for the backend to respond before giving up
--
GitLab
From 096741cd9b1b19efcdf58de078fada9ace20fff0 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Fri, 3 Jan 2025 19:43:01 +0000
Subject: [PATCH 76/78] Update .env.example to rc16 (for haf upgrade fixes)
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 8a63bfe..9ced4f3 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.7rc15
+HIVE_API_NODE_VERSION=1.27.7rc16
# Global settings
--
GitLab
From 554908693411fe043b04990d1d69ea04975ffa94 Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Thu, 9 Jan 2025 22:05:59 +0000
Subject: [PATCH 77/78] For exposed PostgREST services, expose the rewriter,
not the underlying PostgREST service. Also expose more ports on the
healthcheck image
---
exposed/haf_block_explorer.exposed.yaml | 4 ++--
exposed/hafah.exposed.yaml | 4 ++--
exposed/haproxy.exposed.yaml | 2 +-
exposed/reputation_tracker.exposed.yaml | 4 ++--
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/exposed/haf_block_explorer.exposed.yaml b/exposed/haf_block_explorer.exposed.yaml
index 6f3eee2..71f4af6 100644
--- a/exposed/haf_block_explorer.exposed.yaml
+++ b/exposed/haf_block_explorer.exposed.yaml
@@ -1,4 +1,4 @@
services:
- block-explorer-postgrest:
+ block-explorer-postgrest-rewriter:
ports:
- - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7005:3000"
+ - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7005:80"
diff --git a/exposed/hafah.exposed.yaml b/exposed/hafah.exposed.yaml
index cc2c0cc..cea7a3f 100644
--- a/exposed/hafah.exposed.yaml
+++ b/exposed/hafah.exposed.yaml
@@ -1,4 +1,4 @@
services:
- hafah-postgrest:
+ hafah-postgrest-rewriter:
ports:
- - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7003:3000"
+ - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7003:80"
diff --git a/exposed/haproxy.exposed.yaml b/exposed/haproxy.exposed.yaml
index c95ba96..3846ca5 100644
--- a/exposed/haproxy.exposed.yaml
+++ b/exposed/haproxy.exposed.yaml
@@ -3,4 +3,4 @@ services:
# expose the healthchecks so we can connect to them from an external haproxy
# shift the ports by 20000 so they don't conflict with haproxy's (if we want to expose those also)
ports:
- - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_HEALTHCHECK_PORT_PREFIX:-2}7001-${HAF_API_NODE_EXPOSED_HEALTHCHECK_PORT_PREFIX:-2}7005:7001-7005"
+ - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_HEALTHCHECK_PORT_PREFIX:-2}7001-${HAF_API_NODE_EXPOSED_HEALTHCHECK_PORT_PREFIX:-2}7009:7001-7009"
diff --git a/exposed/reputation_tracker.exposed.yaml b/exposed/reputation_tracker.exposed.yaml
index ed4c0c4..6303867 100644
--- a/exposed/reputation_tracker.exposed.yaml
+++ b/exposed/reputation_tracker.exposed.yaml
@@ -1,4 +1,4 @@
services:
- reputation-tracker-postgrest:
+ reputation-tracker-postgrest-rewriter:
ports:
- - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7009:3000"
\ No newline at end of file
+ - "${HAF_API_NODE_EXPOSED_IPADDR:-0.0.0.0}:${HAF_API_NODE_EXPOSED_PORT_PREFIX:-1}7009:80"
--
GitLab
From b683c59c5765ca2f6060673b6f62865e95bccccd Mon Sep 17 00:00:00 2001
From: Dan Notestein <dan@syncad.com>
Date: Wed, 15 Jan 2025 20:10:49 +0000
Subject: [PATCH 78/78] Update .env.example to 1.27.8
---
.env.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 9ced4f3..1920e3c 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ COMPOSE_PROFILES="core,admin,servers,apps"
# To use the same tagged version of all the Hive API node images,
# set it here. You can override the tags for individual images
# below
-HIVE_API_NODE_VERSION=1.27.7rc16
+HIVE_API_NODE_VERSION=1.27.8
# Global settings
--
GitLab