Commit 49ba06df authored by therealwolf's avatar therealwolf 💯
Browse files

fix and eslint

parent 01f9d0bb
import * as mongoose from 'mongoose';// mongoose.Promise = global.Promise
import * as Model from './../models/data.model'
import * as cast from '../helpers/cast'
import { remove_duplicates, create_arr_of_arrays } from '../helpers/convert'
import moment = require('moment')
import _g = require('../_g')
import * as _ from 'lodash'
export let find_extern = async (app?: string, account?: string, data_type?: string) => {
let q: any = {}
if (app) q.app = app
if (account) q.account = account
if (data_type) q.data_type = data_type
return await Model.DataLean.find(q)
}
export let find_intern = async (app?: string, account?: string, data_type?: string) => {
let q: any = {}
if (app) q.app = app
if (account) q.account = account
if (data_type) q.data_type = data_type
return await Model.DataDetailed.find(q)
}
export let get_data_by_date = async (when: string | 'today' | 'last_day' | 'before_last_day' | 'before_last_week' | 'last_week' | 'before_last_month' | 'last_month', app?: string, account?: string, data_type?: string) => {
let x = await find_intern(app, account, data_type)
import * as mongoose from "mongoose"; // mongoose.Promise = global.Promise
import * as Model from "./../models/data.model";
import * as cast from "../helpers/cast";
import { remove_duplicates, create_arr_of_arrays } from "../helpers/convert";
import moment = require("moment");
import _g = require("../_g");
import * as _ from "lodash";
export let find_extern = async (
app?: string,
account?: string,
data_type?: string
) => {
let q: any = {};
if (app) q.app = app;
if (account) q.account = account;
if (data_type) q.data_type = data_type;
return await Model.DataLean.find(q);
};
export let find_intern = async (
app?: string,
account?: string,
data_type?: string
) => {
let q: any = {};
if (app) q.app = app;
if (account) q.account = account;
if (data_type) q.data_type = data_type;
return await Model.DataDetailed.find(q);
};
export let get_data_by_date = async (
when:
| string
| "today"
| "last_day"
| "before_last_day"
| "before_last_week"
| "last_week"
| "before_last_month"
| "last_month",
app?: string,
account?: string,
data_type?: string
) => {
let x = await find_intern(app, account, data_type);
if (!x || x.length <= 0) {
return []
return [];
} else {
x = x[0]
let data = []
let should_length = 1 // can be used if not all data is available or maybe too much data for a day
if (when === 'today') {
data = x.data.filter(y => moment.utc(y.timestamp).format('YYYY-MM-DD') === moment.utc().format('YYYY-MM-DD'))
} else if (when === 'last_day' || when === 'before_last_day') {
let days = when === 'last_day' ? 1 : 2
data = x.data.filter(y => moment.utc(y.timestamp).format('YYYY-MM-DD') === moment.utc().subtract(days, 'd').format('YYYY-MM-DD'))
} else if (when === 'last_week' || when === 'before_last_week' || when === 'last_month' || when === 'before_last_month') {
let sub_days = 1
let days = 7
if (when === 'last_week') {
sub_days = 8 // one more since we don't have data for the current, not finished day
days = 7
} else if (when === 'before_last_week') {
sub_days = 15
days = 7
} else if (when === 'last_month') {
sub_days = 31
days = 30
} else if (when === 'before_last_month') {
sub_days = 61
days = 30
x = x[0];
let data = [];
let should_length = 1; // can be used if not all data is available or maybe too much data for a day
if (when === "today") {
data = x.data.filter(
(y) =>
moment.utc(y.timestamp).format("YYYY-MM-DD") ===
moment.utc().format("YYYY-MM-DD")
);
} else if (when === "last_day" || when === "before_last_day") {
let days = when === "last_day" ? 1 : 2;
data = x.data.filter(
(y) =>
moment.utc(y.timestamp).format("YYYY-MM-DD") ===
moment.utc().subtract(days, "d").format("YYYY-MM-DD")
);
} else if (
when === "last_week" ||
when === "before_last_week" ||
when === "last_month" ||
when === "before_last_month"
) {
let sub_days = 1;
let days = 7;
if (when === "last_week") {
sub_days = 8; // one more since we don't have data for the current, not finished day
days = 7;
} else if (when === "before_last_week") {
sub_days = 15;
days = 7;
} else if (when === "last_month") {
sub_days = 31;
days = 30;
} else if (when === "before_last_month") {
sub_days = 61;
days = 30;
}
data = x.data.filter(y => y.timestamp >= moment.utc().subtract(sub_days, 'd').toDate() && y.timestamp <= moment.utc().subtract(sub_days - days, 'd').toDate())
data = x.data.filter(
(y) =>
y.timestamp >= moment.utc().subtract(sub_days, "d").toDate() &&
y.timestamp <=
moment
.utc()
.subtract(sub_days - days, "d")
.toDate()
);
}
return data
return data;
}
}
};
export const get_sum_from_data = async (when: string | 'today' | 'last_day' | 'before_last_day' | 'before_last_week' | 'last_week' | 'before_last_month' | 'last_month', app?: string, account?: string, data_type?: string) => {
let data = await get_data_by_date(when, app, account, data_type)
export const get_sum_from_data = async (
when:
| string
| "today"
| "last_day"
| "before_last_day"
| "before_last_week"
| "last_week"
| "before_last_month"
| "last_month",
app?: string,
account?: string,
data_type?: string
) => {
let data = await get_data_by_date(when, app, account, data_type);
data = data.map(x => x.value)
let amount = 0
data = data.map((x) => x.value);
let amount = 0;
if (data.length > 0) {
amount = data.reduce((x, y) => x + y)
amount = data.reduce((x, y) => x + y);
}
return parseFloat(amount.toFixed(3))
}
return parseFloat(amount.toFixed(3));
};
export const get_sum_from_data_dau = async (when: string | 'today' | 'last_day' | 'before_last_day' | 'before_last_week' | 'last_week' | 'before_last_month' | 'last_month', app?: string, account?: string, data_type?: string) => {
let data = await get_data_by_date(when, app, account, data_type)
let users = Array.from(new Set(create_arr_of_arrays(data.map(x => x.users))))
_g.temp_global_data.dau[when] = _g.temp_global_data.dau[when].concat(users)
_g.temp_global_data.dau[when] = Array.from(new Set(_g.temp_global_data.dau[when]))
return users.length
}
export const get_sum_from_data_dau = async (
when:
| string
| "today"
| "last_day"
| "before_last_day"
| "before_last_week"
| "last_week"
| "before_last_month"
| "last_month",
app?: string,
account?: string,
data_type?: string
) => {
let data = await get_data_by_date(when, app, account, data_type);
let users = Array.from(
new Set(create_arr_of_arrays(data.map((x) => x.users)))
);
_g.temp_global_data.dau[when] = _g.temp_global_data.dau[when].concat(users);
_g.temp_global_data.dau[when] = Array.from(
new Set(_g.temp_global_data.dau[when])
);
return users.length;
};
export let create_or_add_both = async (app: string, data_type: string, data: Array<any>, account?: string) => {
await create_or_add(true, app, data_type, data, account)
await create_or_add(false, app, data_type, data, account)
}
export let create_or_add_both = async (
app: string,
data_type: string,
data: Array<any>,
account?: string
) => {
await create_or_add(true, app, data_type, data, account);
await create_or_add(false, app, data_type, data, account);
};
export let create_or_add = async (external: boolean, app: string, data_type: string, data: Array<any>, account?: string) => {
let data_obj = external ? await find_extern(app, account, data_type) : await find_intern(app, account, data_type)
export let create_or_add = async (
external: boolean,
app: string,
data_type: string,
data: Array<any>,
account?: string
) => {
let data_obj = external
? await find_extern(app, account, data_type)
: await find_intern(app, account, data_type);
if (data_obj.length > 0) {
// Check that data is not yet existing
data = data.filter(x => data_obj[0].data.filter(y => {
return moment.utc(y.timestamp).isSame(x.timestamp) //moment.utc(y.timestamp).format('YYYY-MM-DD') === moment.utc(x.timestamp).format('YYYY-MM-DD')
}).length <= 0)
data_obj[0].data = data_obj[0].data.concat(data)
data_obj[0].markModified('data')
await data_obj[0].save()
return data_obj[0]
data = data.filter(
(x) =>
data_obj[0].data.filter((y) => {
return moment.utc(y.timestamp).isSame(x.timestamp); //moment.utc(y.timestamp).format('YYYY-MM-DD') === moment.utc(x.timestamp).format('YYYY-MM-DD')
}).length <= 0
);
data_obj[0].data = data_obj[0].data.concat(data);
data_obj[0].markModified("data");
await data_obj[0].save();
return data_obj[0];
} else {
return external ? await Model.DataLean.create({ app, account, data_type, data }) : await Model.DataDetailed.create({ app, account, data_type, data })
console.log("create ", external, data);
return external
? await Model.DataLean.create({ app, account, data_type, data })
: await Model.DataDetailed.create({ app, account, data_type, data });
}
}
};
export let create_total_dau = async (app: string) => {
let data = []
let data = [];
console.log("create_total_dau");
// Get all data from accounts and group based on date
for(let data_type of [_g.data_type.dau_custom, _g.data_type.dau_meta, _g.data_type.dau_transfers, _g.data_type.dau_benefactor, _g.data_type.dau_vote_general]) {
let data_db = await find_intern(app, '', data_type)
for (let data_type of [
_g.data_type.dau_custom,
_g.data_type.dau_meta,
_g.data_type.dau_transfers,
_g.data_type.dau_benefactor,
_g.data_type.dau_vote_general,
]) {
let data_db = await find_intern(app, "", data_type);
for (let d of data_db) {
data = data.concat(d.data)
if (d && d.data && Array.isArray(d.data)) {
data = data.concat(d.data);
}
}
}
// Group data
let data_for_db = create_grouped_data_users(data)
let data_for_db = create_grouped_data_users(data);
// Create/Add as dau_total in DB
await create_or_add_both(app, _g.data_type.dau_total, data_for_db)
}
await create_or_add_both(app, _g.data_type.dau_total, data_for_db);
};
export let create_grouped_data_users = (data) => {
let grouped_data = _(data).groupBy(x => moment.utc(x.timestamp).toISOString()).value()
let data_for_db = []
for(let timestamp in grouped_data) {
let users = []
let grouped_data = _(data)
.groupBy((x) => moment.utc(x.timestamp).toISOString())
.value();
let data_for_db = [];
for (let timestamp in grouped_data) {
let users = [];
for (let g of grouped_data[timestamp]) {
users = users.concat(g.users)
users = users.concat(g.users);
}
// Remove duplicates created by other data_types
users = Array.from(new Set(users))
users = Array.from(new Set(users));
data_for_db.push({
users, value: users.length, timestamp
})
users,
value: users.length,
timestamp,
});
}
return data_for_db
}
return data_for_db;
};
// Experimental for next Update
export let get_last_month_array = async (app:string, arr:Array<{ keys:any, values: any }>) => {
for(let a of arr) {
for(let account of a.keys) {
for(let data_type of a.values) {
let x = await find_intern(app, account, data_type)
let data = x.data.filter(y => y.timestamp >= moment.utc().subtract(30, 'd').toDate())
export let get_last_month_array = async (
app: string,
arr: Array<{ keys: any; values: any }>
) => {
for (let a of arr) {
for (let account of a.keys) {
for (let data_type of a.values) {
let x = await find_intern(app, account, data_type);
let data = x.data.filter(
(y) => y.timestamp >= moment.utc().subtract(30, "d").toDate()
);
}
}
}
}
\ No newline at end of file
};
import * as dotenv from 'dotenv'; dotenv.config({ path: '.env' })
import * as db from './helpers/database'
import * as mongoose from 'mongoose'
import * as moment from 'moment'
import * as dotenv from "dotenv";
dotenv.config({ path: ".env" });
import * as db from "./helpers/database";
import * as mongoose from "mongoose";
import * as moment from "moment";
import { update_data, set_data } from './main/update/apps'
import { update_rank } from './main/update/rank'
import { convert_accepted_submissions } from './main/submission/accept'
import { update_hbd_and_hive_price } from './helpers/exchanges'
import { query } from './main/queries/query'
import { update_data, set_data } from "./main/update/apps";
import { update_rank } from "./main/update/rank";
import { convert_accepted_submissions } from "./main/submission/accept";
import { update_hbd_and_hive_price } from "./helpers/exchanges";
import { query } from "./main/queries/query";
import * as dau_query from './main/queries/dau'
import * as dau_query from "./main/queries/dau";
import * as db_app from './database/app.db'
import * as db_data from './database/data.db'
import * as db_global from './database/global.db'
import * as db_app from "./database/app.db";
import * as db_data from "./database/data.db";
import * as db_global from "./database/global.db";
import { update_global_properties } from './helpers/cast'
import { create_initial_apps } from './main/update/initial'
import { update_global_properties } from "./helpers/cast";
import { create_initial_apps } from "./main/update/initial";
import _g = require('./_g')
import _g = require("./_g");
var connected = false
var connection: any = {}
var connected = false;
var connection: any = {};
export var stop_bool = false
export var stop_bool = false;
let last_day = moment.utc().dayOfYear()
let startup = true
let last_day = moment.utc().dayOfYear();
let startup = true;
const connection_config = {
user: process.env.steemsql_user,
password: process.env.steemsql_pw,
server: process.env.steemsql_server,
database: 'DBHive', // TODO change to DBHive?
database: "DBHive", // TODO change to DBHive?
connectionTimeout: 10000, // 5 Minutes
requestTimeout: 1000 * 60 * 60 // 5 Minutes
}
requestTimeout: 1000 * 60 * 60, // 5 Minutes
};
export let start = async () => {
try {
while (!connected) {
await try_connection()
await try_connection();
}
_g.pool = await new _g.sql.ConnectionPool(connection_config).connect()
_g.global_data = await db_global.find()
if(process.env.NODE_ENV === 'production') await clean_old_data_entries()
main()
_g.pool = await new _g.sql.ConnectionPool(connection_config).connect();
_g.global_data = await db_global.find();
if (process.env.NODE_ENV === "production") await clean_old_data_entries();
main();
} catch (e) {
console.error('Main Error', e)
await _g.wait_sec(10)
if (e.message.includes('504')) {
connection.close()
start()
console.error("Main Error", e);
await _g.wait_sec(10);
if (e.message.includes("504")) {
connection.close();
start();
} else {
start()
start();
}
}
}
};
const format_accounts = async () => {
for(let app of await db_app.find_all()) {
let accounts = []
for(let acc of app.accounts) {
for (let app of await db_app.find_all()) {
let accounts = [];
for (let acc of app.accounts) {
let new_acc = {
id: app.accounts.indexOf(acc),
name: acc.name,
curation: acc.account_types.includes('curation'),
transfer: acc.account_types.includes('transfer'),
transfer_only_dau: acc.account_types.includes('transfer_only_dau'),
meta: acc.account_types.includes('meta'),
delegation: acc.account_types.includes('delegation'),
posting: acc.account_types.includes('posting')
}
accounts.push(new_acc)
console.log(acc, new_acc)
curation: acc.account_types.includes("curation"),
transfer: acc.account_types.includes("transfer"),
transfer_only_dau: acc.account_types.includes("transfer_only_dau"),
meta: acc.account_types.includes("meta"),
delegation: acc.account_types.includes("delegation"),
posting: acc.account_types.includes("posting"),
};
accounts.push(new_acc);
console.log(acc, new_acc);
}
app.accounts = accounts
app.save()
app.accounts = accounts;
app.save();
}
}
};
let main = async () => {
while (true) {
try {
await update_hbd_and_hive_price()
let i = await convert_accepted_submissions()
if(last_day !== moment.utc().dayOfYear() || i > 0 || (startup && process.env.NODE_ENV === 'production')) {
if(last_day !== moment.utc().dayOfYear()) last_day = moment.utc().dayOfYear()
startup = false
await update_hbd_and_hive_price();
let i = await convert_accepted_submissions();
if (
last_day !== moment.utc().dayOfYear() ||
i > 0 ||
(startup && process.env.NODE_ENV === "production")
) {
if (last_day !== moment.utc().dayOfYear())
last_day = moment.utc().dayOfYear();
startup = false;
// Update Global Properties for calculation of Vests to Steempower
console.log('update_global_properties')
await update_global_properties()
console.log("update_global_properties");
await update_global_properties();
// Update the data from approved Apps
console.log('update_data')
await update_data()
console.log("update_data");
await update_data();
// Set the data from approved Apps
console.log('set_data')
await set_data()
console.log("set_data");
await set_data();
// Sets the rank for approved Apps
console.log('update_rank')
await update_rank()
console.log("update_rank");
await update_rank();
}
console.log('Finished round - waiting 0.2 hours')
await _g.wait_hour(0.2)
console.log("Finished round - waiting 0.2 hours");
await _g.wait_hour(0.2);
} catch (error) {
console.error('main', error)
await _g.wait_sec(10)
main()
console.error("main", error);
await _g.wait_sec(10);
main();
}
}
}
};
let try_connection = async () => {
if (mongoose.connection.readyState === 1) {
connected = true
connected = true;
//log.notif('Successfully connected to DB')
} else {
connected = false;
connection = await db.connect()
connection = await db.connect();
}
}
};
const clean_old_data_entries = async () => {
console.log('Cleaning up entries')
console.log("Cleaning up entries");
// Detailed /w users
await