Commit cd6a6f27 authored by therealwolf's avatar therealwolf 💯
Browse files

initital commit after migration

parents
lib-cov
*.seed
*.log
*.csv
*.dat
*.out
*.pid
*.gz
*.swp
pids
logs
results
tmp
#Build
public/css/main.css
# API keys and secrets
.env
# Dependency directory
node_modules
bower_components
# Editors
.idea
*.iml
# OS metadata
.DS_Store
Thumbs.db
# Ignore built ts files
dist
MIT License
Copyright (c) 2018 therealwolf
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# Backend of HiveDapps.com - written in NodeJS
```
yarn
yarn run dev
```
.env File
```
MONGODB_URI_PROD=mongodb+srv://user:pw@link?retryWrites=true&w=majority
MONGODB_URI_STAGING=mongodb+srv://user:pw@link?retryWrites=true&w=majority
MONGODB_URI_DEV=mongodb://localhost:27017/devHiveDapps
steemsql_user=
steemsql_pw=
steemsql_server=
TRANSFER_THRESHOLD=
```
\ No newline at end of file
module.exports = {
apps : [{
name: "hivedapps-api",
script: "./dist/server.js",
node_args: "--max-old-space-size=4096",
env: {
NODE_ENV: "development",
},
env_production: {
NODE_ENV: "production",
},
env_staging: {
NODE_ENV: "staging",
}
}]
}
\ No newline at end of file
{
"name": "hivedapps-backend",
"version": "1.0.0",
"description": "",
"main": "dist/server.js",
"scripts": {
"start": "cross-env NODE_ENV=production ts-node src/server.ts",
"staging": "cross-env NODE_ENV=staging ts-node src/server.ts",
"dev": "cross-env NODE_ENV=development ts-node-dev src/server.ts"
},
"author": "therealwolf42",
"license": "MIT",
"dependencies": {
"@types/cors": "^2.8.4",
"@types/dotenv": "^6.1.0",
"@types/express": "^4.16.0",
"@types/helmet": "^0.0.42",
"@types/lodash": "^4.14.118",
"@types/mssql": "^4.0.11",
"@types/node": "^10.12.11",
"axios": "^0.18.0",
"celebrate": "^9.0.1",
"cors": "^2.8.5",
"cross-env": "^5.2.0",
"dotenv": "^6.1.0",
"dsteem": "^0.11.2",
"dsteem-dhive": "^0.12.13",
"express": "^4.16.4",
"express-promise-router": "^3.0.3",
"express-rate-limit": "^3.3.2",
"helmet": "^3.15.0",
"jsonwebtoken": "^8.4.0",
"lodash": "^4.17.11",
"moment": "^2.22.2",
"mongoose": "^5.3.14",
"mssql": "^4.3.0"
},
"devDependencies": {
"ts-node-dev": "^1.0.0-pre.44",
"typescript": "^3.8.3"
}
}
export const USE_TESTNET = false
export const sql = require('mssql')
export let pool = null
export let hbd_dollar_price = 1
export let hive_dollar_price = 1
export let hbd_hive_ratio = 1
export let global_users = []
export let temp_global_data = {
dau: {
last_day: [],
before_last_day: [],
last_week: [],
before_last_week: [],
last_month: [],
before_last_month: [],
last_month_array: []},
volume: { hbd: [], hive: [] },
tx: [],
rewards: { hbd: [], hive: [] }
}
export let global_data = {
dau: {
last_day: 0,
before_last_day: 0,
last_week: 0,
before_last_week: 0,
last_month: 0,
before_last_month: 0
},
volume: {
hbd: {
last_day: 0,
before_last_day: 0,
last_week: 0,
before_last_week: 0,
last_month: 0,
before_last_month: 0
},
hive: {
last_day: 0,
before_last_day: 0,
last_week: 0,
before_last_week: 0,
last_month: 0,
before_last_month: 0
}
},
tx: {
last_day: 0,
before_last_day: 0,
last_week: 0,
before_last_week: 0,
last_month: 0,
before_last_month: 0
},
rewards: { hbd: {}, hive: {} }
}
export let global_properties = {}
export const category = {
games: 'games',
entertainment: 'entertainment',
exchanges: 'exchanges',
development: 'development',
gambling: 'gambling',
wallet: 'wallet',
finance: 'finance',
promotion: 'promotion',
social: 'social',
media: 'media',
security: 'security',
utility: 'utility',
interface: 'interface',
education: 'education',
health: 'health',
content_discovery: 'content discovery'
}
export const data_type = {
volume_transfers_hbd: 'volume_transfers_hbd',
volume_transfers_hive: 'volume_transfers_hive',
volume_meta: 'volume_meta',
tx_transfers_hbd: 'tx_transfers_hbd',
tx_transfers_hive: 'tx_transfers_hive',
tx_custom: 'tx_custom',
dau_transfers_outgoing: 'dau_transfers_outgoing',
dau_transfers_incoming: 'dau_transfers_incoming',
dau_benefactor: 'dau_benefactor',
dau_transfers: 'dau_transfers',
dau_meta: 'dau_meta',
dau_custom: 'dau_custom',
dau_vote_general: 'dau_vote_general',
dau_total: 'dau_total',
tx_meta: 'tx_meta',
rewards_benefactor_hbd: 'rewards_benefactor_hbd',
rewards_benefactor_hive: 'rewards_benefactor_hive',
rewards_benefactor_vests: 'rewards_benefactor_vests',
rewards_curation: 'rewards_curation'
}
export const app_type = {
app: 'app',
dapp: 'dapp',
interface: 'interface'
}
export const tx_type = {
vote: 'vote',
transfer: 'transfer',
//transfer_to_vesting: 'transfer_to_vesting',
delegate: 'delegate'
//follow: 'follow'
}
export const account_types = {
transfer: 'transfer',
benefactor: 'benefactor',
curation: 'curation',
transfer_only_dau: 'transfer_only_dau',
delegation: 'delegation',
meta: 'meta',
posting: 'posting',
logo: 'logo'
}
export const app_status = {
live: 'live',
beta: 'beta',
alpha: 'alpha',
workinprogress: 'work-in-progress',
concept: 'concept',
abandoned: 'abandoned'
}
export let processing_accounts = []
export let adding_votes = false
export let wait_sec = async (sec) => {
return await Promise.all([
timeout(sec * 1000)
])
}
export let wait_hour = async (hour) => {
return await Promise.all([
timeout(hour * 60 * 60 * 1000)
])
}
let timeout = (ms: any) => {
return new Promise(resolve => setTimeout(resolve, ms));
}
import * as convert from '../helpers/convert'
import * as db_app from '../database/app.db'
import * as SubmissionModel from '../models/submission.model'
export let get_apps = async (req, res) => {
let { approved, sort, order, time, name, category, type } = req.query
if (approved === 'false') {
approved = false
} else {
approved = true
}
let apps = await db_app.find_approved(approved, sort, order, time, name, type, category)
return res.status(200).send({ apps })
}
export const get_app = async (req, res) => {
let { name } = req.params
const apps = await db_app.find(name)
const app = apps.length > 0 ? apps[0] : {}
return res.status(200).send({ app })
}
export const submit = async (req, res) => {
let { name, display_name, link, ref_link, logo, product_screenshot, description, short_description, status, app_type, category, social, accounts, custom_jsons } = req.body
if (!name) name = convert.clean_string(display_name)
let message = ''
await SubmissionModel.Submission.create({
display_name, name, accounts, description, short_description, app_type, logo, product_screenshot, link, ref_link, social, status, category, custom_jsons
})
return res.status(200).send({ message })
}
\ No newline at end of file
import * as db_data from '../database/data.db'
import * as _g from '../_g'
export let index = async (req, res, next) => {
return res.status(200).send({ app_types: _g.app_type })
}
\ No newline at end of file
import * as db_data from '../database/data.db'
import * as _g from '../_g'
export let index = async (req, res, next) => {
return res.status(200).send({ categories: _g.category })
}
\ No newline at end of file
import * as db_data from '../database/data.db'
export let get_data = async (req, res, next) => {
let { app, account, data_type, detailed } = req.query
detailed = false // force lean for now
let data = detailed ? await db_data.find_intern(app, account, data_type) : await db_data.find_extern(app, account, data_type)
return res.status(200).send({ data })
}
\ No newline at end of file
import * as db_data from '../database/data.db'
import * as _g from '../_g'
export let index = async (req, res, next) => {
return res.status(200).send({ data_types: _g.data_type })
}
\ No newline at end of file
import * as db_data from '../database/global.db'
import * as _g from '../_g'
export let index = async (req, res, next) => {
let obj:any = {
data: _g.global_data,
hbd_dollar_price: _g.hbd_dollar_price,
hive_dollar_price: _g.hive_dollar_price
}
return res.status(200).send(obj)
}
\ No newline at end of file
import * as mongoose from 'mongoose';// mongoose.Promise = global.Promise
import * as Model from './../models/app.model'
import * as cast from '../helpers/cast'
import * as moment from 'moment'
export interface AppData {
name: String
}
export let find = async (name?: string) => {
let q:any = {}
if(name) q.name = name
return await Model.App.find(q)
}
export let find_all = async (sort: string = 'rank', order: string = 'asc', time: string = 'last_week') => {
return await Model.App.find({}).sort(create_sort_query(sort, order, time))
}
export let find_approved = async (approved:boolean = null, sort: string = 'rank', order:string = 'asc', time: string = 'last_week', name?:string, app_type?:string, category?:string) => {
let q = {}
if(name) q['name'] = name
if(app_type) q['app_type'] = app_type
if(category) q['category'] = category
if(approved !== null) q['approved'] = approved
q['hived'] = true
return await Model.App.find(q).sort(create_sort_query(sort, order, time))
}
export let find_approved_lean = async (approved:boolean, sort: string = 'rank', order: string = 'asc', time: string = 'last_week') => {
const q = { approved, hived: true}
return await Model.App.find(q,{_id:0}).sort(create_sort_query(sort, order, time))/*.select('name')*/.lean()
}
export let create = async (name: string, display_name: string, description: string, main_account: string, accounts: { name: string, account_types: Model.AccountType[] }[],
logo: string, link: string, social: { discord?: string, twitter?: string, medium?: string, reddit?: string }, status: string, category: string, tags: string[], custom_jsons: string[], app_type: string | 'app' | 'dapp' | 'interface') => {
try {
return await Model.App.create({ name, display_name, description, main_account, accounts, logo, link, social, status, category, tags, custom_jsons, app_type, last_update: moment.utc().toDate() })
} catch (error) {
//console.error(error.message)
}
}
const create_sort_query = (sort: string, order, time) => {
let s = {}
if(sort.includes('volume')) {
sort = `volume.${sort.substring(sort.indexOf('_') + 1)}`
}
if(sort.includes('rewards')) {
sort = `rewards.${sort.substring(sort.indexOf('_') + 1)}`
}
s[`${sort}.${time}`] = order
return s
}
\ No newline at end of file
import * as mongoose from 'mongoose';// mongoose.Promise = global.Promise
import * as Model from './../models/data.model'
import * as cast from '../helpers/cast'
import { remove_duplicates, create_arr_of_arrays } from '../helpers/convert'
import moment = require('moment')
import _g = require('../_g')
import * as _ from 'lodash'
export let find_extern = async (app?: string, account?: string, data_type?: string) => {
let q: any = {}
if (app) q.app = app
if (account) q.account = account
if (data_type) q.data_type = data_type
return await Model.DataLean.find(q)
}
export let find_intern = async (app?: string, account?: string, data_type?: string) => {
let q: any = {}
if (app) q.app = app
if (account) q.account = account
if (data_type) q.data_type = data_type
return await Model.DataDetailed.find(q)
}
export let get_data_by_date = async (when: string | 'today' | 'last_day' | 'before_last_day' | 'before_last_week' | 'last_week' | 'before_last_month' | 'last_month', app?: string, account?: string, data_type?: string) => {
let x = await find_intern(app, account, data_type)
if (!x || x.length <= 0) {
return []
} else {
x = x[0]
let data = []
let should_length = 1 // can be used if not all data is available or maybe too much data for a day
if (when === 'today') {
data = x.data.filter(y => moment.utc(y.timestamp).format('YYYY-MM-DD') === moment.utc().format('YYYY-MM-DD'))
} else if (when === 'last_day' || when === 'before_last_day') {
let days = when === 'last_day' ? 1 : 2
data = x.data.filter(y => moment.utc(y.timestamp).format('YYYY-MM-DD') === moment.utc().subtract(days, 'd').format('YYYY-MM-DD'))
} else if (when === 'last_week' || when === 'before_last_week' || when === 'last_month' || when === 'before_last_month') {
let sub_days = 1
let days = 7
if (when === 'last_week') {
sub_days = 8 // one more since we don't have data for the current, not finished day
days = 7
} else if (when === 'before_last_week') {
sub_days = 15
days = 7
} else if (when === 'last_month') {
sub_days = 31
days = 30
} else if (when === 'before_last_month') {
sub_days = 61
days = 30
}
data = x.data.filter(y => y.timestamp >= moment.utc().subtract(sub_days, 'd').toDate() && y.timestamp <= moment.utc().subtract(sub_days - days, 'd').toDate())
}
return data
}
}
export const get_sum_from_data = async (when: string | 'today' | 'last_day' | 'before_last_day' | 'before_last_week' | 'last_week' | 'before_last_month' | 'last_month', app?: string, account?: string, data_type?: string) => {
let data = await get_data_by_date(when, app, account, data_type)
data = data.map(x => x.value)
let amount = 0
if (data.length > 0) {
amount = data.reduce((x, y) => x + y)
}
return parseFloat(amount.toFixed(3))
}
export const get_sum_from_data_dau = async (when: string | 'today' | 'last_day' | 'before_last_day' | 'before_last_week' | 'last_week' | 'before_last_month' | 'last_month', app?: string, account?: string, data_type?: string) => {
let data = await get_data_by_date(when, app, account, data_type)
let users = Array.from(new Set(create_arr_of_arrays(data.map(x => x.users))))
_g.temp_global_data.dau[when] = _g.temp_global_data.dau[when].concat(users)
_g.temp_global_data.dau[when] = Array.from(new Set(_g.temp_global_data.dau[when]))
return users.length
}
export let create_or_add_both = async (app: string, data_type: string, data: Array<any>, account?: string) => {
await create_or_add(true, app, data_type, data, account)
await create_or_add(false, app, data_type, data, account)
}
export let create_or_add = async (external: boolean, app: string, data_type: string, data: Array<any>, account?: string) => {
let data_obj = external ? await find_extern(app, account, data_type) : await find_intern(app, account, data_type)
if (data_obj.length > 0) {
// Check that data is not yet existing
data = data.filter(x => data_obj[0].data.filter(y => {
return moment.utc(y.timestamp).isSame(x.timestamp) //moment.utc(y.timestamp).format('YYYY-MM-DD') === moment.utc(x.timestamp).format('YYYY-MM-DD')
}).length <= 0)
data_obj[0].data = data_obj[0].data.concat(data)
data_obj[0].markModified('data')
await data_obj[0].save()
return data_obj[0]
} else {
return external ? await Model.DataLean.create({ app, account, data_type, data }) : await Model.DataDetailed.create({ app, account, data_type, data })
}
}
export let create_total_dau = async (app: string) => {
let data = []
// Get all data from accounts and group based on date
for(let data_type of [_g.data_type.dau_custom, _g.data_type.dau_meta, _g.data_type.dau_transfers, _g.data_type.dau_benefactor, _g.data_type.dau_vote_general]) {
let data_db = await find_intern(app, '', data_type)
for (let d of data_db) {
data = data.concat(d.data)
}
}
// Group data
let data_for_db = create_grouped_data_users(data)
// Create/Add as dau_total in DB
await create_or_add_both(app, _g.data_type.dau_total, data_for_db)
}
export let create_grouped_data_users = (data) => {
let grouped_data = _(data).groupBy(x => moment.utc(x.timestamp).toISOString()).value()
let data_for_db = []
for(let timestamp in grouped_data) {
let users = []
for (let g of grouped_data[timestamp]) {
users = users.concat(g.users)
}
// Remove duplicates created by other data_types
users = Array.from(new Set(users))
data_for_db.push({
users, value: users.length, timestamp
})
}
return data_for_db
}
// Experimental for next Update
export let get_last_month_array = async (app:string, arr:Array<{ keys:any, values: any }>) => {
for(let a of arr) {
for(let account of a.keys) {
for(let data_type of a.values) {
let x = await find_intern(app, account, data_type)
let data = x.data.filter(y => y.timestamp >= moment.utc().subtract(30, 'd').toDate())
}
}
}
}
\ No newline at end of file