diff --git a/package.json b/package.json index 58a035d409e62e28adf55330b46425ae4cedea18..49f20a66b1c5bdbded8f8043c30918f45b04e93e 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "path": [ "./dist/bundle" ], - "limit": "140 kB", + "limit": "155 kB", "brotli": false } ], diff --git a/src/chain-observers/classifiers/content-classifier.ts b/src/chain-observers/classifiers/content-classifier.ts new file mode 100644 index 0000000000000000000000000000000000000000..d86d19cbf1bbc3999ef64c933e5d645f449afe02 --- /dev/null +++ b/src/chain-observers/classifiers/content-classifier.ts @@ -0,0 +1,148 @@ +import type { asset, beneficiary_route_type, TAccountName } from "@hiveio/wax"; +import Long from "long"; +import { CollectorClassifierBase } from "./collector-classifier-base"; + +export interface IHiveContent { + /** + * The main topic or community tag for this content. + * @example "hive-106130" + */ + category: string; + + /** + * The author of the content, usually a Hive account name. + * @example "sagarkothari88" + */ + author: TAccountName; + + /** + * The unique identifier for this content, often called a "permlink." + * @example "mentorship-onboarding-and-how-to-create-an-ezabay-account-for-easily-conversion" + */ + permlink: string; + + /** + * The title of the content. This is usually present for posts, but might be empty for comments. + * @example "Mentorship, onboarding and how to create an Ezabay account for easily conversion." + */ + title?: string; + + /** + * The date and time when the content was originally created. + */ + created: Date; + + /** + * The date and time when the content was last edited. + */ + lastUpdated: Date; + + /** + * If this content is a reply, this is the author of the content it replies to. + * Will be empty if it's a top-level post. + * @example "" + */ + parentAuthor?: string; + + /** + * If this content is a reply, this is the unique identifier (permlink) of the content it replies to. + * Will be the category if it's a reply to the main category feed. + * @example "hive-106130" + */ + parentPermlink?: string; + + /** + * The number of direct replies this content has received. + * @example 3 + */ + replyCount: number; + + /** + * Indicates how "deep" this content is in a conversation thread. + * 0 means it's a top-level post. 1 means it's a direct reply to a post, 2 is a reply to a reply, and so on. + * @example 0 + */ + depth: number; + + /** + * This is a crucial number representing the total "Reward Shares" accumulated by this content from upvotes. + * Think of it as the post's "vote weight" or "reward influence." The higher this number, + * the greater its potential share of the Hive reward pool. It's not a direct dollar amount + * but an internal blockchain measure. The final payout is calculated based on this value + * relative to other content paying out at the same time. + * @example 150255792948762 + */ + netRshares: Long; + + /** + * The total number of individual upvotes this content has received. + * While `netRshares` is more directly tied to rewards (as some votes carry more weight than others), + * this shows the raw count of votes. + * @example 245 + */ + netVotes: number; + + /** + * The date and time when the rewards for this content are scheduled to be distributed. + * This is typically 7 days after the content was created. + * Set to "1970-01-01T00:00:00Z" if the content is still pending. + * @example "2025-05-28T23:47:06" + */ + payoutTime: Date; + + /** + * The total value (usually in Hive Backed Dollars - HBD) that was actually paid out for this content. + * This will show "0" or be empty if the `payoutTime` is still in the future. + * The `amount` is the numerical value, `nai` is the asset identifier, and `precision` is the number of decimal places. + * @example { "amount": "0", "nai": "@@000000013", "precision": 3 } + */ + totalPayoutValue: asset; + + /** + * The portion of the `totalPayoutValue` that was distributed to "curators" (people who upvoted the content). + * This will also show "0" or be empty if the `payoutTime` is in the future. + * @example { "amount": "0", "nai": "@@000000013", "precision": 3 } + */ + curatorPayoutValue: asset; + + /** + * A list of other Hive accounts that will receive a percentage of this content's rewards. + * `account` is the username of the beneficiary. + * `weight` represents their share (e.g., 100 means 1%, 1000 means 10%). + * @example [{ account: "sagarkothari88", weight: 100 }, { account: "spk.beneficiary", weight: 1000 }] + */ + beneficiaries: Array; + + /** + * Indicates if other users are allowed to reply to this content. + * @example true + */ + allowsReplies: boolean; + + /** + * Indicates if other users are allowed to vote on this content. + * @example true + */ + allowsVotes: boolean; + + /** + * Indicates if users who upvote this content are eligible to earn curation rewards. + * @example true + */ + allowsCurationRewards: boolean; + + /** + * The amount of rewards, in Hive Power, that the author received after the payout. + * This will be 0 if the post hasn't paid out yet. + * @example 0 + */ + authorRewards: Long; +} + +export type TContentAuthorData = Record; + +export interface IContentData { + contentData: Record; +} + +export class ContentClassifier extends CollectorClassifierBase {} diff --git a/src/chain-observers/classifiers/index.ts b/src/chain-observers/classifiers/index.ts index 21256015657d3087f997eb827232f27c0a698598..3ba69c5dff8afb6a0f09cb47622820e4278608c9 100644 --- a/src/chain-observers/classifiers/index.ts +++ b/src/chain-observers/classifiers/index.ts @@ -10,3 +10,4 @@ export { WitnessClassifier } from "./witness-classifier"; export { ChangeRecoveryInProgressClassifier } from "./change-recovery-in-progress-classifier"; export { DeclineVotingRightsClassifier } from "./decline-voting-rights-classifier"; export { ManabarClassifier } from "./manabar-classifier"; +export { ContentClassifier } from "./content-classifier"; diff --git a/src/chain-observers/collectors/jsonrpc/account-collector.ts b/src/chain-observers/collectors/jsonrpc/account-collector.ts index 88bca69cb083859fa4c8fe840dc8403ea049705f..82af4775d20ef41ed60dc7ebbc9a9dc9b8eea996 100644 --- a/src/chain-observers/collectors/jsonrpc/account-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/account-collector.ts @@ -8,7 +8,7 @@ export interface IAccountCollectorOptions { account: string; } -const MAX_ACCOUNT_GET_LIMIT = 100; +const MAX_ACCOUNT_GET_LIMIT = 1000; export class AccountCollector extends CollectorBase { private readonly accounts: Record = {}; diff --git a/src/chain-observers/collectors/jsonrpc/block-collector.ts b/src/chain-observers/collectors/jsonrpc/block-collector.ts index 060ed9eec0e87f9cfba58862647f959fc43827fb..16431f9f565599d61f6d5c68fe638dce68c55302 100644 --- a/src/chain-observers/collectors/jsonrpc/block-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/block-collector.ts @@ -1,7 +1,7 @@ import { transaction } from "@hiveio/wax"; -import { ITransactionData } from "src/chain-observers/classifiers/block-classifier"; import { WorkerBeeError } from "../../../errors"; import { DynamicGlobalPropertiesClassifier, BlockClassifier } from "../../classifiers"; +import { ITransactionData } from "../../classifiers/block-classifier"; import { TRegisterEvaluationContext } from "../../classifiers/collector-classifier-base"; import { DataEvaluationContext } from "../../factories/data-evaluation-context"; import { CollectorBase, TAvailableClassifiers } from "../collector-base"; diff --git a/src/chain-observers/collectors/jsonrpc/change-recovery-in-progress-collector.ts b/src/chain-observers/collectors/jsonrpc/change-recovery-in-progress-collector.ts index 93874822fd732e31db704571b4b07602a38f5198..e0d42743186c175ef134a5f04e489c5b8dd08953 100644 --- a/src/chain-observers/collectors/jsonrpc/change-recovery-in-progress-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/change-recovery-in-progress-collector.ts @@ -7,7 +7,7 @@ export interface IChangeRecoveryCollectorOptions { changeRecoveryAccount: string; } -const MAX_CHANGE_RECOVERY_GET_LIMIT = 100; +const MAX_CHANGE_RECOVERY_GET_LIMIT = 1000; export class ChangeRecoveryInProgressCollector extends CollectorBase { private readonly changeRecoveryAccounts: Record = {}; diff --git a/src/chain-observers/collectors/jsonrpc/content-collector.ts b/src/chain-observers/collectors/jsonrpc/content-collector.ts new file mode 100644 index 0000000000000000000000000000000000000000..255a02bde32f0159b8647c1e746fb661288036be --- /dev/null +++ b/src/chain-observers/collectors/jsonrpc/content-collector.ts @@ -0,0 +1,99 @@ +import { TAccountName } from "@hiveio/wax"; +import Long from "long"; +import { OrderedQueue } from "../../../types/queue"; +import { ContentClassifier, IHiveContent } from "../../classifiers/content-classifier"; +import { DataEvaluationContext } from "../../factories/data-evaluation-context"; +import { CollectorBase, TAvailableClassifiers } from "../collector-base"; + +export const BUCKET_INTERVAL = 3 * 1000; // 3 seconds (Hive block interval) + +export interface IContentCollectorOptions { + account: string; + permlink: string; + rollbackContractAfter: Date; +} + +export type TContentContractData = [TAccountName, string]; + +const MAX_CONTENT_GET_LIMIT = 1000; + +export class ContentCollector extends CollectorBase { + private readonly contractTimestamps = new OrderedQueue(); + private readonly contentCached = new Set(); + + protected pushOptions(data: IContentCollectorOptions): void { + const permlink = `${data.account}/${data.permlink}`; + + const rollbackAfter = Math.floor(data.rollbackContractAfter.getTime()); + const bucketInterval = rollbackAfter + (BUCKET_INTERVAL - (rollbackAfter % BUCKET_INTERVAL)); + + if (this.contentCached.has(permlink)) + // If we already have this content, we don't need to enqueue it again + return; + + this.contractTimestamps.enqueue(bucketInterval, [data.account, data.permlink]); + this.contentCached.add(permlink); + } + + // Data is automatically managed and popped from the queue + protected popOptions(_data: IContentCollectorOptions): void {} + + public async fetchData(data: DataEvaluationContext) { + const contentData: Record> = {}; + + const allData: Array<[TAccountName, string]> = []; + + const currentTime = Date.now(); + for(const { value } of this.contractTimestamps.dequeueUntil(currentTime)) { + for(const apiData of value) + allData.push(apiData); + + // Cleanup + this.contentCached.delete(`${value[0]}/${value[1]}`); + } + + for (let i = 0; i < allData.length; i += MAX_CONTENT_GET_LIMIT) { + const chunk = allData.slice(i, i + MAX_CONTENT_GET_LIMIT); + + const startFindComments = Date.now(); + const { comments } = await this.worker.chain!.api.database_api.find_comments({ comments: chunk }); + data.addTiming("database_api.find_comments", Date.now() - startFindComments); + + const startCommentsAnalysis = Date.now(); + + for(const comment of comments) { + contentData[comment.author] = contentData[comment.author] || {}; + contentData[comment.author][comment.permlink] = { + author: comment.author, + permlink: comment.permlink, + parentAuthor: comment.parent_author, + parentPermlink: comment.parent_permlink, + allowsCurationRewards: comment.allow_curation_rewards, + allowsReplies: comment.allow_replies, + allowsVotes: comment.allow_votes, + authorRewards: Long.fromValue(comment.author_rewards), + beneficiaries: comment.beneficiaries, + category: comment.category, + created: new Date(`${comment.created}Z`), + curatorPayoutValue: comment.curator_payout_value, + depth: comment.depth, + lastUpdated: new Date(`${comment.last_update}Z`), + netRshares: Long.fromValue(comment.net_rshares), + netVotes: comment.net_votes, + payoutTime: new Date(`${comment.last_payout}Z`), + replyCount: comment.children, + totalPayoutValue: comment.total_payout_value, + title: comment.title + }; + } + + data.addTiming("commentsAnalysis", Date.now() - startCommentsAnalysis); + } + + return { + [ContentClassifier.name]: { + contentData + } as TAvailableClassifiers["ContentClassifier"], + } satisfies Partial; + }; +} diff --git a/src/chain-observers/collectors/jsonrpc/decline-voting-rights-collector.ts b/src/chain-observers/collectors/jsonrpc/decline-voting-rights-collector.ts index 786d9faf26623a438048d48e0f820d5e7c66170e..ad2d399b1c36c7aa126ebc74a22855ab20a34213 100644 --- a/src/chain-observers/collectors/jsonrpc/decline-voting-rights-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/decline-voting-rights-collector.ts @@ -7,7 +7,7 @@ export interface IDeclineVotingRightsCollectorOptions { declineVotingRightsAccount: string; } -const MAX_DECLINED_VOTING_RIGHTS_GET_LIMIT = 100; +const MAX_DECLINED_VOTING_RIGHTS_GET_LIMIT = 1000; export class DeclineVotingRightsCollector extends CollectorBase { private readonly declineVotingRightsAccounts: Record = {}; diff --git a/src/chain-observers/collectors/jsonrpc/rc-account-collector.ts b/src/chain-observers/collectors/jsonrpc/rc-account-collector.ts index db4e884825b47a74361f7545b216c17091b3de68..86ec7b7aa4e0ad2f6e189a418500db4c4db5b55f 100644 --- a/src/chain-observers/collectors/jsonrpc/rc-account-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/rc-account-collector.ts @@ -8,7 +8,7 @@ export interface IRcAccountCollectorOptions { rcAccount: string; } -const MAX_RC_ACCOUNT_GET_LIMIT = 100; +const MAX_RC_ACCOUNT_GET_LIMIT = 1000; export class RcAccountCollector extends CollectorBase { private readonly rcAccounts: Record = {}; diff --git a/src/chain-observers/collectors/jsonrpc/witness-collector.ts b/src/chain-observers/collectors/jsonrpc/witness-collector.ts index 759355e8546de9c547c93253c939871e4b2fecb9..f113041935c351d33295a51f2d8a0d63313f240f 100644 --- a/src/chain-observers/collectors/jsonrpc/witness-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/witness-collector.ts @@ -7,7 +7,7 @@ export interface IWitnessCollectorOptions { witness: string; } -const MAX_WITNESS_GET_LIMIT = 100; +const MAX_WITNESS_GET_LIMIT = 1000; export class WitnessCollector extends CollectorBase { private readonly witnesses: Record = {}; diff --git a/src/chain-observers/factories/data-evaluation-context.ts b/src/chain-observers/factories/data-evaluation-context.ts index 621b68ab9f72930e115583702f3b9a3ff496920a..0288d694ba880c32d39840bea40fafe2eed01789 100644 --- a/src/chain-observers/factories/data-evaluation-context.ts +++ b/src/chain-observers/factories/data-evaluation-context.ts @@ -31,6 +31,32 @@ export class DataEvaluationContext { this.collectors[evaluationContext.name] = collector; } + /** + * Available for dynamically pushing options from the classifier. + */ + public pushClassifierOptions(classifier: T, options: Record) { + const collector = this.collectors[classifier.name]; + if (!collector) + throw new WorkerBeeError(`Could not find collector for classifier: "${classifier.name}" when pushing options`); + + collector.pushOptions(options); + } + + /** + * Available for dynamically popping options from the classifier. + */ + public popClassifierOptions(classifier: T, options: Record) { + const collector = this.collectors[classifier.name]; + if (!collector) + throw new WorkerBeeError(`Could not find collector for classifier: "${classifier.name}" when popping options`); + + collector.popOptions(options); + } + + public hasClassifierRegistered(classifier: T): boolean { + return this.collectors[classifier.name] !== undefined; + } + public async get infer R ? R : never>( evaluationContext: T ): Promise { diff --git a/src/chain-observers/factories/historydata/factory-data.ts b/src/chain-observers/factories/historydata/factory-data.ts index 4c7e5f648388ad79e92433651783b284d83b9971..37aac1969eff0a3eb8806bf9660b4d92272c34a4 100644 --- a/src/chain-observers/factories/historydata/factory-data.ts +++ b/src/chain-observers/factories/historydata/factory-data.ts @@ -1,6 +1,7 @@ import type { WorkerBee } from "../../../bot"; import { BlockClassifier, BlockHeaderClassifier, + ContentClassifier, DynamicGlobalPropertiesClassifier, ImpactedAccountClassifier, OperationClassifier } from "../../classifiers"; import { IEvaluationContextClass } from "../../classifiers/collector-classifier-base"; @@ -9,6 +10,7 @@ import { ImpactedAccountCollector } from "../../collectors/common/impacted-accou import { OperationCollector } from "../../collectors/common/operation-collector"; import { BlockCollector } from "../../collectors/historydata/block-collector"; import { DynamicGlobalPropertiesCollector } from "../../collectors/historydata/dynamic-global-properties-collector"; +import { ContentCollector } from "../../collectors/jsonrpc/content-collector"; export const HistoryDataFactoryData = (worker: WorkerBee, fromBlock: number, toBlock?: number): Array<[IEvaluationContextClass, CollectorBase]> => { const blockClassifier = new BlockCollector(worker, fromBlock, toBlock); @@ -19,5 +21,6 @@ export const HistoryDataFactoryData = (worker: WorkerBee, fromBlock: number, toB [BlockClassifier, blockClassifier], [ImpactedAccountClassifier, new ImpactedAccountCollector(worker)], [OperationClassifier, new OperationCollector(worker)], + [ContentClassifier, new ContentCollector(worker)] ]; }; diff --git a/src/chain-observers/factories/jsonrpc/factory-data.ts b/src/chain-observers/factories/jsonrpc/factory-data.ts index b68cc92e7844067763fd3a1ffff098bc967c87a3..1b52425780fcfc2cd0883ff62e7e7ba02768fef5 100644 --- a/src/chain-observers/factories/jsonrpc/factory-data.ts +++ b/src/chain-observers/factories/jsonrpc/factory-data.ts @@ -2,6 +2,7 @@ import type { WorkerBee } from "../../../bot"; import { AccountClassifier, BlockClassifier, BlockHeaderClassifier, ChangeRecoveryInProgressClassifier, + ContentClassifier, DeclineVotingRightsClassifier, DynamicGlobalPropertiesClassifier, FeedPriceClassifier, ImpactedAccountClassifier, ManabarClassifier, OperationClassifier, RcAccountClassifier, WitnessClassifier @@ -15,6 +16,7 @@ import { OperationCollector } from "../../collectors/common/operation-collector" import { AccountCollector } from "../../collectors/jsonrpc/account-collector"; import { BlockCollector } from "../../collectors/jsonrpc/block-collector"; import { ChangeRecoveryInProgressCollector } from "../../collectors/jsonrpc/change-recovery-in-progress-collector"; +import { ContentCollector } from "../../collectors/jsonrpc/content-collector"; import { DeclineVotingRightsCollector } from "../../collectors/jsonrpc/decline-voting-rights-collector"; import { DynamicGlobalPropertiesCollector } from "../../collectors/jsonrpc/dynamic-global-properties-collector"; import { FeedPriceCollector } from "../../collectors/jsonrpc/feed-price-collector"; @@ -34,4 +36,5 @@ export const JsonRpcFactoryData: (worker: WorkerBee) => Array<[IEvaluationContex [ChangeRecoveryInProgressClassifier, new ChangeRecoveryInProgressCollector(worker)], [DeclineVotingRightsClassifier, new DeclineVotingRightsCollector(worker)], [ManabarClassifier, new ManabarCollector(worker)], + [ContentClassifier, new ContentCollector(worker)] ]); diff --git a/src/chain-observers/filters/content-metadata-filter.ts b/src/chain-observers/filters/content-metadata-filter.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c08c6d4428e2610ce19e57245a07a98d3094049 --- /dev/null +++ b/src/chain-observers/filters/content-metadata-filter.ts @@ -0,0 +1,82 @@ +import { TAccountName } from "@hiveio/wax"; +import type { WorkerBee } from "../../bot"; +import { ContentClassifier } from "../classifiers"; +import type { TRegisterEvaluationContext } from "../classifiers/collector-classifier-base"; +import type { DataEvaluationContext } from "../factories/data-evaluation-context"; +import { FilterBase } from "./filter-base"; + +export interface ICommentData { + parentAuthor: TAccountName; + parentPermlink: string; +} + +// Base class for content filters (posts and comments) +export abstract class ContentMetadataFilter extends FilterBase { + public constructor( + worker: WorkerBee, + accounts: string[], + protected readonly isPost: boolean, + protected readonly parentCommentFilter?: ICommentData + ) { + super(worker); + + this.accounts = new Set(accounts); + } + + protected readonly accounts: Set; + + public usedContexts(): Array { + return [ + ContentClassifier + ]; + } + + public async match(data: DataEvaluationContext): Promise { + const { contentData } = await data.get(ContentClassifier); + + for(const account in contentData) { + if (!this.accounts.has(account)) + continue; + const content = contentData[account]; + for(const permlink in content) { + const operation = content[permlink]; + // Check if post/comment type matches what we're looking for + const postIndicator = operation.parentAuthor && operation.parentAuthor === ""; + if (this.isPost !== postIndicator) + continue; + + // Check parent data if specified + if (!this.isPost && this.parentCommentFilter && ( + operation.parentPermlink !== this.parentCommentFilter.parentPermlink + || operation.parentAuthor !== this.parentCommentFilter.parentAuthor + )) + continue; + + return true; + } + } + + return false; + } +} + +// Filter for comments (replies to posts or other comments) +export class CommentContentMetadataFilter extends ContentMetadataFilter { + public constructor( + worker: WorkerBee, + accounts: string[], + parentCommentFilter?: ICommentData + ) { + super(worker, accounts, false, parentCommentFilter); + } +} + +// Filter for posts (top-level content with empty parent_author) +export class PostContentMetadataFilter extends ContentMetadataFilter { + public constructor( + worker: WorkerBee, + accounts: string[] + ) { + super(worker, accounts, true); + } +} diff --git a/src/chain-observers/index.ts b/src/chain-observers/index.ts index 507b4312caab1c6d781e782f5d5400500a96456b..a7dfeaae23933b165abc883969723a4a0deab703 100644 --- a/src/chain-observers/index.ts +++ b/src/chain-observers/index.ts @@ -10,6 +10,7 @@ export type { IImpactedAccount, IImpactedAccountData } from "./classifiers/impac export type { IOperationBaseData, IOperationData, IOperationTransactionPair } from "./classifiers/operation-classifier"; export type { IRcAccount, IRcAccountData } from "./classifiers/rc-account-classifier"; export type { IWitness, IWitnessData } from "./classifiers/witness-classifier"; +export type { IHiveContent } from "./classifiers/content-classifier"; export type { IAccountProviderData, TAccountProvided } from "./providers/account-provider"; export { EAlarmType, type IAlarmAccountsData, type TAlarmAccounts } from "./providers/alarm-provider"; @@ -34,5 +35,6 @@ export type { IRcAccountProviderData, TRcAccountProvided } from "./providers/rc- export type { ITransactionProviderData, TTransactionProvider } from "./providers/transaction-provider"; export type { IWhaleAlertMetadata, IWhaleAlertProviderData } from "./providers/whale-alert-provider"; export type { IWitnessProviderData, TWitnessProvider } from "./providers/witness-provider"; +export type { IBlogContentMetadataroviderData, TBlogContentMetadataProvided } from "./providers/content-metadata-provider"; export { Exchange } from "../utils/known-exchanges"; diff --git a/src/chain-observers/providers/blog-content-provider.ts b/src/chain-observers/providers/blog-content-provider.ts index 704a46389803d33743fe78d376ee2784225192bd..b87e4d583a186a67e9791b2e1c845a95f9510e1a 100644 --- a/src/chain-observers/providers/blog-content-provider.ts +++ b/src/chain-observers/providers/blog-content-provider.ts @@ -1,11 +1,15 @@ import { comment, TAccountName } from "@hiveio/wax"; import { WorkerBeeArrayIterable, WorkerBeeIterable } from "../../types/iterator"; +import { calculateRelativeTime } from "../../utils/time"; +import { BlockHeaderClassifier, ContentClassifier } from "../classifiers"; import { TRegisterEvaluationContext } from "../classifiers/collector-classifier-base"; import { IOperationTransactionPair, OperationClassifier } from "../classifiers/operation-classifier"; import { DataEvaluationContext } from "../factories/data-evaluation-context"; import { ICommentData } from "../filters/blog-content-filter"; import { ProviderBase } from "./provider-base"; +const ONE_WEEK_MS = 7 * 24 * 60 * 60 * 1000; // One week in milliseconds + // Common interface for blog content data (posts and comments) export type TBlogContentProvided> = { [K in TAccounts[number]]: Array> | WorkerBeeIterable>; @@ -19,11 +23,13 @@ export interface ICommentProviderAuthors { export interface ICommentProviderOptions { authors: ICommentProviderAuthors[]; + collectPostMetadataAfter?: string; } // Post provider implementation export interface IPostProviderOptions { authors: string[]; + collectPostMetadataAfter?: string; } export interface ICommentProviderData> { @@ -34,12 +40,17 @@ export interface IPostProviderData> { posts: Partial>; } +export interface IAuthorsOptions { + commentData?: ICommentData; + collectPostMetadataAfter?: number; +} + // Base class for blog content providers (posts and comments) export abstract class BlogContentProvider< TAccounts extends Array = Array, TOptions extends object = object > extends ProviderBase { - public readonly authors = new Map(); + public readonly authors = new Map(); protected readonly isPost: boolean; public constructor(isPost: boolean) { @@ -56,6 +67,8 @@ export abstract class BlogContentProvider< public async createProviderData(data: DataEvaluationContext): Promise> { const result = {} as TBlogContentProvided; + const { timestamp } = await data.get(BlockHeaderClassifier); + const accounts = await data.get(OperationClassifier); if (accounts.operationsPerType.comment) for (const operation of accounts.operationsPerType.comment) { @@ -65,15 +78,15 @@ export abstract class BlogContentProvider< continue; /// If requested account is a post author - if(this.authors.has(operation.operation.author) === false) + if(this.authors.get(operation.operation.author)?.commentData === undefined) continue; let account = operation.operation.author; /// If requested account is a comment parent-author - const parentCommentFilter = this.authors.get(operation.operation.author); + const parentCommentFilter = this.authors.get(operation.operation.author)!; - if(!this.isPost && parentCommentFilter && operation.operation.parent_permlink !== parentCommentFilter.parentPermlink) + if(!this.isPost && parentCommentFilter.commentData && operation.operation.parent_permlink !== parentCommentFilter.commentData.parentPermlink) continue; else account = operation.operation.parent_author; @@ -83,6 +96,14 @@ export abstract class BlogContentProvider< const storage = result[account] as WorkerBeeArrayIterable>; + // Dynamically inject options for the content classifier when post is created + if (parentCommentFilter.collectPostMetadataAfter && data.hasClassifierRegistered(ContentClassifier)) + data.pushClassifierOptions(ContentClassifier, { + account, + permlink: operation.operation.permlink, + rollbackContractAfter: new Date(timestamp.getTime() + parentCommentFilter.collectPostMetadataAfter) + }); + storage.push({ operation: operation.operation, transaction: operation.transaction @@ -100,7 +121,12 @@ export class CommentProvider = Array> { @@ -117,7 +143,11 @@ export class PostProvider = Array> { diff --git a/src/chain-observers/providers/content-metadata-provider.ts b/src/chain-observers/providers/content-metadata-provider.ts new file mode 100644 index 0000000000000000000000000000000000000000..53ce8b62bfd736577483d01aba843b074e34d9b7 --- /dev/null +++ b/src/chain-observers/providers/content-metadata-provider.ts @@ -0,0 +1,35 @@ +import { TAccountName } from "@hiveio/wax"; +import { WorkerBeeIterable } from "../../types/iterator"; +import { TRegisterEvaluationContext } from "../classifiers/collector-classifier-base"; +import { ContentClassifier, IHiveContent } from "../classifiers/content-classifier"; +import { DataEvaluationContext } from "../factories/data-evaluation-context"; +import { ProviderBase } from "./provider-base"; + +export type TBlogContentMetadataProvided> = { + [K in TAccounts[number]]: Array | WorkerBeeIterable; +}; + +export interface IBlogContentMetadataroviderData> { + contentMetadataPerAccount: Partial>; +}; + +export class BlogContentMetadataProvider = Array> extends ProviderBase { + public usedContexts(): Array { + return [ContentClassifier]; + } + + public async provide(data: DataEvaluationContext): Promise> { + const result = { + contentMetadataPerAccount: {} + }; + + const content = await data.get(ContentClassifier); + for(const account in content.contentData) { + result.contentMetadataPerAccount[account] = content.contentData[account] ?? []; + for(const post in content.contentData[account]) + result.contentMetadataPerAccount[account].push(content.contentData[account][post]); + } + + return result as IBlogContentMetadataroviderData; + } +} diff --git a/src/past-queen.ts b/src/past-queen.ts index 0427d636f430a347f4182dd6c8a56fa9d982a9b1..8318d570d909a99963fa36a51e5bdd4b182abe2a 100644 --- a/src/past-queen.ts +++ b/src/past-queen.ts @@ -5,10 +5,11 @@ import { QueenBee } from "./queen"; export type TPastQueen = Omit< PastQueen, - "onAccountFullManabar" | "onAccountBalanceChange" | "onAccountMetadataChange" | - "onFeedPriceChange" | "onFeedPriceNoChange" | "provideFeedPriceData" | - "onAlarm" | "onWitnessMissedBlocks" | "provideAccounts" | - "provideWitnesses" | "provideRcAccounts" + "onAccountFullManabar" | "onAccountBalanceChange" | "onAccountMetadataChange" | + "onFeedPriceChange" | "onFeedPriceNoChange" | "provideFeedPriceData" | + "onAlarm" | "onWitnessMissedBlocks" | "provideAccounts" | + "provideWitnesses" | "provideRcAccounts" | "onPostsIncomingPayment" | + "onCommentsIncomingPayment" >; export class PastQueen extends QueenBee { diff --git a/src/queen.ts b/src/queen.ts index 93b0a970fbbfc93fe47e58aeab4cf2da42ab186c..7ce9a2d7adcb56969f644323338097e317603a41 100644 --- a/src/queen.ts +++ b/src/queen.ts @@ -9,6 +9,7 @@ import { BalanceChangeFilter } from "./chain-observers/filters/balance-change-fi import { BlockNumberFilter } from "./chain-observers/filters/block-filter"; import { CommentFilter, PostFilter } from "./chain-observers/filters/blog-content-filter"; import { LogicalAndFilter, LogicalOrFilter } from "./chain-observers/filters/composite-filter"; +import { CommentContentMetadataFilter, PostContentMetadataFilter } from "./chain-observers/filters/content-metadata-filter"; import { CustomOperationFilter } from "./chain-observers/filters/custom-operation-filter"; import { ExchangeTransferFilter } from "./chain-observers/filters/exchange-transfer-filter"; import { FeedPriceChangeFilter } from "./chain-observers/filters/feed-price-change-percent-filter"; @@ -30,6 +31,7 @@ import { AlarmProvider } from "./chain-observers/providers/alarm-provider"; import { BlockHeaderProvider } from "./chain-observers/providers/block-header-provider"; import { BlockProvider } from "./chain-observers/providers/block-provider"; import { CommentProvider, PostProvider } from "./chain-observers/providers/blog-content-provider"; +import { BlogContentMetadataProvider } from "./chain-observers/providers/content-metadata-provider"; import { CustomOperationProvider } from "./chain-observers/providers/custom-operation-provider"; import { ExchangeTransferProvider } from "./chain-observers/providers/exchange-transfer-provider"; import { FeedPriceProvider } from "./chain-observers/providers/feed-price-provider"; @@ -417,6 +419,74 @@ export class QueenBee { return this; } + /** + * Subscribes to notifications when a post is given relative time close to the incoming payment time + * + * Automatically provides the post metadata (including e.g. net rshares) in the `next` callback. + * + * Note: This method implicitly applies the OR operator between the specified accounts. + * + * @example + * ```ts + * workerbee.observe.onPostsIncomingPayment("-6s", "username", "username2").subscribe({ + * next: (data) => { // Will notify 6 seconds before the post receives the payment + * for(const account in data.contentMetadataPerAccount) + * for(const { author, permlink, netVotes } of data.posts[account]) + * console.log(`@${author}/${permlink} received ${netVotes} votes`); + * } + * }); + * ``` + * + * @param authors account names of the authors to monitor for post incoming payout. + * @returns itself + */ + public onPostsIncomingPayment< + TAccounts extends TAccountName[] + >(relativeTime: string, ...authors: TAccounts): QueenBee["provide"]>>> { + this.operands.push(new PostContentMetadataFilter(this.worker, authors)); + + this.pushProvider(PostProvider, { collectPostMetadataAfter: relativeTime, authors }); + + this.pushProvider(BlogContentMetadataProvider); + + return this; + } + + /** + * Subscribes to notifications when a comment is given relative time close to the incoming payment time + * + * Automatically provides the comment metadata (including e.g. net rshares) in the `next` callback. + * + * Note: This method implicitly applies the OR operator between the specified accounts. + * + * @example + * ```ts + * workerbee.observe.onCommentsIncomingPayment("-6s", "username", "username2").subscribe({ + * next: (data) => { // Will notify 6 seconds before the comment receives the payment + * for(const account in data.contentMetadataPerAccount) + * for(const { author, permlink, netVotes } of data.comments[account]) + * console.log(`@${author}/${permlink} received ${netVotes} votes`); + * } + * }); + * ``` + * + * @param authors account names of the authors to monitor for comment incoming payout. + * @returns itself + */ + public onCommentsIncomingPayment< + TAccounts extends TAccountName[] + >(relativeTime: string, ...authors: TAccounts): QueenBee["provide"]>>> { + // TODO: Handle parentPostOrComment?: ICommentData + + this.operands.push(new CommentContentMetadataFilter(this.worker, authors)); + + this.pushProvider(CommentProvider, { collectPostMetadataAfter: relativeTime, authors: authors.map(account => ({ account })) }); + + this.pushProvider(BlogContentMetadataProvider); + + return this; + } + /** * Subscribes to notifications when a new account is created. * diff --git a/src/types/queue.ts b/src/types/queue.ts new file mode 100644 index 0000000000000000000000000000000000000000..24a0d9d2b5c966e440b5b3b31e563b9970e41810 --- /dev/null +++ b/src/types/queue.ts @@ -0,0 +1,56 @@ +import { WorkerBeeError } from "../errors"; + +export interface IOrderedItem> { + key: T; + value: K; +} + +/** + * A sorted queue that allows enqueuing items and dequeuing them in sorted order. + * Items are sorted in ascending order, and the queue can be drained until a specified value. + * If an item is enqueued with a key that is less than the last enqueued key, an error is thrown. + * If an item is enqueued with the same key, it is added to the existing array of values for that key. + * Requires items to be comparable (e.g., numbers, Dates) + */ +export class OrderedQueue { + private buf: IOrderedItem[] = []; + private head = 0; + private tail = 0; + + public enqueue(key: K, value: V): void { + if (this.buf[this.tail] !== undefined) { + if (this.buf[this.tail].key > key) + throw new WorkerBeeError("Items must be enqueued in sorted order."); + + if (this.buf[this.tail].key === key) + this.buf[this.tail].value.push(value); + else + this.buf[this.tail++] = { + key, + value: [value] + }; + } else + this.buf[this.tail++] = { + key, + value: [value] + }; + + } + + public *dequeueUntil(maxValue: K): Generator> { + while (this.head < this.tail && this.buf[this.head] !== undefined && this.buf[this.head].key <= maxValue) { + const item = this.buf[this.head]; + this.buf[this.head++] = undefined as any; + if (this.head === this.tail) { + this.head = this.tail = 0; + this.buf.length = 0; + } + + yield item; + } + } + + public get size(): number { + return this.tail - this.head; + } +} diff --git a/src/wax/index.ts b/src/wax/index.ts index 795b6a731dd391a933f138899d1449b7ed85e7c0..a659655f0946d7b56d5f69f9b06fa99c4e5ac1d9 100644 --- a/src/wax/index.ts +++ b/src/wax/index.ts @@ -1,4 +1,4 @@ -import { createHiveChain, IHiveChainInterface, IWaxOptionsChain, price, TWaxExtended } from "@hiveio/wax"; +import { asset, beneficiary_route_type, createHiveChain, IHiveChainInterface, IWaxOptionsChain, price, TAccountName, TWaxExtended } from "@hiveio/wax"; export type WaxExtendTypes = { database_api: { @@ -22,6 +22,45 @@ export type WaxExtendTypes = { // ... }>; }; }; + find_comments: { + params: { comments: Array<[TAccountName, string]> }; + result: { comments: Array<{ + abs_rshares: 150255792948762, + allow_curation_rewards: boolean; + allow_replies: boolean; + allow_votes: boolean; + author: TAccountName; + author_rewards: number; + beneficiaries: Array; + body: string; + cashout_time: string; + category: string + children: number; + children_abs_rshares: number; + created: string; + curator_payout_value: asset; + depth: number; + id: number; + json_metadata: string; + last_payout: string; + last_update: string; + max_accepted_payout: asset; + max_cashout_time: string; + net_rshares: number | string; + net_votes: number; + parent_author: string + parent_permlink: string; + percent_hbd: number; + permlink: string; + reward_weight: string; + root_author: TAccountName; + root_permlink: string; + title: string; + total_payout_value: asset; + total_vote_weight: number | string; + vote_rshares: number | string; + }>; }; + }; find_decline_voting_rights_requests: { params: { accounts: string[]; }; result: { requests: Array<{