From 779705c370c8d9e3e9f5f0d22dabdbcfa1a466a4 Mon Sep 17 00:00:00 2001 From: mtyszczak Date: Thu, 22 May 2025 11:18:57 +0200 Subject: [PATCH 1/3] Extend max get limit to 1000 --- src/chain-observers/collectors/jsonrpc/account-collector.ts | 2 +- .../collectors/jsonrpc/change-recovery-in-progress-collector.ts | 2 +- .../collectors/jsonrpc/decline-voting-rights-collector.ts | 2 +- src/chain-observers/collectors/jsonrpc/rc-account-collector.ts | 2 +- src/chain-observers/collectors/jsonrpc/witness-collector.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/chain-observers/collectors/jsonrpc/account-collector.ts b/src/chain-observers/collectors/jsonrpc/account-collector.ts index 88bca69..82af477 100644 --- a/src/chain-observers/collectors/jsonrpc/account-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/account-collector.ts @@ -8,7 +8,7 @@ export interface IAccountCollectorOptions { account: string; } -const MAX_ACCOUNT_GET_LIMIT = 100; +const MAX_ACCOUNT_GET_LIMIT = 1000; export class AccountCollector extends CollectorBase { private readonly accounts: Record = {}; diff --git a/src/chain-observers/collectors/jsonrpc/change-recovery-in-progress-collector.ts b/src/chain-observers/collectors/jsonrpc/change-recovery-in-progress-collector.ts index 9387482..e0d4274 100644 --- a/src/chain-observers/collectors/jsonrpc/change-recovery-in-progress-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/change-recovery-in-progress-collector.ts @@ -7,7 +7,7 @@ export interface IChangeRecoveryCollectorOptions { changeRecoveryAccount: string; } -const MAX_CHANGE_RECOVERY_GET_LIMIT = 100; +const MAX_CHANGE_RECOVERY_GET_LIMIT = 1000; export class ChangeRecoveryInProgressCollector extends CollectorBase { private readonly changeRecoveryAccounts: Record = {}; diff --git a/src/chain-observers/collectors/jsonrpc/decline-voting-rights-collector.ts b/src/chain-observers/collectors/jsonrpc/decline-voting-rights-collector.ts index 786d9fa..ad2d399 100644 --- a/src/chain-observers/collectors/jsonrpc/decline-voting-rights-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/decline-voting-rights-collector.ts @@ -7,7 +7,7 @@ export interface IDeclineVotingRightsCollectorOptions { declineVotingRightsAccount: string; } -const MAX_DECLINED_VOTING_RIGHTS_GET_LIMIT = 100; +const MAX_DECLINED_VOTING_RIGHTS_GET_LIMIT = 1000; export class DeclineVotingRightsCollector extends CollectorBase { private readonly declineVotingRightsAccounts: Record = {}; diff --git a/src/chain-observers/collectors/jsonrpc/rc-account-collector.ts b/src/chain-observers/collectors/jsonrpc/rc-account-collector.ts index db4e884..86ec7b7 100644 --- a/src/chain-observers/collectors/jsonrpc/rc-account-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/rc-account-collector.ts @@ -8,7 +8,7 @@ export interface IRcAccountCollectorOptions { rcAccount: string; } -const MAX_RC_ACCOUNT_GET_LIMIT = 100; +const MAX_RC_ACCOUNT_GET_LIMIT = 1000; export class RcAccountCollector extends CollectorBase { private readonly rcAccounts: Record = {}; diff --git a/src/chain-observers/collectors/jsonrpc/witness-collector.ts b/src/chain-observers/collectors/jsonrpc/witness-collector.ts index 759355e..f113041 100644 --- a/src/chain-observers/collectors/jsonrpc/witness-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/witness-collector.ts @@ -7,7 +7,7 @@ export interface IWitnessCollectorOptions { witness: string; } -const MAX_WITNESS_GET_LIMIT = 100; +const MAX_WITNESS_GET_LIMIT = 1000; export class WitnessCollector extends CollectorBase { private readonly witnesses: Record = {}; -- GitLab From 82110428e1ac97f144f30d211b60645933b557ae Mon Sep 17 00:00:00 2001 From: mtyszczak Date: Thu, 22 May 2025 11:21:22 +0200 Subject: [PATCH 2/3] Implement contractable content collector --- .../classifiers/content-classifier.ts | 148 ++++++++++++++++++ src/chain-observers/classifiers/index.ts | 1 + .../collectors/jsonrpc/block-collector.ts | 2 +- .../collectors/jsonrpc/content-collector.ts | 99 ++++++++++++ .../factories/data-evaluation-context.ts | 26 +++ .../factories/historydata/factory-data.ts | 3 + .../factories/jsonrpc/factory-data.ts | 3 + .../filters/content-metadata-filter.ts | 82 ++++++++++ src/chain-observers/index.ts | 2 + .../providers/blog-content-provider.ts | 42 ++++- .../providers/content-metadata-provider.ts | 35 +++++ src/past-queen.ts | 9 +- src/queen.ts | 70 +++++++++ src/types/queue.ts | 56 +++++++ src/wax/index.ts | 41 ++++- 15 files changed, 607 insertions(+), 12 deletions(-) create mode 100644 src/chain-observers/classifiers/content-classifier.ts create mode 100644 src/chain-observers/collectors/jsonrpc/content-collector.ts create mode 100644 src/chain-observers/filters/content-metadata-filter.ts create mode 100644 src/chain-observers/providers/content-metadata-provider.ts create mode 100644 src/types/queue.ts diff --git a/src/chain-observers/classifiers/content-classifier.ts b/src/chain-observers/classifiers/content-classifier.ts new file mode 100644 index 0000000..d86d19c --- /dev/null +++ b/src/chain-observers/classifiers/content-classifier.ts @@ -0,0 +1,148 @@ +import type { asset, beneficiary_route_type, TAccountName } from "@hiveio/wax"; +import Long from "long"; +import { CollectorClassifierBase } from "./collector-classifier-base"; + +export interface IHiveContent { + /** + * The main topic or community tag for this content. + * @example "hive-106130" + */ + category: string; + + /** + * The author of the content, usually a Hive account name. + * @example "sagarkothari88" + */ + author: TAccountName; + + /** + * The unique identifier for this content, often called a "permlink." + * @example "mentorship-onboarding-and-how-to-create-an-ezabay-account-for-easily-conversion" + */ + permlink: string; + + /** + * The title of the content. This is usually present for posts, but might be empty for comments. + * @example "Mentorship, onboarding and how to create an Ezabay account for easily conversion." + */ + title?: string; + + /** + * The date and time when the content was originally created. + */ + created: Date; + + /** + * The date and time when the content was last edited. + */ + lastUpdated: Date; + + /** + * If this content is a reply, this is the author of the content it replies to. + * Will be empty if it's a top-level post. + * @example "" + */ + parentAuthor?: string; + + /** + * If this content is a reply, this is the unique identifier (permlink) of the content it replies to. + * Will be the category if it's a reply to the main category feed. + * @example "hive-106130" + */ + parentPermlink?: string; + + /** + * The number of direct replies this content has received. + * @example 3 + */ + replyCount: number; + + /** + * Indicates how "deep" this content is in a conversation thread. + * 0 means it's a top-level post. 1 means it's a direct reply to a post, 2 is a reply to a reply, and so on. + * @example 0 + */ + depth: number; + + /** + * This is a crucial number representing the total "Reward Shares" accumulated by this content from upvotes. + * Think of it as the post's "vote weight" or "reward influence." The higher this number, + * the greater its potential share of the Hive reward pool. It's not a direct dollar amount + * but an internal blockchain measure. The final payout is calculated based on this value + * relative to other content paying out at the same time. + * @example 150255792948762 + */ + netRshares: Long; + + /** + * The total number of individual upvotes this content has received. + * While `netRshares` is more directly tied to rewards (as some votes carry more weight than others), + * this shows the raw count of votes. + * @example 245 + */ + netVotes: number; + + /** + * The date and time when the rewards for this content are scheduled to be distributed. + * This is typically 7 days after the content was created. + * Set to "1970-01-01T00:00:00Z" if the content is still pending. + * @example "2025-05-28T23:47:06" + */ + payoutTime: Date; + + /** + * The total value (usually in Hive Backed Dollars - HBD) that was actually paid out for this content. + * This will show "0" or be empty if the `payoutTime` is still in the future. + * The `amount` is the numerical value, `nai` is the asset identifier, and `precision` is the number of decimal places. + * @example { "amount": "0", "nai": "@@000000013", "precision": 3 } + */ + totalPayoutValue: asset; + + /** + * The portion of the `totalPayoutValue` that was distributed to "curators" (people who upvoted the content). + * This will also show "0" or be empty if the `payoutTime` is in the future. + * @example { "amount": "0", "nai": "@@000000013", "precision": 3 } + */ + curatorPayoutValue: asset; + + /** + * A list of other Hive accounts that will receive a percentage of this content's rewards. + * `account` is the username of the beneficiary. + * `weight` represents their share (e.g., 100 means 1%, 1000 means 10%). + * @example [{ account: "sagarkothari88", weight: 100 }, { account: "spk.beneficiary", weight: 1000 }] + */ + beneficiaries: Array; + + /** + * Indicates if other users are allowed to reply to this content. + * @example true + */ + allowsReplies: boolean; + + /** + * Indicates if other users are allowed to vote on this content. + * @example true + */ + allowsVotes: boolean; + + /** + * Indicates if users who upvote this content are eligible to earn curation rewards. + * @example true + */ + allowsCurationRewards: boolean; + + /** + * The amount of rewards, in Hive Power, that the author received after the payout. + * This will be 0 if the post hasn't paid out yet. + * @example 0 + */ + authorRewards: Long; +} + +export type TContentAuthorData = Record; + +export interface IContentData { + contentData: Record; +} + +export class ContentClassifier extends CollectorClassifierBase {} diff --git a/src/chain-observers/classifiers/index.ts b/src/chain-observers/classifiers/index.ts index 2125601..3ba69c5 100644 --- a/src/chain-observers/classifiers/index.ts +++ b/src/chain-observers/classifiers/index.ts @@ -10,3 +10,4 @@ export { WitnessClassifier } from "./witness-classifier"; export { ChangeRecoveryInProgressClassifier } from "./change-recovery-in-progress-classifier"; export { DeclineVotingRightsClassifier } from "./decline-voting-rights-classifier"; export { ManabarClassifier } from "./manabar-classifier"; +export { ContentClassifier } from "./content-classifier"; diff --git a/src/chain-observers/collectors/jsonrpc/block-collector.ts b/src/chain-observers/collectors/jsonrpc/block-collector.ts index 060ed9e..16431f9 100644 --- a/src/chain-observers/collectors/jsonrpc/block-collector.ts +++ b/src/chain-observers/collectors/jsonrpc/block-collector.ts @@ -1,7 +1,7 @@ import { transaction } from "@hiveio/wax"; -import { ITransactionData } from "src/chain-observers/classifiers/block-classifier"; import { WorkerBeeError } from "../../../errors"; import { DynamicGlobalPropertiesClassifier, BlockClassifier } from "../../classifiers"; +import { ITransactionData } from "../../classifiers/block-classifier"; import { TRegisterEvaluationContext } from "../../classifiers/collector-classifier-base"; import { DataEvaluationContext } from "../../factories/data-evaluation-context"; import { CollectorBase, TAvailableClassifiers } from "../collector-base"; diff --git a/src/chain-observers/collectors/jsonrpc/content-collector.ts b/src/chain-observers/collectors/jsonrpc/content-collector.ts new file mode 100644 index 0000000..255a02b --- /dev/null +++ b/src/chain-observers/collectors/jsonrpc/content-collector.ts @@ -0,0 +1,99 @@ +import { TAccountName } from "@hiveio/wax"; +import Long from "long"; +import { OrderedQueue } from "../../../types/queue"; +import { ContentClassifier, IHiveContent } from "../../classifiers/content-classifier"; +import { DataEvaluationContext } from "../../factories/data-evaluation-context"; +import { CollectorBase, TAvailableClassifiers } from "../collector-base"; + +export const BUCKET_INTERVAL = 3 * 1000; // 3 seconds (Hive block interval) + +export interface IContentCollectorOptions { + account: string; + permlink: string; + rollbackContractAfter: Date; +} + +export type TContentContractData = [TAccountName, string]; + +const MAX_CONTENT_GET_LIMIT = 1000; + +export class ContentCollector extends CollectorBase { + private readonly contractTimestamps = new OrderedQueue(); + private readonly contentCached = new Set(); + + protected pushOptions(data: IContentCollectorOptions): void { + const permlink = `${data.account}/${data.permlink}`; + + const rollbackAfter = Math.floor(data.rollbackContractAfter.getTime()); + const bucketInterval = rollbackAfter + (BUCKET_INTERVAL - (rollbackAfter % BUCKET_INTERVAL)); + + if (this.contentCached.has(permlink)) + // If we already have this content, we don't need to enqueue it again + return; + + this.contractTimestamps.enqueue(bucketInterval, [data.account, data.permlink]); + this.contentCached.add(permlink); + } + + // Data is automatically managed and popped from the queue + protected popOptions(_data: IContentCollectorOptions): void {} + + public async fetchData(data: DataEvaluationContext) { + const contentData: Record> = {}; + + const allData: Array<[TAccountName, string]> = []; + + const currentTime = Date.now(); + for(const { value } of this.contractTimestamps.dequeueUntil(currentTime)) { + for(const apiData of value) + allData.push(apiData); + + // Cleanup + this.contentCached.delete(`${value[0]}/${value[1]}`); + } + + for (let i = 0; i < allData.length; i += MAX_CONTENT_GET_LIMIT) { + const chunk = allData.slice(i, i + MAX_CONTENT_GET_LIMIT); + + const startFindComments = Date.now(); + const { comments } = await this.worker.chain!.api.database_api.find_comments({ comments: chunk }); + data.addTiming("database_api.find_comments", Date.now() - startFindComments); + + const startCommentsAnalysis = Date.now(); + + for(const comment of comments) { + contentData[comment.author] = contentData[comment.author] || {}; + contentData[comment.author][comment.permlink] = { + author: comment.author, + permlink: comment.permlink, + parentAuthor: comment.parent_author, + parentPermlink: comment.parent_permlink, + allowsCurationRewards: comment.allow_curation_rewards, + allowsReplies: comment.allow_replies, + allowsVotes: comment.allow_votes, + authorRewards: Long.fromValue(comment.author_rewards), + beneficiaries: comment.beneficiaries, + category: comment.category, + created: new Date(`${comment.created}Z`), + curatorPayoutValue: comment.curator_payout_value, + depth: comment.depth, + lastUpdated: new Date(`${comment.last_update}Z`), + netRshares: Long.fromValue(comment.net_rshares), + netVotes: comment.net_votes, + payoutTime: new Date(`${comment.last_payout}Z`), + replyCount: comment.children, + totalPayoutValue: comment.total_payout_value, + title: comment.title + }; + } + + data.addTiming("commentsAnalysis", Date.now() - startCommentsAnalysis); + } + + return { + [ContentClassifier.name]: { + contentData + } as TAvailableClassifiers["ContentClassifier"], + } satisfies Partial; + }; +} diff --git a/src/chain-observers/factories/data-evaluation-context.ts b/src/chain-observers/factories/data-evaluation-context.ts index 621b68a..0288d69 100644 --- a/src/chain-observers/factories/data-evaluation-context.ts +++ b/src/chain-observers/factories/data-evaluation-context.ts @@ -31,6 +31,32 @@ export class DataEvaluationContext { this.collectors[evaluationContext.name] = collector; } + /** + * Available for dynamically pushing options from the classifier. + */ + public pushClassifierOptions(classifier: T, options: Record) { + const collector = this.collectors[classifier.name]; + if (!collector) + throw new WorkerBeeError(`Could not find collector for classifier: "${classifier.name}" when pushing options`); + + collector.pushOptions(options); + } + + /** + * Available for dynamically popping options from the classifier. + */ + public popClassifierOptions(classifier: T, options: Record) { + const collector = this.collectors[classifier.name]; + if (!collector) + throw new WorkerBeeError(`Could not find collector for classifier: "${classifier.name}" when popping options`); + + collector.popOptions(options); + } + + public hasClassifierRegistered(classifier: T): boolean { + return this.collectors[classifier.name] !== undefined; + } + public async get infer R ? R : never>( evaluationContext: T ): Promise { diff --git a/src/chain-observers/factories/historydata/factory-data.ts b/src/chain-observers/factories/historydata/factory-data.ts index 4c7e5f6..37aac19 100644 --- a/src/chain-observers/factories/historydata/factory-data.ts +++ b/src/chain-observers/factories/historydata/factory-data.ts @@ -1,6 +1,7 @@ import type { WorkerBee } from "../../../bot"; import { BlockClassifier, BlockHeaderClassifier, + ContentClassifier, DynamicGlobalPropertiesClassifier, ImpactedAccountClassifier, OperationClassifier } from "../../classifiers"; import { IEvaluationContextClass } from "../../classifiers/collector-classifier-base"; @@ -9,6 +10,7 @@ import { ImpactedAccountCollector } from "../../collectors/common/impacted-accou import { OperationCollector } from "../../collectors/common/operation-collector"; import { BlockCollector } from "../../collectors/historydata/block-collector"; import { DynamicGlobalPropertiesCollector } from "../../collectors/historydata/dynamic-global-properties-collector"; +import { ContentCollector } from "../../collectors/jsonrpc/content-collector"; export const HistoryDataFactoryData = (worker: WorkerBee, fromBlock: number, toBlock?: number): Array<[IEvaluationContextClass, CollectorBase]> => { const blockClassifier = new BlockCollector(worker, fromBlock, toBlock); @@ -19,5 +21,6 @@ export const HistoryDataFactoryData = (worker: WorkerBee, fromBlock: number, toB [BlockClassifier, blockClassifier], [ImpactedAccountClassifier, new ImpactedAccountCollector(worker)], [OperationClassifier, new OperationCollector(worker)], + [ContentClassifier, new ContentCollector(worker)] ]; }; diff --git a/src/chain-observers/factories/jsonrpc/factory-data.ts b/src/chain-observers/factories/jsonrpc/factory-data.ts index b68cc92..1b52425 100644 --- a/src/chain-observers/factories/jsonrpc/factory-data.ts +++ b/src/chain-observers/factories/jsonrpc/factory-data.ts @@ -2,6 +2,7 @@ import type { WorkerBee } from "../../../bot"; import { AccountClassifier, BlockClassifier, BlockHeaderClassifier, ChangeRecoveryInProgressClassifier, + ContentClassifier, DeclineVotingRightsClassifier, DynamicGlobalPropertiesClassifier, FeedPriceClassifier, ImpactedAccountClassifier, ManabarClassifier, OperationClassifier, RcAccountClassifier, WitnessClassifier @@ -15,6 +16,7 @@ import { OperationCollector } from "../../collectors/common/operation-collector" import { AccountCollector } from "../../collectors/jsonrpc/account-collector"; import { BlockCollector } from "../../collectors/jsonrpc/block-collector"; import { ChangeRecoveryInProgressCollector } from "../../collectors/jsonrpc/change-recovery-in-progress-collector"; +import { ContentCollector } from "../../collectors/jsonrpc/content-collector"; import { DeclineVotingRightsCollector } from "../../collectors/jsonrpc/decline-voting-rights-collector"; import { DynamicGlobalPropertiesCollector } from "../../collectors/jsonrpc/dynamic-global-properties-collector"; import { FeedPriceCollector } from "../../collectors/jsonrpc/feed-price-collector"; @@ -34,4 +36,5 @@ export const JsonRpcFactoryData: (worker: WorkerBee) => Array<[IEvaluationContex [ChangeRecoveryInProgressClassifier, new ChangeRecoveryInProgressCollector(worker)], [DeclineVotingRightsClassifier, new DeclineVotingRightsCollector(worker)], [ManabarClassifier, new ManabarCollector(worker)], + [ContentClassifier, new ContentCollector(worker)] ]); diff --git a/src/chain-observers/filters/content-metadata-filter.ts b/src/chain-observers/filters/content-metadata-filter.ts new file mode 100644 index 0000000..4c08c6d --- /dev/null +++ b/src/chain-observers/filters/content-metadata-filter.ts @@ -0,0 +1,82 @@ +import { TAccountName } from "@hiveio/wax"; +import type { WorkerBee } from "../../bot"; +import { ContentClassifier } from "../classifiers"; +import type { TRegisterEvaluationContext } from "../classifiers/collector-classifier-base"; +import type { DataEvaluationContext } from "../factories/data-evaluation-context"; +import { FilterBase } from "./filter-base"; + +export interface ICommentData { + parentAuthor: TAccountName; + parentPermlink: string; +} + +// Base class for content filters (posts and comments) +export abstract class ContentMetadataFilter extends FilterBase { + public constructor( + worker: WorkerBee, + accounts: string[], + protected readonly isPost: boolean, + protected readonly parentCommentFilter?: ICommentData + ) { + super(worker); + + this.accounts = new Set(accounts); + } + + protected readonly accounts: Set; + + public usedContexts(): Array { + return [ + ContentClassifier + ]; + } + + public async match(data: DataEvaluationContext): Promise { + const { contentData } = await data.get(ContentClassifier); + + for(const account in contentData) { + if (!this.accounts.has(account)) + continue; + const content = contentData[account]; + for(const permlink in content) { + const operation = content[permlink]; + // Check if post/comment type matches what we're looking for + const postIndicator = operation.parentAuthor && operation.parentAuthor === ""; + if (this.isPost !== postIndicator) + continue; + + // Check parent data if specified + if (!this.isPost && this.parentCommentFilter && ( + operation.parentPermlink !== this.parentCommentFilter.parentPermlink + || operation.parentAuthor !== this.parentCommentFilter.parentAuthor + )) + continue; + + return true; + } + } + + return false; + } +} + +// Filter for comments (replies to posts or other comments) +export class CommentContentMetadataFilter extends ContentMetadataFilter { + public constructor( + worker: WorkerBee, + accounts: string[], + parentCommentFilter?: ICommentData + ) { + super(worker, accounts, false, parentCommentFilter); + } +} + +// Filter for posts (top-level content with empty parent_author) +export class PostContentMetadataFilter extends ContentMetadataFilter { + public constructor( + worker: WorkerBee, + accounts: string[] + ) { + super(worker, accounts, true); + } +} diff --git a/src/chain-observers/index.ts b/src/chain-observers/index.ts index 507b431..a7dfeaa 100644 --- a/src/chain-observers/index.ts +++ b/src/chain-observers/index.ts @@ -10,6 +10,7 @@ export type { IImpactedAccount, IImpactedAccountData } from "./classifiers/impac export type { IOperationBaseData, IOperationData, IOperationTransactionPair } from "./classifiers/operation-classifier"; export type { IRcAccount, IRcAccountData } from "./classifiers/rc-account-classifier"; export type { IWitness, IWitnessData } from "./classifiers/witness-classifier"; +export type { IHiveContent } from "./classifiers/content-classifier"; export type { IAccountProviderData, TAccountProvided } from "./providers/account-provider"; export { EAlarmType, type IAlarmAccountsData, type TAlarmAccounts } from "./providers/alarm-provider"; @@ -34,5 +35,6 @@ export type { IRcAccountProviderData, TRcAccountProvided } from "./providers/rc- export type { ITransactionProviderData, TTransactionProvider } from "./providers/transaction-provider"; export type { IWhaleAlertMetadata, IWhaleAlertProviderData } from "./providers/whale-alert-provider"; export type { IWitnessProviderData, TWitnessProvider } from "./providers/witness-provider"; +export type { IBlogContentMetadataroviderData, TBlogContentMetadataProvided } from "./providers/content-metadata-provider"; export { Exchange } from "../utils/known-exchanges"; diff --git a/src/chain-observers/providers/blog-content-provider.ts b/src/chain-observers/providers/blog-content-provider.ts index 704a463..b87e4d5 100644 --- a/src/chain-observers/providers/blog-content-provider.ts +++ b/src/chain-observers/providers/blog-content-provider.ts @@ -1,11 +1,15 @@ import { comment, TAccountName } from "@hiveio/wax"; import { WorkerBeeArrayIterable, WorkerBeeIterable } from "../../types/iterator"; +import { calculateRelativeTime } from "../../utils/time"; +import { BlockHeaderClassifier, ContentClassifier } from "../classifiers"; import { TRegisterEvaluationContext } from "../classifiers/collector-classifier-base"; import { IOperationTransactionPair, OperationClassifier } from "../classifiers/operation-classifier"; import { DataEvaluationContext } from "../factories/data-evaluation-context"; import { ICommentData } from "../filters/blog-content-filter"; import { ProviderBase } from "./provider-base"; +const ONE_WEEK_MS = 7 * 24 * 60 * 60 * 1000; // One week in milliseconds + // Common interface for blog content data (posts and comments) export type TBlogContentProvided> = { [K in TAccounts[number]]: Array> | WorkerBeeIterable>; @@ -19,11 +23,13 @@ export interface ICommentProviderAuthors { export interface ICommentProviderOptions { authors: ICommentProviderAuthors[]; + collectPostMetadataAfter?: string; } // Post provider implementation export interface IPostProviderOptions { authors: string[]; + collectPostMetadataAfter?: string; } export interface ICommentProviderData> { @@ -34,12 +40,17 @@ export interface IPostProviderData> { posts: Partial>; } +export interface IAuthorsOptions { + commentData?: ICommentData; + collectPostMetadataAfter?: number; +} + // Base class for blog content providers (posts and comments) export abstract class BlogContentProvider< TAccounts extends Array = Array, TOptions extends object = object > extends ProviderBase { - public readonly authors = new Map(); + public readonly authors = new Map(); protected readonly isPost: boolean; public constructor(isPost: boolean) { @@ -56,6 +67,8 @@ export abstract class BlogContentProvider< public async createProviderData(data: DataEvaluationContext): Promise> { const result = {} as TBlogContentProvided; + const { timestamp } = await data.get(BlockHeaderClassifier); + const accounts = await data.get(OperationClassifier); if (accounts.operationsPerType.comment) for (const operation of accounts.operationsPerType.comment) { @@ -65,15 +78,15 @@ export abstract class BlogContentProvider< continue; /// If requested account is a post author - if(this.authors.has(operation.operation.author) === false) + if(this.authors.get(operation.operation.author)?.commentData === undefined) continue; let account = operation.operation.author; /// If requested account is a comment parent-author - const parentCommentFilter = this.authors.get(operation.operation.author); + const parentCommentFilter = this.authors.get(operation.operation.author)!; - if(!this.isPost && parentCommentFilter && operation.operation.parent_permlink !== parentCommentFilter.parentPermlink) + if(!this.isPost && parentCommentFilter.commentData && operation.operation.parent_permlink !== parentCommentFilter.commentData.parentPermlink) continue; else account = operation.operation.parent_author; @@ -83,6 +96,14 @@ export abstract class BlogContentProvider< const storage = result[account] as WorkerBeeArrayIterable>; + // Dynamically inject options for the content classifier when post is created + if (parentCommentFilter.collectPostMetadataAfter && data.hasClassifierRegistered(ContentClassifier)) + data.pushClassifierOptions(ContentClassifier, { + account, + permlink: operation.operation.permlink, + rollbackContractAfter: new Date(timestamp.getTime() + parentCommentFilter.collectPostMetadataAfter) + }); + storage.push({ operation: operation.operation, transaction: operation.transaction @@ -100,7 +121,12 @@ export class CommentProvider = Array> { @@ -117,7 +143,11 @@ export class PostProvider = Array> { diff --git a/src/chain-observers/providers/content-metadata-provider.ts b/src/chain-observers/providers/content-metadata-provider.ts new file mode 100644 index 0000000..53ce8b6 --- /dev/null +++ b/src/chain-observers/providers/content-metadata-provider.ts @@ -0,0 +1,35 @@ +import { TAccountName } from "@hiveio/wax"; +import { WorkerBeeIterable } from "../../types/iterator"; +import { TRegisterEvaluationContext } from "../classifiers/collector-classifier-base"; +import { ContentClassifier, IHiveContent } from "../classifiers/content-classifier"; +import { DataEvaluationContext } from "../factories/data-evaluation-context"; +import { ProviderBase } from "./provider-base"; + +export type TBlogContentMetadataProvided> = { + [K in TAccounts[number]]: Array | WorkerBeeIterable; +}; + +export interface IBlogContentMetadataroviderData> { + contentMetadataPerAccount: Partial>; +}; + +export class BlogContentMetadataProvider = Array> extends ProviderBase { + public usedContexts(): Array { + return [ContentClassifier]; + } + + public async provide(data: DataEvaluationContext): Promise> { + const result = { + contentMetadataPerAccount: {} + }; + + const content = await data.get(ContentClassifier); + for(const account in content.contentData) { + result.contentMetadataPerAccount[account] = content.contentData[account] ?? []; + for(const post in content.contentData[account]) + result.contentMetadataPerAccount[account].push(content.contentData[account][post]); + } + + return result as IBlogContentMetadataroviderData; + } +} diff --git a/src/past-queen.ts b/src/past-queen.ts index 0427d63..8318d57 100644 --- a/src/past-queen.ts +++ b/src/past-queen.ts @@ -5,10 +5,11 @@ import { QueenBee } from "./queen"; export type TPastQueen = Omit< PastQueen, - "onAccountFullManabar" | "onAccountBalanceChange" | "onAccountMetadataChange" | - "onFeedPriceChange" | "onFeedPriceNoChange" | "provideFeedPriceData" | - "onAlarm" | "onWitnessMissedBlocks" | "provideAccounts" | - "provideWitnesses" | "provideRcAccounts" + "onAccountFullManabar" | "onAccountBalanceChange" | "onAccountMetadataChange" | + "onFeedPriceChange" | "onFeedPriceNoChange" | "provideFeedPriceData" | + "onAlarm" | "onWitnessMissedBlocks" | "provideAccounts" | + "provideWitnesses" | "provideRcAccounts" | "onPostsIncomingPayment" | + "onCommentsIncomingPayment" >; export class PastQueen extends QueenBee { diff --git a/src/queen.ts b/src/queen.ts index 93b0a97..7ce9a2d 100644 --- a/src/queen.ts +++ b/src/queen.ts @@ -9,6 +9,7 @@ import { BalanceChangeFilter } from "./chain-observers/filters/balance-change-fi import { BlockNumberFilter } from "./chain-observers/filters/block-filter"; import { CommentFilter, PostFilter } from "./chain-observers/filters/blog-content-filter"; import { LogicalAndFilter, LogicalOrFilter } from "./chain-observers/filters/composite-filter"; +import { CommentContentMetadataFilter, PostContentMetadataFilter } from "./chain-observers/filters/content-metadata-filter"; import { CustomOperationFilter } from "./chain-observers/filters/custom-operation-filter"; import { ExchangeTransferFilter } from "./chain-observers/filters/exchange-transfer-filter"; import { FeedPriceChangeFilter } from "./chain-observers/filters/feed-price-change-percent-filter"; @@ -30,6 +31,7 @@ import { AlarmProvider } from "./chain-observers/providers/alarm-provider"; import { BlockHeaderProvider } from "./chain-observers/providers/block-header-provider"; import { BlockProvider } from "./chain-observers/providers/block-provider"; import { CommentProvider, PostProvider } from "./chain-observers/providers/blog-content-provider"; +import { BlogContentMetadataProvider } from "./chain-observers/providers/content-metadata-provider"; import { CustomOperationProvider } from "./chain-observers/providers/custom-operation-provider"; import { ExchangeTransferProvider } from "./chain-observers/providers/exchange-transfer-provider"; import { FeedPriceProvider } from "./chain-observers/providers/feed-price-provider"; @@ -417,6 +419,74 @@ export class QueenBee { return this; } + /** + * Subscribes to notifications when a post is given relative time close to the incoming payment time + * + * Automatically provides the post metadata (including e.g. net rshares) in the `next` callback. + * + * Note: This method implicitly applies the OR operator between the specified accounts. + * + * @example + * ```ts + * workerbee.observe.onPostsIncomingPayment("-6s", "username", "username2").subscribe({ + * next: (data) => { // Will notify 6 seconds before the post receives the payment + * for(const account in data.contentMetadataPerAccount) + * for(const { author, permlink, netVotes } of data.posts[account]) + * console.log(`@${author}/${permlink} received ${netVotes} votes`); + * } + * }); + * ``` + * + * @param authors account names of the authors to monitor for post incoming payout. + * @returns itself + */ + public onPostsIncomingPayment< + TAccounts extends TAccountName[] + >(relativeTime: string, ...authors: TAccounts): QueenBee["provide"]>>> { + this.operands.push(new PostContentMetadataFilter(this.worker, authors)); + + this.pushProvider(PostProvider, { collectPostMetadataAfter: relativeTime, authors }); + + this.pushProvider(BlogContentMetadataProvider); + + return this; + } + + /** + * Subscribes to notifications when a comment is given relative time close to the incoming payment time + * + * Automatically provides the comment metadata (including e.g. net rshares) in the `next` callback. + * + * Note: This method implicitly applies the OR operator between the specified accounts. + * + * @example + * ```ts + * workerbee.observe.onCommentsIncomingPayment("-6s", "username", "username2").subscribe({ + * next: (data) => { // Will notify 6 seconds before the comment receives the payment + * for(const account in data.contentMetadataPerAccount) + * for(const { author, permlink, netVotes } of data.comments[account]) + * console.log(`@${author}/${permlink} received ${netVotes} votes`); + * } + * }); + * ``` + * + * @param authors account names of the authors to monitor for comment incoming payout. + * @returns itself + */ + public onCommentsIncomingPayment< + TAccounts extends TAccountName[] + >(relativeTime: string, ...authors: TAccounts): QueenBee["provide"]>>> { + // TODO: Handle parentPostOrComment?: ICommentData + + this.operands.push(new CommentContentMetadataFilter(this.worker, authors)); + + this.pushProvider(CommentProvider, { collectPostMetadataAfter: relativeTime, authors: authors.map(account => ({ account })) }); + + this.pushProvider(BlogContentMetadataProvider); + + return this; + } + /** * Subscribes to notifications when a new account is created. * diff --git a/src/types/queue.ts b/src/types/queue.ts new file mode 100644 index 0000000..24a0d9d --- /dev/null +++ b/src/types/queue.ts @@ -0,0 +1,56 @@ +import { WorkerBeeError } from "../errors"; + +export interface IOrderedItem> { + key: T; + value: K; +} + +/** + * A sorted queue that allows enqueuing items and dequeuing them in sorted order. + * Items are sorted in ascending order, and the queue can be drained until a specified value. + * If an item is enqueued with a key that is less than the last enqueued key, an error is thrown. + * If an item is enqueued with the same key, it is added to the existing array of values for that key. + * Requires items to be comparable (e.g., numbers, Dates) + */ +export class OrderedQueue { + private buf: IOrderedItem[] = []; + private head = 0; + private tail = 0; + + public enqueue(key: K, value: V): void { + if (this.buf[this.tail] !== undefined) { + if (this.buf[this.tail].key > key) + throw new WorkerBeeError("Items must be enqueued in sorted order."); + + if (this.buf[this.tail].key === key) + this.buf[this.tail].value.push(value); + else + this.buf[this.tail++] = { + key, + value: [value] + }; + } else + this.buf[this.tail++] = { + key, + value: [value] + }; + + } + + public *dequeueUntil(maxValue: K): Generator> { + while (this.head < this.tail && this.buf[this.head] !== undefined && this.buf[this.head].key <= maxValue) { + const item = this.buf[this.head]; + this.buf[this.head++] = undefined as any; + if (this.head === this.tail) { + this.head = this.tail = 0; + this.buf.length = 0; + } + + yield item; + } + } + + public get size(): number { + return this.tail - this.head; + } +} diff --git a/src/wax/index.ts b/src/wax/index.ts index 795b6a7..a659655 100644 --- a/src/wax/index.ts +++ b/src/wax/index.ts @@ -1,4 +1,4 @@ -import { createHiveChain, IHiveChainInterface, IWaxOptionsChain, price, TWaxExtended } from "@hiveio/wax"; +import { asset, beneficiary_route_type, createHiveChain, IHiveChainInterface, IWaxOptionsChain, price, TAccountName, TWaxExtended } from "@hiveio/wax"; export type WaxExtendTypes = { database_api: { @@ -22,6 +22,45 @@ export type WaxExtendTypes = { // ... }>; }; }; + find_comments: { + params: { comments: Array<[TAccountName, string]> }; + result: { comments: Array<{ + abs_rshares: 150255792948762, + allow_curation_rewards: boolean; + allow_replies: boolean; + allow_votes: boolean; + author: TAccountName; + author_rewards: number; + beneficiaries: Array; + body: string; + cashout_time: string; + category: string + children: number; + children_abs_rshares: number; + created: string; + curator_payout_value: asset; + depth: number; + id: number; + json_metadata: string; + last_payout: string; + last_update: string; + max_accepted_payout: asset; + max_cashout_time: string; + net_rshares: number | string; + net_votes: number; + parent_author: string + parent_permlink: string; + percent_hbd: number; + permlink: string; + reward_weight: string; + root_author: TAccountName; + root_permlink: string; + title: string; + total_payout_value: asset; + total_vote_weight: number | string; + vote_rshares: number | string; + }>; }; + }; find_decline_voting_rights_requests: { params: { accounts: string[]; }; result: { requests: Array<{ -- GitLab From 757bad77332080e50757c2a88c596ba301669b7e Mon Sep 17 00:00:00 2001 From: mtyszczak Date: Thu, 22 May 2025 12:14:24 +0200 Subject: [PATCH 3/3] Increase package size limit to 155 kB --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 58a035d..49f20a6 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "path": [ "./dist/bundle" ], - "limit": "140 kB", + "limit": "155 kB", "brotli": false } ], -- GitLab