add bookmark functionality #330

hive/db/db_state.py

@@ -7,7 +7,8 @@
 
 from hive.db.schema import (setup, reset_autovac, build_metadata,
                             build_metadata_community, teardown, DB_VERSION,
-                            build_metadata_blacklist, build_trxid_block_num)
+                            build_metadata_blacklist, build_trxid_block_num,
+                            build_metadata_bookmarks)
 from hive.db.adapter import Db
 
 log = logging.getLogger(__name__)
@@ -331,6 +332,12 @@ def _check_migrations(cls):
             cls.db().query("CREATE INDEX hive_block_num_ix1 ON hive_trxid_block_num (block_num)")
             cls.db().query("CREATE UNIQUE INDEX hive_trxid_ix1 ON hive_trxid_block_num (trx_id) WHERE trx_id IS NOT NULL")
             cls._set_ver(20)
+        if cls._ver == 20:
+            if not cls.db().query_col("SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name='hive_bookmarks')")[0]:
+                build_metadata_bookmarks().create_all(cls.db().engine())
+                cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk1 FOREIGN KEY (account) REFERENCES hive_accounts(name);")
+                cls.db().query("ALTER TABLE hive_bookmarks ADD CONSTRAINT hive_bookmarks_fk2 FOREIGN KEY (post_id) REFERENCES hive_posts(id);")
+            cls._set_ver(21)
 
         reset_autovac(cls.db())
 

hive/db/schema.py

@@ -10,7 +10,7 @@
 
 #pylint: disable=line-too-long, too-many-lines, bad-whitespace
 
-DB_VERSION = 20
+DB_VERSION = 21
 
 def build_metadata():
     """Build schema def with SqlAlchemy"""
@@ -243,6 +243,8 @@ def build_metadata():
 
     metadata = build_trxid_block_num(metadata)
 
+    metadata = build_metadata_bookmarks(metadata)
+
     return metadata
 
 def build_metadata_community(metadata=None):
@@ -356,6 +358,23 @@ def build_trxid_block_num(metadata=None):
 
     return metadata
 
+def build_metadata_bookmarks(metadata=None):
+    if not metadata:
+        metadata = sa.MetaData()
+
+    sa.Table(
+        'hive_bookmarks', metadata,
+        sa.Column('account', VARCHAR(16), nullable=False),
+        sa.Column('post_id', sa.Integer, nullable=False),
+        sa.Column('bookmarked_at', sa.DateTime, nullable=False),
+
+        sa.UniqueConstraint('account', 'post_id', name='hive_bookmarks_ux1'),
+        sa.Index('hive_bookmarks_ix1', 'post_id'), 
+        sa.Index('hive_bookmarks_ix2', 'account', 'post_id'), 
+        sa.Index('hive_bookmarks_ix3', 'account', 'bookmarked_at'),
+    )
+
+    return metadata
 
 def teardown(db):
     """Drop all tables"""

hive/indexer/blocks.py

@@ -229,6 +229,7 @@ def _pop(cls, blocks):
             DB.query("DELETE FROM hive_payments    WHERE block_num = :num", num=num)
             DB.query("DELETE FROM hive_blocks      WHERE num = :num", num=num)
             DB.query("DELETE FROM hive_trxid_block_num WHERE block_num = :num", num=num)
+            DB.query("DELETE FROM hive_bookmarks   WHERE bookmarked_at >= :date", date=date)
 
         DB.query("COMMIT")
         log.warning("[FORK] recovery complete")

hive/indexer/bookmark.py

@@ -0,0 +1,79 @@
+"""Handles bookmark operations."""
+
+import logging
+
+from hive.db.adapter import Db
+from hive.indexer.accounts import Accounts
+from hive.indexer.posts import Posts
+
+log = logging.getLogger(__name__)
+
+DB = Db.instance()
+
+
+class Bookmark:
+    """Handles processing of adding and removing bookmarks and flushing to db."""
+
+    @classmethod
+    def bookmark_op(cls, account, op_json, date):
+        """Process an incoming bookmark op."""
+        op = cls._validated_op(account, op_json, date)
+        if not op:
+            return
+
+        # perform add bookmark
+        if op['action'] == 'add':
+            sql = """INSERT INTO hive_bookmarks (account, post_id, bookmarked_at)
+                     VALUES (:account, :post_id, :at)"""
+            DB.query(sql, **op)
+
+        # perform remove bookmark
+        elif op['action'] == 'remove':
+            sql = """DELETE FROM hive_bookmarks
+                     WHERE account = :account AND post_id = :post_id"""
+            DB.query(sql, **op)
+
+    @classmethod
+    def _validated_op(cls, account, op, date):
+        """Validate and normalize the operation."""
+
+        min_params = ['account', 'author', 'permlink', 'action', 'category']
+        if any(param not in op for param in min_params):
+            # invalid op
+            return None
+
+        if account != op['account']:
+            # impersonation
+            return None
+
+        if op['action'] not in ['add', 'remove']:
+            # invalid action
+            return None
+
+        account_id = Accounts.get_id(account)
+        if not account_id:
+            # invalid account
+            return None
+
+        post_id = Posts.get_id(op['author'], op['permlink'])
+        if not post_id:
+            # invalid post
+            return None
+
+        is_bookmarked = cls._is_bookmarked(account, post_id)
+        if ((is_bookmarked and op['action'] == 'add')              # already bookmarked
+           or (not is_bookmarked and op['action'] == 'remove')):   # not bookmarked
+            # invalid action
+            return None
+
+        return dict(account=account,
+                    post_id=post_id,
+                    action=op['action'],
+                    at=date)
+
+    @classmethod
+    def _is_bookmarked(cls, account, post_id):
+        """Return bookmark if it exists."""
+        sql = """SELECT 1 FROM hive_bookmarks
+                 WHERE account = :account AND post_id = :post_id"""
+        return DB.query_one(sql, account=account, post_id=post_id)

hive/indexer/custom_op.py

@@ -10,6 +10,7 @@
 from hive.indexer.feed_cache import FeedCache
 from hive.indexer.follow import Follow
 from hive.indexer.notify import Notify
+from hive.indexer.bookmark import Bookmark
 
 from hive.indexer.community import process_json_community_op, START_BLOCK
 from hive.utils.normalize import load_json_key
@@ -78,7 +79,7 @@ def _process_notify(cls, account, op_json, block_date):
 
     @classmethod
     def _process_legacy(cls, account, op_json, block_date):
-        """Handle legacy 'follow' plugin ops (follow/mute/clear, reblog)
+        """Handle legacy 'follow' plugin ops (follow/mute/clear, reblog, bookmark)
 
         follow {follower: {type: 'account'},
                 following: {type: 'account'},
@@ -87,12 +88,17 @@ def _process_legacy(cls, account, op_json, block_date):
                 author: {type: 'account'},
                 permlink: {type: 'permlink'},
                 delete: {type: 'str', optional: True}}
+        bookmark {account: {type: 'account'},
+                  author: {type: 'account'}, 
+                  permlink: {type: 'permlink'}, 
+                  action: {type: 'str'}, 
+                  category: {type: 'str'}} // category currently unused
         """
         if not isinstance(op_json, list):
             return
         if len(op_json) != 2:
             return
-        if first(op_json) not in ['follow', 'reblog']:
+        if first(op_json) not in ['follow', 'reblog', 'bookmark']:
             return
         if not isinstance(second(op_json), dict):
             return
@@ -102,6 +108,8 @@ def _process_legacy(cls, account, op_json, block_date):
             Follow.follow_op(account, op_json, block_date)
         elif cmd == 'reblog':
             cls.reblog(account, op_json, block_date)
+        elif cmd == 'bookmark':
+            Bookmark.bookmark_op(account, op_json, block_date)
 
     @classmethod
     def reblog(cls, account, op_json, block_date):

hive/server/bridge_api/cursor.py

@@ -465,3 +465,55 @@ async def pids_by_payout(db, account: str, start_author: str = '',
     """ % seek
 
     return await db.query_col(sql, account=account, start_id=start_id, limit=limit)
+
+async def pids_by_bookmarks(db, account: str, sort: str = 'bookmarks', category: str = '', start_author: str = '',
+                            start_permlink: str = '', limit: int = 20):
+    """Get a list of post_ids for an author's bookmarks."""
+    seek = ''
+    join = ''
+    start_id = await _get_post_id(db, start_author, start_permlink) if start_permlink else None
+
+    if sort == 'bookmarks':
+        # order by age of bookmarks
+        order_by = "bookmarks.bookmarked_at DESC"
+        if start_permlink:
+            seek = """
+                AND bookmarks.bookmarked_at < (
+                    SELECT bookmarked_at FROM hive_bookmarks
+                     WHERE post_id = :start_id
+                       AND account = :account
+                )
+            """
+
+    elif sort == 'posts':
+        # order by age of posts
+        order_by = "bookmarks.post_id DESC"
+        if start_permlink:
+            seek = "AND bookmarks.post_id < :start_id"
+
+    elif sort == 'authors':
+        # order by name of authors
+        # sort by author, then by post_id 
+        # for paging we need a sort if more than one post by the same author is bookmarked
+        join = "JOIN hive_posts AS posts ON bookmarks.post_id = posts.id"
+        order_by = "posts.author ASC, bookmarks.post_id DESC"
+        if start_permlink:
+            seek = """
+                AND (posts.author > :start_author
+                OR (posts.author = :start_author AND bookmarks.post_id < :start_id))
+            """
+
+    sql = """
+        SELECT bookmarks.post_id
+          FROM hive_bookmarks AS bookmarks
+          %s
+         WHERE bookmarks.account = :account %s
+      ORDER BY %s
+         LIMIT :limit
+    """ % (join, seek, order_by)
+
+    return await db.query_col(sql, 
+                              account=account, 
+                              start_id=start_id, 
+                              start_author=start_author, 
+                              limit=limit)

hive/server/bridge_api/methods.py

@@ -156,3 +156,36 @@ async def get_account_posts(context, sort, account, start_author='', start_perml
             if pid in ids:
                 ids.remove(pid)
         return await load_posts(context['db'], ids)
+
+@return_error_info
+async def get_bookmarked_posts(context, account, sort='bookmarks', category='', start_author='', start_permlink='',
+                               limit=20, observer=None):
+    """Get bookmarked posts for an account"""
+
+    # valid sorts are by age of posts, age of bookmarks or authors
+    valid_sorts = ['posts', 'bookmarks', 'authors']
+    assert sort in valid_sorts, 'invalid bookmark sort'
+    assert account, 'account is required'
+
+    db = context['db']
+    account = valid_account(account)
+    start_author = valid_account(start_author, allow_empty=True)
+    start_permlink = valid_permlink(start_permlink, allow_empty=True)
+    start = (start_author, start_permlink)
+    limit = valid_limit(limit, 50)
+    category = '' # currently unused
+
+    # check blacklist accounts
+    _id = await db.query_one("SELECT id FROM hive_posts_status WHERE author = :n AND list_type = '3'", n=account)
+    if _id:
+        return []
+
+    ids = await cursor.pids_by_bookmarks(
+        context['db'],
+        account,
+        sort,
+        category,
+        *start,
+        limit)
+
+    return await load_posts(context['db'], ids)    

hive/server/bridge_api/objects.py

@@ -75,6 +75,14 @@ async def load_posts_keyed(db, ids, truncate_body=0):
                 ctx[cid] = []
             ctx[cid].append(author['id'])
 
+    # fetch bookmarks
+    sql = """SELECT post_id, string_agg(account, ',') AS bookmarked_by
+               FROM hive_bookmarks
+              WHERE post_id IN :ids
+           GROUP BY post_id"""
+    bookmarks = await db.query_all(sql, ids=tuple(ids))
+    bookmarks = {row[0]:row[1].split(',') for row in bookmarks}
+
     # TODO: optimize
     titles = {}
     roles = {}
@@ -105,6 +113,8 @@ async def load_posts_keyed(db, ids, truncate_body=0):
                                      or len(post['blacklists']) >= 2)
         post['stats']['hide'] = 'irredeemables' in post['blacklists']
 
+        # add bookmarked account names
+        post["bookmarked_by"] = bookmarks.get(pid, [])
 
     sql = """SELECT id FROM hive_posts
               WHERE id IN :ids AND is_pinned = '1' AND is_deleted = '0'"""

hive/server/serve.py

@@ -127,6 +127,7 @@ def build_methods():
         bridge_api.get_ranked_posts,
         bridge_api.get_profile,
         bridge_api.get_trending_topics,
+        bridge_api.get_bookmarked_posts,
         hive_api_notify.post_notifications,
         hive_api_notify.account_notifications,
         hive_api_notify.unread_notifications,