From f932c1f44a28e4223a06afa04a5de7512f9d3761 Mon Sep 17 00:00:00 2001 From: Muneeb Ali Date: Tue, 11 Aug 2015 20:32:30 -0400 Subject: [PATCH 1/5] This commit adds 1) PEP8 changes, mostly 3 spaces to 4 spaces, 2) removes has_key(), 3) adds connectivity check to getinfo call --- bin/blockstore-cli | 344 ++++---- blockstore_client/client.py | 1468 +++++++++++++++++------------------ 2 files changed, 915 insertions(+), 897 deletions(-) diff --git a/bin/blockstore-cli b/bin/blockstore-cli index f16cad7bd5..d52d8a1880 100755 --- a/bin/blockstore-cli +++ b/bin/blockstore-cli @@ -5,14 +5,14 @@ ~~~~~ copyright: (c) 2014 by Halfmoon Labs, Inc. copyright: (c) 2015 by Blockstack.org - + This file is part of Blockstore-client. - + Blockstore-client is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Blockstore-client is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the @@ -35,291 +35,311 @@ parent_dir = os.path.abspath(current_dir + "/../") sys.path.insert(0, parent_dir) from blockstore_client import config, client, schemas, parsing, user +from blockstore_client import plugins + +log = config.log + -log = config.log +def pretty_dump(json_str): + """ pretty dump + """ + return json.dumps(json_str, sort_keys=True, indent=4, separators=(',', ': ')) -def pretty_dump( json_str ): - """ pretty dump - """ - return json.dumps(json_str, sort_keys=True, indent=4, separators=(',', ': ')) -def print_result( json_str ): - print pretty_dump( json_str ) - +def print_result(json_str): + print pretty_dump(json_str) + def run_cli(): - """ run cli - """ - - # TODO: read config file - proxy = client.session( config.BLOCKSTORED_SERVER, config.BLOCKSTORED_PORT ) - - from blockstore_client import plugins - client.register_storage( plugins.disk ) - - parser = argparse.ArgumentParser(description='Blockstore Cli version {}'.format(config.VERSION)) - - parser.add_argument( + """ run cli + """ + + # TODO: read config file + proxy = client.session(config.BLOCKSTORED_SERVER, config.BLOCKSTORED_PORT) + + client.register_storage(plugins.disk) + + parser = argparse.ArgumentParser( + description='Blockstore Cli version {}'.format(config.VERSION)) + + parser.add_argument( '--blockstored-server', help="""the hostname or IP address of the blockstored RPC server (default: {})""".format(config.BLOCKSTORED_SERVER)) - - parser.add_argument( + + parser.add_argument( '--blockstored-port', type=int, help="""the blockstored RPC port to connect to (default: {})""".format(config.BLOCKSTORED_PORT)) - parser.add_argument( + parser.add_argument( '--txid', type=str, help="the transaction hash for a partially-failed storage operation") - - subparsers = parser.add_subparsers( + + subparsers = parser.add_subparsers( dest='action', help='the action to be taken') - subparser = subparsers.add_parser( + subparser = subparsers.add_parser( 'getinfo', help='get basic info from the blockstored server') - subparser = subparsers.add_parser( + subparser = subparsers.add_parser( 'ping', help='check if the blockstored server is up') - # ------------------------------------ - subparser = subparsers.add_parser( + # ------------------------------------ + subparser = subparsers.add_parser( 'preorder', help=' | preorder a name') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name that you want to preorder') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the private key of the Bitcoin address that will own the name') - # ------------------------------------ - subparser = subparsers.add_parser( + # ------------------------------------ + subparser = subparsers.add_parser( 'register', help=' | register/claim a name') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name that you want to register/claim') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the private key of the Bitcoin address that will own the name') - # ------------------------------------ - subparser = subparsers.add_parser( + # ------------------------------------ + subparser = subparsers.add_parser( 'update', help=' | update storage index data and store it into the storage providers') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name that you want to update') - subparser.add_argument( + subparser.add_argument( 'storage_index_json', type=str, help='the JSON-encoded storage index to associate with the name') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the privatekey of the owner Bitcoin address') - - # ------------------------------------ - subparser = subparsers.add_parser( + + # ------------------------------------ + subparser = subparsers.add_parser( 'transfer', help='
| transfer a name') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name that you want to register/claim') - subparser.add_argument( + subparser.add_argument( 'address', type=str, help='the new owner Bitcoin address') - subparser.add_argument( + subparser.add_argument( 'keepdata', type=bool, help='whether or not the storage index should remain associated with the name') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the privatekey of the owner Bitcoin address') - # ------------------------------------ - subparser = subparsers.add_parser( + # ------------------------------------ + subparser = subparsers.add_parser( 'renew', help=' | renew a name') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name that you want to renew') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the privatekey of the owner Bitcoin address') - # ------------------------------------ - subparser = subparsers.add_parser( + # ------------------------------------ + subparser = subparsers.add_parser( 'namespace_preorder', help=' | preorder a namespace, in order to claim the namespace ID and begin populating it.') - subparser.add_argument( + subparser.add_argument( 'namespace_id', type=str, help='the human-readable namespace identifier') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the privatekey of the namespace creator') - - # ------------------------------------ - subparser = subparsers.add_parser( + + # ------------------------------------ + subparser = subparsers.add_parser( 'namespace_define', help=' | define a namespace\'s parameters, in preparation for importing names.') - subparser.add_argument( + subparser.add_argument( 'namespace_id', type=str, help='the human-readable namespace identifier') - subparser.add_argument( + subparser.add_argument( 'lifetime', type=int, help='the number of blocks for which a name will be valid (any value less than zero means "forever")') - subparser.add_argument( + subparser.add_argument( 'base_name_cost', type=int, help='the cost (in satoshis) for a 1-character name in this namespace') - subparser.add_argument( + subparser.add_argument( 'cost_decay_rate', type=float, help='the rate at which the value of a name decays, based on its length: if L is the length, R is the rate (this argument), and B is the base name cost, then the cost per name shall be ceil(B / (R^(L-1)))') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the privatekey of the namespace creator') - - # ------------------------------------ - subparser = subparsers.add_parser( + + # ------------------------------------ + subparser = subparsers.add_parser( 'namespace_begin', help=' | begin the namespace, completing its definition and opening it for registration by other parties.') - subparser.add_argument( + subparser.add_argument( 'namespace_id', type=str, help='the human-readable namespace identifier') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the privatekey of the namespace creator') - - # ------------------------------------ - subparser = subparsers.add_parser( + + # ------------------------------------ + subparser = subparsers.add_parser( 'put_mutable', help=' [] | Store mutable data into the storage providers, creating it if it does not exist.') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name that owns this data') - subparser.add_argument( + subparser.add_argument( 'data_id', type=str, help='the unchanging identifier for this data') - subparser.add_argument( + subparser.add_argument( 'data', type=str, help='the data to store') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the private key assocated with the name') - - # ------------------------------------ - subparser = subparsers.add_parser( + + # ------------------------------------ + subparser = subparsers.add_parser( 'put_immutable', help=' | Store immutable data into the storage providers, creating it if it does not exist.') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name that owns this data') - subparser.add_argument( + subparser.add_argument( 'data', type=str, help='the data to store') - subparser.add_argument( + subparser.add_argument( 'privatekey', type=str, help='the private key associated with the name') - # ------------------------------------ - subparser = subparsers.add_parser( + # ------------------------------------ + subparser = subparsers.add_parser( 'get_mutable', help=' | Get mutable data from the storage providers, and verify that the named user wrote it.') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name associated with the data') - subparser.add_argument( + subparser.add_argument( 'data_id', type=str, help='the unchanging identifier for this data') - - # ------------------------------------ - subparser = subparsers.add_parser( + + # ------------------------------------ + subparser = subparsers.add_parser( 'get_immutable', help=' | Get immutable data from the storage providers, and verify that the named user wrote it.') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name of the user') - subparser.add_argument( + subparser.add_argument( 'hash', type=str, help='the hash of the data') - # ------------------------------------ - subparser = subparsers.add_parser( + # ------------------------------------ + subparser = subparsers.add_parser( 'lookup', help=' | get the name record for a given name') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name to look up') - # ------------------------------------ - subparser = subparsers.add_parser( + # ------------------------------------ + subparser = subparsers.add_parser( 'getindex', help=' | get the storage index for a given name') - subparser.add_argument( + subparser.add_argument( 'name', type=str, help='the name to look up') - - # Print default help message, if no argument is given - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - args = parser.parse_args() - result = {} - - if args.action == 'getinfo': - result = client.getinfo() - - elif args.action == 'ping': - result = client.ping() - - elif args.action == 'preorder': - result = client.preorder( str(args.name), str(args.privatekey) ) - - elif args.action == 'register': - result = client.register( str(args.name), str(args.privatekey) ) - - elif args.action == 'update': - - txid = None - if args.txid is not None: - txid = str(args.txid) - - result = client.update( str(args.name), str(args.storage_index_json), str(args.privatekey), txid=txid ) - - elif args.action == 'transfer': - result = client.transfer( str(args.name), str(args.address), bool(args.keepdata), str(args.privatekey)) - - elif args.action == 'renew': - result = client.renew( str(args.name), str(args.privatekey) ) - - elif args.action == 'namespace_preorder': - result = client.namespace_preorder( str(args.namespace_id), str(args.privatekey) ) - - elif args.action == 'namespace_define': - result = client.namespace_define( str(args.namespace_id), int(args.lifetime), int(args.base_name_cost), float(args.cost_decay_rate), str(args.privatekey) ) - - elif args.action == 'namespace_begin': - result = client.namespace_begin( str(args.namespace_id), str(args.privatekey) ) - - elif args.action == 'put_mutable': - result = client.put_mutable( str(args.name), str(args.data_id), str(args.data), str(args.privatekey) ) - - elif args.action == 'put_immutable': - result = client.put_immutable( str(args.name), str(args.data), str(args.privatekey) ) - - elif args.action == 'get_mutable': - result = client.get_mutable( str(args.name), str(args.data_id) ) - - elif args.action == 'get_immutable': - result = client.get_immutable( str(args.name), str(args.data_hash) ) - - elif args.action == 'lookup': - result = client.lookup( str(args.name) ) - - elif args.action == 'getindex': - result = client.get_user_record( str(args.name) ) - - print_result( result ) + + # Print default help message, if no argument is given + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + args = parser.parse_args() + result = {} + + if args.action == 'getinfo': + result = client.getinfo() + + elif args.action == 'ping': + result = client.ping() + + elif args.action == 'preorder': + result = client.preorder(str(args.name), str(args.privatekey)) + + elif args.action == 'register': + result = client.register(str(args.name), str(args.privatekey)) + + elif args.action == 'update': + + txid = None + if args.txid is not None: + txid = str(args.txid) + + result = client.update(str(args.name), + str(args.storage_index_json), + str(args.privatekey), + txid=txid) + + elif args.action == 'transfer': + result = client.transfer(str(args.name), + str(args.address), + bool(args.keepdata), + str(args.privatekey)) + + elif args.action == 'renew': + result = client.renew(str(args.name), str(args.privatekey)) + + elif args.action == 'namespace_preorder': + result = client.namespace_preorder(str(args.namespace_id), + str(args.privatekey)) + + elif args.action == 'namespace_define': + result = client.namespace_define(str(args.namespace_id), + int(args.lifetime), + int(args.base_name_cost), + float(args.cost_decay_rate), + str(args.privatekey)) + + elif args.action == 'namespace_begin': + result = client.namespace_begin(str(args.namespace_id), + str(args.privatekey)) + + elif args.action == 'put_mutable': + result = client.put_mutable(str(args.name), + str(args.data_id), + str(args.data), + str(args.privatekey)) + + elif args.action == 'put_immutable': + result = client.put_immutable(str(args.name), + str(args.data), + str(args.privatekey)) + + elif args.action == 'get_mutable': + result = client.get_mutable(str(args.name), str(args.data_id)) + + elif args.action == 'get_immutable': + result = client.get_immutable(str(args.name), str(args.data_hash)) + + elif args.action == 'lookup': + result = client.lookup(str(args.name)) + + elif args.action == 'getindex': + result = client.get_user_record(str(args.name)) + + print_result(result) if __name__ == '__main__': run_cli() diff --git a/blockstore_client/client.py b/blockstore_client/client.py index 678c93989e..7cac3745a5 100644 --- a/blockstore_client/client.py +++ b/blockstore_client/client.py @@ -5,14 +5,14 @@ ~~~~~ copyright: (c) 2014 by Halfmoon Labs, Inc. copyright: (c) 2015 by Blockstack.org - + This file is part of Blockstore-client. - + Blockstore-client is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Blockstore-client is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the @@ -25,7 +25,7 @@ import sys import json import traceback -import types +import types import socket import uuid @@ -41,787 +41,785 @@ # default API endpoint proxy to blockstored default_proxy = None -# ancillary storage providers +# ancillary storage providers STORAGE_IMPL = None class BlockstoreRPCClient(object): """ Not-quite-JSONRPC client for Blockstore. - - Blockstore's not-quite-JSONRPC server expects a raw Netstring that encodes - a JSON object with a "method" string and an "args" list. It will ignore - "id" and "version", and will not accept keyword arguments. It also does + + Blockstore's not-quite-JSONRPC server expects a raw Netstring that encodes + a JSON object with a "method" string and an "args" list. It will ignore + "id" and "version", and will not accept keyword arguments. It also does not guarantee that the "result" and "error" keywords will be present. """ - + def __init__(self, server, port, max_rpc_len=MAX_RPC_LEN): - self.server = server - self.port = port + self.server = server + self.port = port self.sock = None - self.max_rpc_len = max_rpc_len - - + self.max_rpc_len = max_rpc_len + def __getattr__(self, key): try: return object.__getattr__(self, key) except AttributeError: return self.dispatch(key) - def socket(): return self.sock - - def default(self, *args ): + + def default(self, *args): self.params = args return self.request() def dispatch(self, key): self.method = key return self.default - - def ensure_connected( self ): + + def ensure_connected(self): if self.sock is None: - self.sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM, 0 ) - self.sock.connect( (self.server, self.port) ) - + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + self.sock.connect((self.server, self.port)) + return True - - + def request(self): - + self.ensure_connected() - request_id = str(uuid.uuid4()) - parameters = { 'id': request_id, 'method': self.method, 'params': self.params, 'version': '2.0' } - - data = json.dumps( parameters ) - + + data = json.dumps(parameters) data_netstring = str(len(data)) + ":" + data + "," - - # send request - self.sock.sendall( data_netstring ) - - # get response: expect comma-ending netstring - # get the length first + + # send request + self.sock.sendall(data_netstring) + + # get response: expect comma-ending netstring + # get the length first len_buf = "" + while True: - c = self.sock.recv(1) - if len(c) == 0: - # connection closed - self.sock.close() - raise Exception("Server closed remote connection") - - c = c[0] - - if c == ':': - break - else: - len_buf += c - buf_len = 0 - - # ensure it's an int - try: - buf_len = int(len_buf) - except Exception, e: - # invalid - self.sock.close() - raise Exception("Invalid response: invalid netstring length") - - # ensure it's not too big - if buf_len >= self.max_rpc_len: - self.sock.close() - raise Exception("Invalid response: message too big") - - # receive message - response = self.sock.recv( buf_len+1 ) - - # ensure that the message is terminated with a comma + c = self.sock.recv(1) + if len(c) == 0: + # connection closed + self.sock.close() + raise Exception("Server closed remote connection") + + c = c[0] + + if c == ':': + break + else: + len_buf += c + buf_len = 0 + + # ensure it's an int + try: + buf_len = int(len_buf) + except Exception, e: + # invalid + self.sock.close() + raise Exception("Invalid response: invalid netstring length") + + # ensure it's not too big + if buf_len >= self.max_rpc_len: + self.sock.close() + raise Exception("Invalid response: message too big") + + # receive message + response = self.sock.recv(buf_len+1) + + # ensure that the message is terminated with a comma if response[-1] != ',': - self.sock.close() - raise Exception("Invalid response: invalid netstring termination") - + self.sock.close() + raise Exception("Invalid response: invalid netstring termination") + # trim ',' response = response[:-1] - + # parse the response try: result = json.loads(response) except Exception, e: - - # try to clean up + + # try to clean up self.sock.close() raise Exception("Invalid response: not a JSON string") - - return result - -def session( server_host, server_port, username=None, password=None, set_global=True ): - """ - Create a JSONRPC API proxy to blockstore - """ - - global default_proxy - proxy = BlockstoreRPCClient( server_host, server_port ) + return result + + +def session(server_host, server_port, username=None, password=None, + set_global=True): + """ + Create a JSONRPC API proxy to blockstore + """ + + global default_proxy + proxy = BlockstoreRPCClient(server_host, server_port) + + if default_proxy is None and set_global: + default_proxy = proxy - if default_proxy is None and set_global: - default_proxy = proxy - - return proxy + return proxy def get_default_proxy(): - """ - Get the default API proxy to blockstore. - """ - global default_proxy - return default_proxy - - -def register_storage( storage_impl ): - """ - Register a storage implementation. - """ - rc = storage.register_storage( storage_impl ) - if rc: - storage_impl.storage_init() - - -def load_user( record_hash ): - """ - Load a user record from the storage implementation with the given hex string hash, - The user record hash should have been loaded from the blockchain, and thereby be the - authentic hash. - - Return the user record on success - Return None on error - """ - - user_json = storage.get_immutable_data( record_hash ) - if user_json is None: - log.error("Failed to load user record '%s'" % record_hash) - return None - - # verify integrity - user_record_hash = storage.get_data_hash( user_json ) - if user_record_hash != record_hash: - log.error("Profile hash mismatch: expected '%s', got '%s'" % record_hash, user_record_hash ) - return None - - user = user_db.parse_user( user_json ) - return user - - -def get_user_record( name, create=False ): - """ - Given the name of the user, look up the user's record hash, - and then get the record itself from storage. - - Returns a dict that contains the record, - or a dict with "error" defined and a message. - """ - - # find name record first - name_record = lookup( name ) - if len(name_record) == 0: - - return {"error": "No such name"} - - name_record = name_record[0] - if name_record.has_key('error'): - - # failed to look up - return name_record - - # sanity check - if not name_record.has_key('value_hash'): - - return {"error": "Name has no user record hash defined"} - - # is there a user record loaded? - if name_record['value_hash'] in [None, "null", ""]: - - # no user data - if not create: - return {"error": "No user data"} - - else: - # make an empty one and return that - user_resp = user_db.make_empty_user( name, name_record ) - return user_resp - - # get record - user_record_hash = name_record['value_hash'] - user_resp = load_user( user_record_hash ) - - if user_resp is None: - - # no user record data + """ + Get the default API proxy to blockstore. + """ + global default_proxy + return default_proxy + + +def register_storage(storage_impl): + """ + Register a storage implementation. + """ + rc = storage.register_storage(storage_impl) + if rc: + storage_impl.storage_init() + + +def load_user(record_hash): + """ + Load a user record from the storage implementation with the given hex string hash, + The user record hash should have been loaded from the blockchain, and thereby be the + authentic hash. + + Return the user record on success + Return None on error + """ + + user_json = storage.get_immutable_data(record_hash) + if user_json is None: + log.error("Failed to load user record '%s'" % record_hash) + return None + + # verify integrity + user_record_hash = storage.get_data_hash(user_json) + if user_record_hash != record_hash: + log.error("Profile hash mismatch: expected '%s', got '%s'" % record_hash, user_record_hash) + return None + + user = user_db.parse_user(user_json) + return user + + +def get_user_record(name, create=False): + """ + Given the name of the user, look up the user's record hash, + and then get the record itself from storage. + + Returns a dict that contains the record, + or a dict with "error" defined and a message. + """ + + # find name record first + name_record = lookup(name) + if len(name_record) == 0: + return {"error": "No such name"} + + name_record = name_record[0] + if 'error' in name_record: + + # failed to look up + return name_record + + # sanity check + if 'value_hash' not in name_record: + + return {"error": "Name has no user record hash defined"} + + # is there a user record loaded? + if name_record['value_hash'] in [None, "null", ""]: + + # no user data + if not create: + return {"error": "No user data"} + else: + # make an empty one and return that + user_resp = user_db.make_empty_user(name, name_record) + return user_resp + + # get record + user_record_hash = name_record['value_hash'] + user_resp = load_user(user_record_hash) + + if user_resp is None: + + # no user record data return {"error": "User data could not be loaded from storage"} - - return user_resp - - -def store_user( user, txid ): - """ - Store JSON user record data to the immutable storage providers, synchronously. - - Return (True, hash(user)) on success - Return (False, hash(user)) on failure - """ - - username = user_db.name( user ) - - # serialize - user_json = None - try: - user_json = user_db.serialize_user( user ) - except Exception, e: - log.error("Failed to serialize '%s'" % user ) - return False - - data_hash = storage.get_data_hash( user_json ) - result = storage.put_immutable_data( user_json, txid, replication_strategy=storage.REPLICATE_ALL ) - - rc = None - if result is None: - rc = False - else: - rc = True - - return (rc, data_hash ) - - -def getinfo( proxy=None ): - """ - getinfo - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.getinfo() - - -def ping( proxy=None ): - """ - ping - """ - - if proxy is None: - proxy = get_default_proxy() - - response = proxy.ping() - return response - - -def lookup( name, proxy=None ): - """ - lookup - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.lookup( name ) - - -def preorder( name, privatekey, proxy=None ): - """ - preorder - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.preorder( name, privatekey ) - - -def register( name, privatekey, proxy=None ): - """ - register - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.register( name, privatekey ) - - -def update( name, user_json_or_hash, privatekey, txid=None, proxy=None ): - """ - update - - Optionally supply a txid. The reason for doing so is to try to replicate user - data to new storage systems, or to recover from a transient error encountered - earlier. - """ - - if proxy is None: - proxy = get_default_proxy() - - user_record_hash = None - user_data = None - - # 160-bit hex string? - if len(user_json_or_hash) == 40 and len(user_json_or_hash.translate( None, "0123456789ABCDEFabcdef")) == 0: - - user_record_hash = user_json_or_hash.lower() - - else: - - # user record json. hash it - user_data = user_db.parse_user( user_json_or_hash ) - if user_data is None: - return {'error': 'Invalid user record JSON'} - - user_record_hash = pybitcoin.hash.hex_hash160( user_db.serialize_user( user_data ) ) - - result = {} - - # no transaction: go put one - if txid is None: - - print user_json_or_hash - print user_data - print user_record_hash - - result = proxy.update( name, user_record_hash, privatekey ) - result = result[0] - - if result.has_key('error'): - - # failed - return result - - if not result.has_key('transaction_hash'): - - # failed - result['error'] = "No transaction hash given" - return result - - txid = result['transaction_hash'] - - else: - - # embed the txid into the result nevertheless - result['transaction_hash'] = txid - - # store new user data - rc = False - data_hash = None - if user_data is not None: - - rc, data_hash = store_user( user_data, txid ) - - if rc: - result['status'] = rc - result['value_hash'] = data_hash - result["transaction_hash"] = txid - - else: - result['error'] = "Failed to store updated user record" - - return result - - -def transfer( name, address, keep_data, privatekey, proxy=None ): - """ - transfer - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.transfer( name, address, keep_data, privatekey ) - - -def renew( name, privatekey, proxy=None ): - """ - renew - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.renew( name, privatekey ) - - -def revoke( name, privatekey, proxy=None ): - """ - revoke - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.revoke( name, privatekey ) - - -def namespace_preorder( namespace_id, privatekey, proxy=None ): - """ - namespace preorder - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.namespace_preorder( namespace_id, privatekey ) - - -def namespace_define( namespace_id, lifetime, base_name_cost, cost_decay_rate, privatekey, proxy=None ): - """ - namesapce_define - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.namespace_define( namespace_id, lifetime, base_name_cost, cost_decay_rate, privatekey ) - - -def namespace_begin( namespace_id, privatekey, proxy=None ): - """ - namespace_begin - """ - - if proxy is None: - proxy = get_default_proxy() - - return proxy.namespace_begin( namespace_id, privatekey ) - - -def get_immutable( name, data_key ): - """ - get_immutable - """ - - user = get_user_record( name ) - if user.has_key('error'): - - # no user data - return {'error': "Unable to load user record: %s" % user['error']} - - if not user_db.has_immutable_data( user, data_key ): - - # no data - return {'error': 'Profile has no such immutable data'} - - data = storage.get_immutable_data( data_key ) - if data is None: - - # no data - return {'error': 'No immutable data found'} - - return {'data': data} - - - -def get_mutable( name, data_id, nonce_min=None, nonce_max=None, nonce_check=None ): - """ - get_mutable - """ - - user = get_user_record( name ) - if user.has_key('error'): - - # no user data - return {'error': "Unable to load user record: %s" % user['error']} - - # find the mutable data ID - data_route = user_db.get_mutable_data_route( user, data_id ) - if data_route is None: - - # no data - return {'error': 'No such route'} - - # go and fetch the data - data = storage.get_mutable_data( data_route, nonce_min=nonce_min, nonce_max=nonce_max, nonce_check=nonce_check ) - if data is None: - - # no data - return {'error': 'No mutable data found'} - - # include the route - data['route'] = data_route - return data - - -def put_immutable( name, data, privatekey, txid=None, proxy=None ): - """ - put_immutable - - Optionally include a txid from the user record update, in order to retry a failed - data replication (in which case, this txid corresponds to the succeeded name - update operation). This is to avoid needing to pay for each replication retry. - """ - - if proxy is None: - global default_proxy - proxy = default_proxy - - # need to put the transaction ID into the data record we put - user = get_user_record( name, create=True ) - if user.has_key('error'): - - # no user data - return {'error': "Unable to load user record: %s" % user['error']} - - data_hash = pybitcoin.hash.hex_hash160( data ) - user_db.add_immutable_data( user, data_hash ) - - user_json = user_db.serialize_user( user ) - if user_json is None: - raise Exception("BUG: failed to serialize user record") - - if txid is None: - - # haven't updated the user record yet. Do so now. - # put the new user record hash - update_result = update( name, user_json, privatekey, proxy=proxy ) - if update_result.has_key('error'): - - # failed to replicate user record - # NOTE: result will have the txid in it; pass it as txid to try again! - return update_result - - txid = update_result['transaction_hash'] - - result = { 'transaction_hash': txid } - - # replicate the data - rc = storage.put_immutable_data( data, txid ) - if not rc: - result['error'] = 'Failed to store immutable data' - return result - - else: - result['status'] = True - return result - - -def put_mutable( name, data_id, data_text, privatekey, proxy=None, create=True, txid=None, nonce=None, make_nonce=None, replication_strategy=storage.REPLICATE_ALL ): - """ - put_mutable - - ** Consistency ** - - nonce, if given, is the nonce to include in the data. - make_nonce, if given, is a callback that takes the data_id and data_text and generates a nonce to be included in the data record uploaded. - If nonce is not given, but make_nonce is, then make_nonce will be used to generate the nonce. - If neither nonce nor make_nonce are given, the mutable data (if it already exists) is fetched, and the nonce is calculated as existing['nonce'] + 1. - - ** Durability ** - - replication_strategy defines how rigorous blockstore is when it comes to replicating data to its storage providers. - If set to REPLICATE_ALL (the default), then this method only succeeds if we successfully replicate to *every* storage provider. - If set to REPLICATE_ANY, then this method succeeds if we successfully replicate to one storage provider. - Storage providers are contacted in the order they are registered. - """ - - if proxy is None: - proxy = get_default_proxy() - - result = {} - user = get_user_record( name, create=create ) - if user.has_key('error'): - - return {'error': "Unable to load user record: %s" % user['error']} - - route = None - exists = True - - # do we have a route for this data yet? - if not user_db.has_mutable_data_route( user, data_id ): - - if not create: - # won't create; expect it to exist - return {'error': 'No such route'} - - - # need to put one - urls = storage.make_mutable_urls( data_id ) - if len(urls) == 0: - return {"error": "No routes constructed"} - - writer_pubkey = pybitcointools.privkey_to_pubkey( privatekey ) - - route = storage.mutable_data_route( data_id, data_urls, writer_pubkey=writer_pubkey ) - - user_db.add_mutable_data_route( user, route ) - - user_json = user_db.serialize_user( user ) - - # update the user record with the new route - update_result = update( name, user_json, privatekey, txid=txid, proxy=proxy ) - if update_result.has_key('error'): - - # update failed; caller should try again - return update_result - - txid = update_result['transaction_hash'] - - exists = False - - else: - - route = user_db.get_mutable_data_route( user, data_id ) - if route is None: - - return {"error": "No such route"} - - - if nonce is None: - - # need a nonce - if make_nonce is not None: - nonce = make_nonce( data_id, data_text ) - - if exists: - - existing_data = get_mutable( name, data_id ) - if existing_data is None: - - result['error'] = "No nonce calculated" - result['transaction_hash'] = txid - return result - - nonce = existing_data['nonce'] - nonce += 1 - - else: - nonce = 1 - - - # upload the data - data = storage.mutable_data( data_id, data_text, nonce, privkey=privatekey ) - if data is None: - return {"error": "Failed to generate data record"} - - data_json = parsing.json_stable_serialize( data ) - - store_rc = storage.put_mutable_data( data_id, nonce, data['sig'], data_json ) - if not store_rc: - result['error'] = "Failed to store mutable data" - - else: - result['status'] = True - - result['transaction_hash'] = txid - return result - - -def delete_immutable( name, data_key, privatekey, proxy=None, txid=None ): - """ - delete_immutable - """ - - if proxy is None: - proxy = get_default_proxy() - - result = {} - user = get_user_record( name ) - if user.has_key('error'): - - # no user data - return {'error': "Unable to load user record: %s" % user['error']} - - # does the user record have this data? - if not user_db.has_immutable_data( user, data_key ): - - # already deleted - return {'status': True} - - # remove hash from the user record and update it - user_db.remove_immutable_data( user, data_key ) - - user_json = user_db.serialize_user( user ) - - update_result = update( name, user_json, privatekey, txid=txid, proxy=proxy ) - if update_result.has_key('error'): - - # update failed; caller should try again - return update_result - - txid = update_result['transaction_hash'] - - # remove the data itself data - delete_result = storage.delete_immutable_data( data_key, txid ) - if delete_result: - - result['status'] = True - - else: - - # be sure to give back the update transaction hash, so this call can be retried - result['error'] = 'Failed to delete immutable data' - - result['transaction_hash'] = txid - return result - - - -def delete_mutable( name, data_id, privatekey, proxy=default_proxy, txid=None, route=None ): - """ - delete_mutable - """ - - if proxy is None: - proxy = get_default_proxy() - - result = {} - user = get_user_record( name ) - if user.has_key('error'): - - # no user data - return {'error': "Unable to load user record: %s" % user['error']} - - # does the user have a route to this data? - if not user_db.has_mutable_data_route( user, data_id ) and txid is None: - - # nope--we're good - return {'status': True} - - # blow away the data - storage_rc = storage.delete_mutable_data( data_id, privatekey ) - if not storage_rc: - result['error'] = "Failed to delete mutable data" - return result - - # remove the route from the user record - user_db.remove_mutable_data_route( user, data_id ) - user_json = user_db.serialize_user( user ) - - # update the user record - update_status = update( name, user_json, privatekey, txid=txid, proxy=proxy ) - if update_status.has_key('error'): - - # failed; caller should try again - return update_status - - if txid is None: - txid = update_status['transaction_hash'] - - # blow away the route - if route is None: - route = user_db.get_mutable_data_route( user, data_id ) - - route_hash = storage.get_mutable_data_route_hash( route ) - storage_rc = storage.delete_immutable_data( route_hash, txid ) - if not storage_rc: - - result['error'] = "Failed to delete immutable data route" - result['route'] = route - - else: - result['status'] = True - - return result - - \ No newline at end of file + + return user_resp + + +def store_user(user, txid): + """ + Store JSON user record data to the immutable storage providers, synchronously. + + Return (True, hash(user)) on success + Return (False, hash(user)) on failure + """ + + username = user_db.name(user) + + # serialize + user_json = None + try: + user_json = user_db.serialize_user(user) + except Exception, e: + log.error("Failed to serialize '%s'" % user) + return False + + data_hash = storage.get_data_hash(user_json) + result = storage.put_immutable_data(user_json, txid, replication_strategy=storage.REPLICATE_ALL) + + rc = None + if result is None: + rc = False + else: + rc = True + + return (rc, data_hash) + + +def getinfo(proxy=None): + """ + getinfo + """ + + resp = {} + + if proxy is None: + proxy = get_default_proxy() + + try: + resp = proxy.getinfo() + except Exception, e: + resp['error'] = str(e) + return resp + + +def ping(proxy=None): + """ + ping + """ + + if proxy is None: + proxy = get_default_proxy() + + response = proxy.ping() + return response + + +def lookup(name, proxy=None): + """ + lookup + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.lookup(name) + + +def preorder(name, privatekey, proxy=None): + """ + preorder + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.preorder(name, privatekey) + + +def register(name, privatekey, proxy=None): + """ + register + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.register(name, privatekey) + + +def update(name, user_json_or_hash, privatekey, txid=None, proxy=None): + """ + update + + Optionally supply a txid. The reason for doing so is to try to replicate user + data to new storage systems, or to recover from a transient error encountered + earlier. + """ + + if proxy is None: + proxy = get_default_proxy() + + user_record_hash = None + user_data = None + + # 160-bit hex string? + if len(user_json_or_hash) == 40 and len(user_json_or_hash.translate(None, "0123456789ABCDEFabcdef")) == 0: + + user_record_hash = user_json_or_hash.lower() + else: + + # user record json. hash it + user_data = user_db.parse_user(user_json_or_hash) + if user_data is None: + return {'error': 'Invalid user record JSON'} + + user_record_hash = pybitcoin.hash.hex_hash160(user_db.serialize_user(user_data)) + + result = {} + + # no transaction: go put one + if txid is None: + + print user_json_or_hash + print user_data + print user_record_hash + + result = proxy.update(name, user_record_hash, privatekey) + result = result[0] + + if 'error' in result: + # failed + return result + + if 'transaction_hash' not in result: + # failed + result['error'] = "No transaction hash given" + return result + + txid = result['transaction_hash'] + + else: + + # embed the txid into the result nevertheless + result['transaction_hash'] = txid + + # store new user data + rc = False + data_hash = None + if user_data is not None: + + rc, data_hash = store_user(user_data, txid) + if rc: + result['status'] = rc + result['value_hash'] = data_hash + result["transaction_hash"] = txid + + else: + result['error'] = "Failed to store updated user record" + + return result + + +def transfer(name, address, keep_data, privatekey, proxy=None): + """ + transfer + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.transfer(name, address, keep_data, privatekey) + + +def renew(name, privatekey, proxy=None): + """ + renew + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.renew(name, privatekey) + + +def revoke(name, privatekey, proxy=None): + """ + revoke + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.revoke(name, privatekey) + + +def namespace_preorder(namespace_id, privatekey, proxy=None): + """ + namespace preorder + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.namespace_preorder(namespace_id, privatekey) + + +def namespace_define(namespace_id, lifetime, base_name_cost, cost_decay_rate, + privatekey, proxy=None): + """ + namesapce_define + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.namespace_define(namespace_id, lifetime, base_name_cost, + cost_decay_rate, privatekey) + + +def namespace_begin(namespace_id, privatekey, proxy=None): + """ + namespace_begin + """ + + if proxy is None: + proxy = get_default_proxy() + + return proxy.namespace_begin(namespace_id, privatekey) + + +def get_immutable(name, data_key): + """ + get_immutable + """ + + user = get_user_record(name) + if 'error' in user: + + # no user data + return {'error': "Unable to load user record: %s" % user['error']} + + if not user_db.has_immutable_data(user, data_key): + + # no data + return {'error': 'Profile has no such immutable data'} + + data = storage.get_immutable_data(data_key) + if data is None: + + # no data + return {'error': 'No immutable data found'} + + return {'data': data} + + +def get_mutable(name, data_id, nonce_min=None, nonce_max=None, nonce_check=None): + """ + get_mutable + """ + + user = get_user_record(name) + if 'error' in user: + + # no user data + return {'error': "Unable to load user record: %s" % user['error']} + + # find the mutable data ID + data_route = user_db.get_mutable_data_route(user, data_id) + if data_route is None: + + # no data + return {'error': 'No such route'} + + # go and fetch the data + data = storage.get_mutable_data(data_route, nonce_min=nonce_min, + nonce_max=nonce_max, + nonce_check=nonce_check) + if data is None: + + # no data + return {'error': 'No mutable data found'} + + # include the route + data['route'] = data_route + return data + + +def put_immutable(name, data, privatekey, txid=None, proxy=None): + """ + put_immutable + + Optionally include a txid from the user record update, in order to retry a failed + data replication (in which case, this txid corresponds to the succeeded name + update operation). This is to avoid needing to pay for each replication retry. + """ + + if proxy is None: + global default_proxy + proxy = default_proxy + + # need to put the transaction ID into the data record we put + user = get_user_record(name, create=True) + if 'error' in user: + + # no user data + return {'error': "Unable to load user record: %s" % user['error']} + + data_hash = pybitcoin.hash.hex_hash160(data) + user_db.add_immutable_data(user, data_hash) + + user_json = user_db.serialize_user(user) + if user_json is None: + raise Exception("BUG: failed to serialize user record") + + if txid is None: + + # haven't updated the user record yet. Do so now. + # put the new user record hash + update_result = update(name, user_json, privatekey, proxy=proxy) + if 'error' in update_result: + + # failed to replicate user record + # NOTE: result will have the txid in it; pass it as txid to try again! + return update_result + + txid = update_result['transaction_hash'] + + result = {'transaction_hash': txid} + + # replicate the data + rc = storage.put_immutable_data(data, txid) + if not rc: + result['error'] = 'Failed to store immutable data' + return result + + else: + result['status'] = True + return result + + +def put_mutable(name, data_id, data_text, privatekey, proxy=None, create=True, + txid=None, nonce=None, make_nonce=None, + replication_strategy=storage.REPLICATE_ALL): + """ + put_mutable + + ** Consistency ** + + nonce, if given, is the nonce to include in the data. + make_nonce, if given, is a callback that takes the data_id and data_text and generates a nonce to be included in the data record uploaded. + If nonce is not given, but make_nonce is, then make_nonce will be used to generate the nonce. + If neither nonce nor make_nonce are given, the mutable data (if it already exists) is fetched, and the nonce is calculated as existing['nonce'] + 1. + + ** Durability ** + + replication_strategy defines how rigorous blockstore is when it comes to replicating data to its storage providers. + If set to REPLICATE_ALL (the default), then this method only succeeds if we successfully replicate to *every* storage provider. + If set to REPLICATE_ANY, then this method succeeds if we successfully replicate to one storage provider. + Storage providers are contacted in the order they are registered. + """ + + if proxy is None: + proxy = get_default_proxy() + + result = {} + user = get_user_record(name, create=create) + if 'error' in user: + + return {'error': "Unable to load user record: %s" % user['error']} + + route = None + exists = True + + # do we have a route for this data yet? + if not user_db.has_mutable_data_route(user, data_id): + + if not create: + # won't create; expect it to exist + return {'error': 'No such route'} + + # need to put one + urls = storage.make_mutable_urls(data_id) + if len(urls) == 0: + return {"error": "No routes constructed"} + + writer_pubkey = pybitcointools.privkey_to_pubkey(privatekey) + + route = storage.mutable_data_route(data_id, data_urls, + writer_pubkey=writer_pubkey) + + user_db.add_mutable_data_route(user, route) + + user_json = user_db.serialize_user(user) + + # update the user record with the new route + update_result = update(name, user_json, privatekey, txid=txid, proxy=proxy) + if 'error' in update_result: + + # update failed; caller should try again + return update_result + + txid = update_result['transaction_hash'] + + exists = False + + else: + + route = user_db.get_mutable_data_route(user, data_id) + if route is None: + + return {"error": "No such route"} + + if nonce is None: + + # need a nonce + if make_nonce is not None: + nonce = make_nonce(data_id, data_text) + + if exists: + + existing_data = get_mutable(name, data_id) + if existing_data is None: + + result['error'] = "No nonce calculated" + result['transaction_hash'] = txid + return result + + nonce = existing_data['nonce'] + nonce += 1 + + else: + nonce = 1 + + # upload the data + data = storage.mutable_data(data_id, data_text, nonce, privkey=privatekey) + if data is None: + return {"error": "Failed to generate data record"} + + data_json = parsing.json_stable_serialize(data) + + store_rc = storage.put_mutable_data(data_id, nonce, data['sig'], data_json) + if not store_rc: + result['error'] = "Failed to store mutable data" + + else: + result['status'] = True + + result['transaction_hash'] = txid + return result + + +def delete_immutable(name, data_key, privatekey, proxy=None, txid=None): + """ + delete_immutable + """ + + if proxy is None: + proxy = get_default_proxy() + + result = {} + user = get_user_record(name) + if 'error' in user: + + # no user data + return {'error': "Unable to load user record: %s" % user['error']} + + # does the user record have this data? + if not user_db.has_immutable_data(user, data_key): + + # already deleted + return {'status': True} + + # remove hash from the user record and update it + user_db.remove_immutable_data(user, data_key) + + user_json = user_db.serialize_user(user) + + update_result = update(name, user_json, privatekey, txid=txid, proxy=proxy) + if 'error' in update_result: + + # update failed; caller should try again + return update_result + + txid = update_result['transaction_hash'] + + # remove the data itself data + delete_result = storage.delete_immutable_data(data_key, txid) + if delete_result: + + result['status'] = True + + else: + + # be sure to give back the update transaction hash, so this call can be retried + result['error'] = 'Failed to delete immutable data' + + result['transaction_hash'] = txid + return result + + +def delete_mutable(name, data_id, privatekey, proxy=default_proxy, txid=None, + route=None): + """ + delete_mutable + """ + + if proxy is None: + proxy = get_default_proxy() + + result = {} + user = get_user_record(name) + if 'error' in user: + + # no user data + return {'error': "Unable to load user record: %s" % user['error']} + + # does the user have a route to this data? + if not user_db.has_mutable_data_route(user, data_id) and txid is None: + + # nope--we're good + return {'status': True} + + # blow away the data + storage_rc = storage.delete_mutable_data(data_id, privatekey) + if not storage_rc: + result['error'] = "Failed to delete mutable data" + return result + + # remove the route from the user record + user_db.remove_mutable_data_route(user, data_id) + user_json = user_db.serialize_user(user) + + # update the user record + update_status = update(name, user_json, privatekey, txid=txid, + proxy=proxy) + if 'error' in update_status: + + # failed; caller should try again + return update_status + + if txid is None: + txid = update_status['transaction_hash'] + + # blow away the route + if route is None: + route = user_db.get_mutable_data_route(user, data_id) + + route_hash = storage.get_mutable_data_route_hash(route) + storage_rc = storage.delete_immutable_data(route_hash, txid) + if not storage_rc: + + result['error'] = "Failed to delete immutable data route" + result['route'] = route + + else: + result['status'] = True + + return result From 8e945928fe5d9bb8906fa2199a374d2e1bffc6ba Mon Sep 17 00:00:00 2001 From: Muneeb Ali Date: Wed, 12 Aug 2015 20:05:13 -0400 Subject: [PATCH 2/5] Reorg files This commit changes the following things: * organization of the folders: 1) blockstore instead of blockstore_client, so that we can do from blockstore.client import x 2) blockstore.plugins are renamed to blockstore.drivers (thinking of them as disk drivers) --- {blockstore_client => blockstore}/__init__.py | 0 {blockstore_client => blockstore}/client.py | 0 {blockstore_client => blockstore}/config.py | 0 {blockstore_client/plugins => blockstore/drivers}/README.md | 0 {blockstore_client/plugins => blockstore/drivers}/__init__.py | 0 {blockstore_client/plugins => blockstore/drivers}/dht.py | 0 {blockstore_client/plugins => blockstore/drivers}/disk.py | 0 {blockstore_client => blockstore}/parsing.py | 0 {blockstore_client => blockstore}/schemas.py | 0 {blockstore_client => blockstore}/storage.py | 0 {blockstore_client => blockstore}/user.py | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename {blockstore_client => blockstore}/__init__.py (100%) rename {blockstore_client => blockstore}/client.py (100%) rename {blockstore_client => blockstore}/config.py (100%) rename {blockstore_client/plugins => blockstore/drivers}/README.md (100%) rename {blockstore_client/plugins => blockstore/drivers}/__init__.py (100%) rename {blockstore_client/plugins => blockstore/drivers}/dht.py (100%) rename {blockstore_client/plugins => blockstore/drivers}/disk.py (100%) rename {blockstore_client => blockstore}/parsing.py (100%) rename {blockstore_client => blockstore}/schemas.py (100%) rename {blockstore_client => blockstore}/storage.py (100%) rename {blockstore_client => blockstore}/user.py (100%) diff --git a/blockstore_client/__init__.py b/blockstore/__init__.py similarity index 100% rename from blockstore_client/__init__.py rename to blockstore/__init__.py diff --git a/blockstore_client/client.py b/blockstore/client.py similarity index 100% rename from blockstore_client/client.py rename to blockstore/client.py diff --git a/blockstore_client/config.py b/blockstore/config.py similarity index 100% rename from blockstore_client/config.py rename to blockstore/config.py diff --git a/blockstore_client/plugins/README.md b/blockstore/drivers/README.md similarity index 100% rename from blockstore_client/plugins/README.md rename to blockstore/drivers/README.md diff --git a/blockstore_client/plugins/__init__.py b/blockstore/drivers/__init__.py similarity index 100% rename from blockstore_client/plugins/__init__.py rename to blockstore/drivers/__init__.py diff --git a/blockstore_client/plugins/dht.py b/blockstore/drivers/dht.py similarity index 100% rename from blockstore_client/plugins/dht.py rename to blockstore/drivers/dht.py diff --git a/blockstore_client/plugins/disk.py b/blockstore/drivers/disk.py similarity index 100% rename from blockstore_client/plugins/disk.py rename to blockstore/drivers/disk.py diff --git a/blockstore_client/parsing.py b/blockstore/parsing.py similarity index 100% rename from blockstore_client/parsing.py rename to blockstore/parsing.py diff --git a/blockstore_client/schemas.py b/blockstore/schemas.py similarity index 100% rename from blockstore_client/schemas.py rename to blockstore/schemas.py diff --git a/blockstore_client/storage.py b/blockstore/storage.py similarity index 100% rename from blockstore_client/storage.py rename to blockstore/storage.py diff --git a/blockstore_client/user.py b/blockstore/user.py similarity index 100% rename from blockstore_client/user.py rename to blockstore/user.py From a851b3d777ccc26f8561ca2e3838d8806ae355c7 Mon Sep 17 00:00:00 2001 From: Muneeb Ali Date: Wed, 12 Aug 2015 20:07:13 -0400 Subject: [PATCH 3/5] added list of cli commands --- README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ca9cec9b2b..8c2784b08f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,27 @@ Blockstore Client ================= -This package contains the client library for Blockstore. It implements an API that encapsulates creating and managing blockchain IDs, blockchain ID namespaces, and storage records associated with them. +This package contains the client library for Blockstore. It talks to blockstored and provides an interface for creating and managing blockchain IDs, blockchain ID namespaces, and storage records associated with them. + +## blockstore-cli + +The blockstore-cli currently supports the following commands: + +* get_immutable +* get_mutable +* getindex +* getinfo +* lookup +* namespace_begin +* namespace_define +* namespace_preorder +* ping +* preorder +* put_immutable +* put_mutable +* register +* renew +* transfer +* update + + From 4a658c09db3115b756b1089cad3d4ce5cc8a672f Mon Sep 17 00:00:00 2001 From: Muneeb Ali Date: Wed, 12 Aug 2015 20:07:37 -0400 Subject: [PATCH 4/5] sorted display of cli commands and added a helper function for sorting future commands --- bin/blockstore-cli | 174 +++++++++++++++++++++++++-------------------- 1 file changed, 96 insertions(+), 78 deletions(-) diff --git a/bin/blockstore-cli b/bin/blockstore-cli index d52d8a1880..0bb6be359c 100755 --- a/bin/blockstore-cli +++ b/bin/blockstore-cli @@ -34,8 +34,8 @@ parent_dir = os.path.abspath(current_dir + "/../") sys.path.insert(0, parent_dir) -from blockstore_client import config, client, schemas, parsing, user -from blockstore_client import plugins +from blockstore import config, client, schemas, parsing, user +from blockstore import drivers log = config.log @@ -50,6 +50,21 @@ def print_result(json_str): print pretty_dump(json_str) +def get_sorted_commands(): + """ when adding new commands to the parser, use this function to + check the correct sorted order + """ + + command_list = ['getinfo', 'ping', 'preorder', 'register', 'update', + 'transfer', 'renew', 'namespace_preorder', + 'namespace_define', 'namespace_begin', 'put_mutable', + 'put_immutable', 'get_mutable', 'get_immutable', + 'lookup', 'getindex'] + + for x in sorted(command_list): + print x + + def run_cli(): """ run cli """ @@ -57,7 +72,7 @@ def run_cli(): # TODO: read config file proxy = client.session(config.BLOCKSTORED_SERVER, config.BLOCKSTORED_PORT) - client.register_storage(plugins.disk) + client.register_storage(drivers.disk) parser = argparse.ArgumentParser( description='Blockstore Cli version {}'.format(config.VERSION)) @@ -78,82 +93,54 @@ def run_cli(): dest='action', help='the action to be taken') - subparser = subparsers.add_parser( - 'getinfo', - help='get basic info from the blockstored server') - - subparser = subparsers.add_parser( - 'ping', - help='check if the blockstored server is up') - # ------------------------------------ + # start commands subparser = subparsers.add_parser( - 'preorder', - help=' | preorder a name') + 'get_immutable', + help=' | Get immutable data from the storage providers, and verify that the named user wrote it.') subparser.add_argument( 'name', type=str, - help='the name that you want to preorder') + help='the name of the user') subparser.add_argument( - 'privatekey', type=str, - help='the private key of the Bitcoin address that will own the name') + 'hash', type=str, + help='the hash of the data') # ------------------------------------ subparser = subparsers.add_parser( - 'register', - help=' | register/claim a name') + 'get_mutable', + help=' | Get mutable data from the storage providers, and verify that the named user wrote it.') subparser.add_argument( 'name', type=str, - help='the name that you want to register/claim') + help='the name associated with the data') subparser.add_argument( - 'privatekey', type=str, - help='the private key of the Bitcoin address that will own the name') + 'data_id', type=str, + help='the unchanging identifier for this data') # ------------------------------------ subparser = subparsers.add_parser( - 'update', - help=' | update storage index data and store it into the storage providers') + 'getindex', + help=' | get the storage index for a given name') subparser.add_argument( 'name', type=str, - help='the name that you want to update') - subparser.add_argument( - 'storage_index_json', type=str, - help='the JSON-encoded storage index to associate with the name') - subparser.add_argument( - 'privatekey', type=str, - help='the privatekey of the owner Bitcoin address') + help='the name to look up') # ------------------------------------ subparser = subparsers.add_parser( - 'transfer', - help='
| transfer a name') - subparser.add_argument( - 'name', type=str, - help='the name that you want to register/claim') - subparser.add_argument( - 'address', type=str, - help='the new owner Bitcoin address') - subparser.add_argument( - 'keepdata', type=bool, - help='whether or not the storage index should remain associated with the name') - subparser.add_argument( - 'privatekey', type=str, - help='the privatekey of the owner Bitcoin address') + 'getinfo', + help='get basic info from the blockstored server') # ------------------------------------ subparser = subparsers.add_parser( - 'renew', - help=' | renew a name') + 'lookup', + help=' | get the name record for a given name') subparser.add_argument( 'name', type=str, - help='the name that you want to renew') - subparser.add_argument( - 'privatekey', type=str, - help='the privatekey of the owner Bitcoin address') + help='the name to look up') # ------------------------------------ subparser = subparsers.add_parser( - 'namespace_preorder', - help=' | preorder a namespace, in order to claim the namespace ID and begin populating it.') + 'namespace_begin', + help=' | begin the namespace, completing its definition and opening it for registration by other parties.') subparser.add_argument( 'namespace_id', type=str, help='the human-readable namespace identifier') @@ -183,8 +170,8 @@ def run_cli(): # ------------------------------------ subparser = subparsers.add_parser( - 'namespace_begin', - help=' | begin the namespace, completing its definition and opening it for registration by other parties.') + 'namespace_preorder', + help=' | preorder a namespace, in order to claim the namespace ID and begin populating it.') subparser.add_argument( 'namespace_id', type=str, help='the human-readable namespace identifier') @@ -194,20 +181,19 @@ def run_cli(): # ------------------------------------ subparser = subparsers.add_parser( - 'put_mutable', - help=' [] | Store mutable data into the storage providers, creating it if it does not exist.') + 'ping', + help='check if the blockstored server is up') + + # ------------------------------------ + subparser = subparsers.add_parser( + 'preorder', + help=' | preorder a name') subparser.add_argument( 'name', type=str, - help='the name that owns this data') - subparser.add_argument( - 'data_id', type=str, - help='the unchanging identifier for this data') - subparser.add_argument( - 'data', type=str, - help='the data to store') + help='the name that you want to preorder') subparser.add_argument( 'privatekey', type=str, - help='the private key assocated with the name') + help='the private key of the Bitcoin address that will own the name') # ------------------------------------ subparser = subparsers.add_parser( @@ -225,41 +211,73 @@ def run_cli(): # ------------------------------------ subparser = subparsers.add_parser( - 'get_mutable', - help=' | Get mutable data from the storage providers, and verify that the named user wrote it.') + 'put_mutable', + help=' [] | Store mutable data into the storage providers, creating it if it does not exist.') subparser.add_argument( 'name', type=str, - help='the name associated with the data') + help='the name that owns this data') subparser.add_argument( 'data_id', type=str, help='the unchanging identifier for this data') + subparser.add_argument( + 'data', type=str, + help='the data to store') + subparser.add_argument( + 'privatekey', type=str, + help='the private key assocated with the name') # ------------------------------------ subparser = subparsers.add_parser( - 'get_immutable', - help=' | Get immutable data from the storage providers, and verify that the named user wrote it.') + 'register', + help=' | register/claim a name') subparser.add_argument( 'name', type=str, - help='the name of the user') + help='the name that you want to register/claim') subparser.add_argument( - 'hash', type=str, - help='the hash of the data') + 'privatekey', type=str, + help='the private key of the Bitcoin address that will own the name') # ------------------------------------ subparser = subparsers.add_parser( - 'lookup', - help=' | get the name record for a given name') + 'renew', + help=' | renew a name') subparser.add_argument( 'name', type=str, - help='the name to look up') + help='the name that you want to renew') + subparser.add_argument( + 'privatekey', type=str, + help='the privatekey of the owner Bitcoin address') # ------------------------------------ subparser = subparsers.add_parser( - 'getindex', - help=' | get the storage index for a given name') + 'transfer', + help='
| transfer a name') subparser.add_argument( 'name', type=str, - help='the name to look up') + help='the name that you want to register/claim') + subparser.add_argument( + 'address', type=str, + help='the new owner Bitcoin address') + subparser.add_argument( + 'keepdata', type=bool, + help='whether or not the storage index should remain associated with the name') + subparser.add_argument( + 'privatekey', type=str, + help='the privatekey of the owner Bitcoin address') + + # ------------------------------------ + subparser = subparsers.add_parser( + 'update', + help=' | update storage index data and store it into the storage providers') + subparser.add_argument( + 'name', type=str, + help='the name that you want to update') + subparser.add_argument( + 'storage_index_json', type=str, + help='the JSON-encoded storage index to associate with the name') + subparser.add_argument( + 'privatekey', type=str, + help='the privatekey of the owner Bitcoin address') # Print default help message, if no argument is given if len(sys.argv) == 1: From ed4855f6a733006055868e8167065d0e88a1a52e Mon Sep 17 00:00:00 2001 From: Muneeb Ali Date: Wed, 12 Aug 2015 20:07:57 -0400 Subject: [PATCH 5/5] PEP8 + error checking * makes some PEP8 changes (mostly 4 line spaces instead of 3) * adds error displays for ping and getinfo command (users generally tend to use these two to test connectivity to the server) --- blockstore/__init__.py | 20 +++++++------ blockstore/client.py | 23 ++++++++++++--- blockstore/config.py | 13 ++++----- blockstore/parsing.py | 65 +++++++++++++++++++++--------------------- 4 files changed, 70 insertions(+), 51 deletions(-) diff --git a/blockstore/__init__.py b/blockstore/__init__.py index ff60126b89..eba01b307e 100644 --- a/blockstore/__init__.py +++ b/blockstore/__init__.py @@ -5,14 +5,14 @@ ~~~~~ copyright: (c) 2014 by Halfmoon Labs, Inc. copyright: (c) 2015 by Blockstack.org - + This file is part of Blockstore-client. - + Blockstore-client is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Blockstore-client is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the @@ -21,12 +21,16 @@ along with Blockstore-client. If not, see . """ -import client -import config +import client +import config import schemas import parsing import user -import plugins +import drivers -from client import getinfo, get_user_record, ping, lookup, preorder, update, transfer, renew, revoke, namespace_preorder, namespace_define, namespace_begin, \ - get_immutable, get_mutable, put_immutable, put_mutable, delete_immutable, delete_mutable, session, register_storage \ No newline at end of file +from client import getinfo, get_user_record, ping, lookup +from client import preorder, update, transfer, renew, revoke +from client import namespace_preorder, namespace_define, namespace_begin +from client import get_immutable, get_mutable +from client import put_immutable, put_mutable, delete_immutable, delete_mutable +from client import session, register_storage \ No newline at end of file diff --git a/blockstore/client.py b/blockstore/client.py index 7cac3745a5..5a8f58c00d 100644 --- a/blockstore/client.py +++ b/blockstore/client.py @@ -177,6 +177,7 @@ def get_default_proxy(): Get the default API proxy to blockstore. """ global default_proxy + return default_proxy @@ -304,8 +305,9 @@ def getinfo(proxy=None): try: resp = proxy.getinfo() - except Exception, e: + except Exception as e: resp['error'] = str(e) + return resp @@ -314,11 +316,17 @@ def ping(proxy=None): ping """ + resp = {} + if proxy is None: proxy = get_default_proxy() - response = proxy.ping() - return response + try: + resp = proxy.ping() + except Exception as e: + resp['error'] = str(e) + + return resp def lookup(name, proxy=None): @@ -337,10 +345,17 @@ def preorder(name, privatekey, proxy=None): preorder """ + resp = {} + if proxy is None: proxy = get_default_proxy() - return proxy.preorder(name, privatekey) + try: + resp = proxy.preorder(name, privatekey) + except Exception as e: + resp['error'] = str(e) + + return resp def register(name, privatekey, proxy=None): diff --git a/blockstore/config.py b/blockstore/config.py index c61d2235f8..562d1b0b48 100644 --- a/blockstore/config.py +++ b/blockstore/config.py @@ -5,14 +5,14 @@ ~~~~~ copyright: (c) 2014 by Halfmoon Labs, Inc. copyright: (c) 2015 by Blockstack.org - + This file is part of Blockstore-client. - + Blockstore-client is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Blockstore-client is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the @@ -21,20 +21,19 @@ along with Blockstore-client. If not, see . """ +import logging + BLOCKSTORED_PORT = 6264 BLOCKSTORED_SERVER = "127.0.0.1" DEBUG = True VERSION = "v0.01-beta" MAX_RPC_LEN = 1024 * 1024 * 1024 -import logging - log = logging.getLogger() log.setLevel(logging.DEBUG if DEBUG else logging.INFO) console = logging.StreamHandler() console.setLevel(logging.DEBUG if DEBUG else logging.INFO) log_format = ('[%(levelname)s] [%(module)s:%(lineno)d] %(message)s' if DEBUG else '%(message)s') -formatter = logging.Formatter( log_format ) +formatter = logging.Formatter(log_format) console.setFormatter(formatter) log.addHandler(console) - diff --git a/blockstore/parsing.py b/blockstore/parsing.py index 4f9a623fad..55d86539e8 100644 --- a/blockstore/parsing.py +++ b/blockstore/parsing.py @@ -5,14 +5,14 @@ ~~~~~ copyright: (c) 2014 by Halfmoon Labs, Inc. copyright: (c) 2015 by Blockstack.org - + This file is part of Blockstore-client. - + Blockstore-client is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Blockstore-client is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the @@ -21,32 +21,33 @@ along with Blockstore-client. If not, see . """ -import json - -def json_stable_serialize( json_data ): - """ - Serialize a dict to JSON, but ensure that key/value pairs are serialized - in a predictable, stable, total order. - """ - - if isinstance( json_data, list ) or isinstance( json_data, tuple ): - json_serialized_list = [] - for json_element in json_data: - json_serialized_list.append( json_stable_serialize( json_element ) ) - - return "[" + ", ".join( json_serialized_list ) + "]" - - elif isinstance( json_data, dict ): - json_serialized_dict = {} - for key in json_data.keys(): - json_serialized_dict[key] = json_stable_serialize( json_data[key] ) - - key_order = [k for k in json_serialized_dict.keys()] - key_order.sort() - - return "{" + ", ".join( ['"%s": %s' % (k, json_serialized_dict[k]) for k in key_order] ) + "}" - - elif isinstance( json_data, str ) or isinstance( json_data, unicode ): - return '"' + json_data + '"' - - return '"' + str(json_data) + '"' \ No newline at end of file +import json + + +def json_stable_serialize(json_data): + """ + Serialize a dict to JSON, but ensure that key/value pairs are serialized + in a predictable, stable, total order. + """ + + if isinstance(json_data, list) or isinstance(json_data, tuple): + json_serialized_list = [] + for json_element in json_data: + json_serialized_list.append(json_stable_serialize(json_element)) + + return "[" + ", ".join(json_serialized_list) + "]" + + elif isinstance(json_data, dict): + json_serialized_dict = {} + for key in json_data.keys(): + json_serialized_dict[key] = json_stable_serialize(json_data[key]) + + key_order = [k for k in json_serialized_dict.keys()] + key_order.sort() + + return "{" + ", ".join(['"%s": %s' % (k, json_serialized_dict[k]) for k in key_order]) + "}" + + elif isinstance(json_data, str) or isinstance(json_data, unicode): + return '"' + json_data + '"' + + return '"' + str(json_data) + '"'