BQ DB-API: Use unicode for string type in params

- improved docstring formatting
- used namedtuple for column descriptions

bigquery/google/cloud/bigquery/dbapi/__init__.py

@@ -26,31 +26,31 @@
    or deprecation policy.
 """
 
-from google.cloud.bigquery.dbapi.connection import connect  # noqa
-from google.cloud.bigquery.dbapi.connection import Connection  # noqa
-from google.cloud.bigquery.dbapi.cursor import Cursor  # noqa
-from google.cloud.bigquery.dbapi.exceptions import Warning  # noqa
-from google.cloud.bigquery.dbapi.exceptions import Error  # noqa
-from google.cloud.bigquery.dbapi.exceptions import InterfaceError  # noqa
-from google.cloud.bigquery.dbapi.exceptions import DatabaseError  # noqa
-from google.cloud.bigquery.dbapi.exceptions import DataError  # noqa
-from google.cloud.bigquery.dbapi.exceptions import OperationalError  # noqa
-from google.cloud.bigquery.dbapi.exceptions import IntegrityError  # noqa
-from google.cloud.bigquery.dbapi.exceptions import InternalError  # noqa
-from google.cloud.bigquery.dbapi.exceptions import ProgrammingError  # noqa
-from google.cloud.bigquery.dbapi.exceptions import NotSupportedError  # noqa
-from google.cloud.bigquery.dbapi.types import Binary  # noqa
-from google.cloud.bigquery.dbapi.types import Date  # noqa
-from google.cloud.bigquery.dbapi.types import DateFromTicks  # noqa
-from google.cloud.bigquery.dbapi.types import Time  # noqa
-from google.cloud.bigquery.dbapi.types import TimeFromTicks  # noqa
-from google.cloud.bigquery.dbapi.types import Timestamp  # noqa
-from google.cloud.bigquery.dbapi.types import TimestampFromTicks  # noqa
-from google.cloud.bigquery.dbapi.types import BINARY  # noqa
-from google.cloud.bigquery.dbapi.types import DATETIME  # noqa
-from google.cloud.bigquery.dbapi.types import NUMBER  # noqa
-from google.cloud.bigquery.dbapi.types import ROWID  # noqa
-from google.cloud.bigquery.dbapi.types import STRING  # noqa
+from google.cloud.bigquery.dbapi.connection import connect
+from google.cloud.bigquery.dbapi.connection import Connection
+from google.cloud.bigquery.dbapi.cursor import Cursor
+from google.cloud.bigquery.dbapi.exceptions import Warning
+from google.cloud.bigquery.dbapi.exceptions import Error
+from google.cloud.bigquery.dbapi.exceptions import InterfaceError
+from google.cloud.bigquery.dbapi.exceptions import DatabaseError
+from google.cloud.bigquery.dbapi.exceptions import DataError
+from google.cloud.bigquery.dbapi.exceptions import OperationalError
+from google.cloud.bigquery.dbapi.exceptions import IntegrityError
+from google.cloud.bigquery.dbapi.exceptions import InternalError
+from google.cloud.bigquery.dbapi.exceptions import ProgrammingError
+from google.cloud.bigquery.dbapi.exceptions import NotSupportedError
+from google.cloud.bigquery.dbapi.types import Binary
+from google.cloud.bigquery.dbapi.types import Date
+from google.cloud.bigquery.dbapi.types import DateFromTicks
+from google.cloud.bigquery.dbapi.types import Time
+from google.cloud.bigquery.dbapi.types import TimeFromTicks
+from google.cloud.bigquery.dbapi.types import Timestamp
+from google.cloud.bigquery.dbapi.types import TimestampFromTicks
+from google.cloud.bigquery.dbapi.types import BINARY
+from google.cloud.bigquery.dbapi.types import DATETIME
+from google.cloud.bigquery.dbapi.types import NUMBER
+from google.cloud.bigquery.dbapi.types import ROWID
+from google.cloud.bigquery.dbapi.types import STRING
 
 
 apilevel = "2.0"
@@ -59,3 +59,12 @@
 threadsafety = 1
 
 paramstyle = "pyformat"
+
+__all__ = [
+    'apilevel', 'threadsafety', 'paramstyle', 'connect', 'Connection',
+    'Cursor', 'Warning', 'Error', 'InterfaceError', 'DatabaseError',
+    'DataError', 'OperationalError', 'IntegrityError', 'InternalError',
+    'ProgrammingError', 'NotSupportedError', 'Binary', 'Date', 'DateFromTicks',
+    'Time', 'TimeFromTicks', 'Timestamp', 'TimestampFromTicks', 'BINARY',
+    'DATETIME', 'NUMBER', 'ROWID', 'STRING',
+]

bigquery/google/cloud/bigquery/dbapi/_helpers.py

@@ -34,30 +34,28 @@ def wait_for_job(job):
         job.reload()
         if job.state == 'DONE':
             if job.error_result:
-                # TODO: raise a more specific exception, based on the error.
-                # See: https://cloud.google.com/bigquery/troubleshooting-errors
                 raise exceptions.DatabaseError(job.errors)
             return
         time.sleep(1)
 
 
-def scalar_to_query_parameter(name=None, value=None):
+def scalar_to_query_parameter(value, name=None):
     """Convert a scalar value into a query parameter.
 
-    Note: the bytes type cannot be distinguished from a string in Python 2.
+    Note: You must use the unicode type for string parameters in Python 2.
 
     Raises a :class:`~ google.cloud.bigquery.dbapi.exceptions.ProgrammingError`
     if the type cannot be determined.
 
     For more information about BigQuery data types, see:
     https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
 
-    :type: str
-    :param name: Optional name of the query parameter.
-
     :type: any
     :param value: A scalar value to convert into a query parameter.
 
+    :type: str
+    :param name: Optional name of the query parameter.
+
     :rtype: :class:`~google.cloud.bigquery.ScalarQueryParameter`
     """
     parameter_type = None
@@ -68,9 +66,9 @@ def scalar_to_query_parameter(name=None, value=None):
         parameter_type = 'INT64'
     elif isinstance(value, numbers.Real):
         parameter_type = 'FLOAT64'
-    elif isinstance(value, six.string_types):
+    elif isinstance(value, six.text_type):
         parameter_type = 'STRING'
-    elif isinstance(value, bytes):
+    elif isinstance(value, six.binary_type):
         parameter_type = 'BYTES'
     elif isinstance(value, datetime.datetime):
         parameter_type = 'TIMESTAMP' if value.tzinfo else 'DATETIME'
@@ -88,43 +86,38 @@ def scalar_to_query_parameter(name=None, value=None):
 def to_query_parameters_list(parameters):
     """Converts a list of parameter values into query parameters.
 
-    :type: list
-    :param parameters: List of query parameter values.
+    :type: Sequence[Any]
+    :param parameters: Sequence of query parameter values.
 
     :rtype:
         list of :class:`~google.cloud.bigquery._helpers.AbstractQueryParameter`
     """
-    query_parameters = []
-
-    for value in parameters:
-        query_parameters.append(scalar_to_query_parameter(value=value))
-
-    return query_parameters
+    return [scalar_to_query_parameter(value) for value in parameters]
 
 
 def to_query_parameters_dict(parameters):
     """Converts a dictionary of parameter values into query parameters.
 
-    :type: dict
+    :type: Mapping[str, Any]
     :param parameters: Dictionary of query parameter values.
 
     :rtype:
         list of :class:`~google.cloud.bigquery._helpers.AbstractQueryParameter`
     """
-    query_parameters = []
-
-    for name in parameters:
-        value = parameters[name]
-        query_parameters.append(scalar_to_query_parameter(name, value))
-
-    return query_parameters
+    return [
+        scalar_to_query_parameter(value, name=name)
+        for name, value
+        in six.iteritems(parameters)]
 
 
 def to_query_parameters(parameters):
     """Converts DB-API parameter values into query parameters.
 
-    :type: dict or list
-    :param parameters: Optional dictionary or list of query parameter values.
+    STRUCT/RECORD and REPEATED type parameters are not yet supported.
+    https://github.com/GoogleCloudPlatform/google-cloud-python/issues/3524
+
+    :type: Mapping[str, Any] or Sequence[Any]
+    :param parameters: A dictionary or sequence of query parameter values.
 
     :rtype:
         list of :class:`~google.cloud.bigquery._helpers.AbstractQueryParameter`

bigquery/google/cloud/bigquery/dbapi/cursor.py

@@ -23,7 +23,16 @@
 from google.cloud.bigquery.dbapi import exceptions
 
 
-_ARRAYSIZE_DEFAULT = 20
+# Per PEP 249: A 7-item sequence containing information describing one result
+# column. The first two items (name and type_code) are mandatory, the other
+# five are optional and are set to None if no meaningful values can be
+# provided.
+Column = collections.namedtuple(
+    'Column',
+    [
+        'name', 'type_code', 'display_size', 'internal_size', 'precision',
+        'scale', 'null_ok',
+    ])
 
 
 class Cursor(object):
@@ -35,8 +44,13 @@ class Cursor(object):
     def __init__(self, connection):
         self.connection = connection
         self.description = None
+        # Per PEP 249: The attribute is -1 in case no .execute*() has been
+        # performed on the cursor or the rowcount of the last operation
+        # cannot be determined by the interface.
         self.rowcount = -1
-        self.arraysize = None
+        # Per PEP 249: The arraysize attribute defaults to 1, meaning to fetch
+        # a single row at a time.
+        self.arraysize = 1
         self._query_data = None
         self._page_token = None
         self._has_fetched_all_rows = True
@@ -47,23 +61,23 @@ def close(self):
     def _set_description(self, schema):
         """Set description from schema.
 
-        :type: list of :class:`~google.cloud.bigquery.schema.SchemaField`
+        :type: Sequence[:class:`~google.cloud.bigquery.schema.SchemaField`]
+        :param schema: A description of fields in the schema.
         """
         if schema is None:
             self.description = None
             return
 
-        desc = []
-        for field in schema:
-            desc.append(tuple([
+        self.description = tuple([
+            Column(
                 field.name,
                 field.field_type,
                 None,
                 None,
                 None,
                 None,
-                field.mode == 'NULLABLE']))
-        self.description = tuple(desc)
+                field.mode == 'NULLABLE')
+            for field in schema])
 
     def _set_rowcount(self, query_results):
         """Set the rowcount from query results.
@@ -86,13 +100,16 @@ def _set_rowcount(self, query_results):
         self.rowcount = total_rows
 
     def _format_operation_list(self, operation, parameters):
-        """Formats parameters in operation in way BigQuery expects.
+        """Formats parameters in operation in the way BigQuery expects.
+
+        The input operation will be a query like ``SELECT %s`` and the output
+        will be a query like ``SELECT ?``.
 
         :type: str
         :param operation: A Google BigQuery query string.
 
-        :type: list
-        :param parameters: List parameter values.
+        :type: Sequence[Any]
+        :param parameters: Sequence of parameter values.
         """
         formatted_params = ['?' for _ in parameters]
 
@@ -102,12 +119,15 @@ def _format_operation_list(self, operation, parameters):
             raise exceptions.ProgrammingError(ex)
 
     def _format_operation_dict(self, operation, parameters):
-        """Formats parameters in operation in way BigQuery expects.
+        """Formats parameters in operation in the way BigQuery expects.
+
+        The input operation will be a query like ``SELECT %(namedparam)s`` and
+        the output will be a query like ``SELECT @namedparam``.
 
         :type: str
         :param operation: A Google BigQuery query string.
 
-        :type: dict
+        :type: Mapping[str, Any]
         :param parameters: Dictionary of parameter values.
         """
         formatted_params = {}
@@ -128,7 +148,7 @@ def _format_operation(self, operation, parameters=None):
         :type: str
         :param operation: A Google BigQuery query string.
 
-        :type: dict or list
+        :type: Mapping[str, Any] or Sequence[Any]
         :param parameters: Optional parameter values.
         """
         if parameters is None:
@@ -145,7 +165,7 @@ def execute(self, operation, parameters=None):
         :type: str
         :param operation: A Google BigQuery query string.
 
-        :type: dict or list
+        :type: Mapping[str, Any] or Sequence[Any]
         :param parameters: Optional dictionary or sequence of parameter values.
         """
         self._query_results = None
@@ -191,7 +211,7 @@ def executemany(self, operation, seq_of_parameters):
         :type: str
         :param operation: A Google BigQuery query string.
 
-        :type: list
+        :type: Sequence[Mapping[str, Any] or Sequence[Any]]
         :param parameters: Sequence of many sets of parameter values.
         """
         for parameters in seq_of_parameters:
@@ -218,23 +238,22 @@ def fetchmany(self, size=None):
         """Fetch multiple results from the last ``execute*()`` call.
 
         Note that the size parameter is not used for the request/response size.
-        Use ``arraysize()`` before calling ``execute()`` to set the batch size.
+        Set the ``arraysize`` attribute before calling ``execute()`` to set the
+        batch size.
 
         :type: int
         :param size:
-            Optional maximum number of rows to return. Defaults to the
-            arraysize attribute.
+            (Optional) Maximum number of rows to return. Defaults to the
+            ``arraysize`` property value.
 
-        :rtype: tuple
+        :rtype: Sequence[tuple]
         :returns: A list of rows.
         """
         if self._query_data is None:
             raise exceptions.InterfaceError(
                 'No query results: execute() must be called before fetch.')
         if size is None:
             size = self.arraysize
-        if size is None:
-            size = _ARRAYSIZE_DEFAULT
 
         rows = []
         for row in self._query_data:
@@ -246,7 +265,7 @@ def fetchmany(self, size=None):
     def fetchall(self):
         """Fetch all remaining results from the last ``execute*()`` call.
 
-        :rtype: list of tuples
+        :rtype: Sequence[tuple]
         :returns: A list of all the rows in the results.
         """
         if self._query_data is None:

bigquery/google/cloud/bigquery/dbapi/types.py

@@ -22,13 +22,15 @@
 
 import datetime
 
+import six
+
 
 Date = datetime.date
 Time = datetime.time
 Timestamp = datetime.datetime
 DateFromTicks = datetime.date.fromtimestamp
 TimestampFromTicks = datetime.datetime.fromtimestamp
-Binary = bytes
+Binary = six.binary_type
 
 
 def TimeFromTicks(ticks, tz=None):