From b7d37b856f9240688fe11b65f72e2c9cc42b1267 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 25 May 2018 02:50:26 +0100 Subject: [PATCH 01/14] Added support for cftime types --- holoviews/core/data/xarray.py | 11 +++++++++++ holoviews/plotting/bokeh/element.py | 4 +++- holoviews/plotting/bokeh/plot.py | 25 +++++++++++++++++++++---- holoviews/plotting/bokeh/util.py | 9 +++++---- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/holoviews/core/data/xarray.py b/holoviews/core/data/xarray.py index ae7394f33c..cac57018a7 100644 --- a/holoviews/core/data/xarray.py +++ b/holoviews/core/data/xarray.py @@ -12,6 +12,17 @@ from .grid import GridInterface from .interface import Interface, DataError, dask_array_module +try: + import cftime + util.datetime_types += ( + cftime._cftime.DatetimeGregorian, + cftime._cftime.Datetime360Day, + cftime._cftime.DatetimeJulian, + cftime._cftime.DatetimeNoLeap, + cftime._cftime.DatetimeProlepticGregorian) +except: + pass + class XArrayInterface(GridInterface): diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py index b0e3364771..210243893b 100644 --- a/holoviews/plotting/bokeh/element.py +++ b/holoviews/plotting/bokeh/element.py @@ -40,7 +40,7 @@ from .util import ( bokeh_version, decode_bytes, get_tab_title, glyph_order, py2js_tickformatter, recursive_model_update, theme_attr_json, - cds_column_replace, hold_policy, match_dim_specs + cds_column_replace, hold_policy, match_dim_specs, date_to_integer ) @@ -634,6 +634,8 @@ def _update_range(self, axis_range, low, high, factors, invert, shared, log, str if reset_supported: updates['reset_end'] = updates['end'] for k, (old, new) in updates.items(): + if isinstance(new, util.datetime_types): + new = date_to_integer(new) axis_range.update(**{k:new}) if streaming and not k.startswith('reset_'): axis_range.trigger(k, old, new) diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py index ab1d675d6d..66bcdb9bd1 100644 --- a/holoviews/plotting/bokeh/plot.py +++ b/holoviews/plotting/bokeh/plot.py @@ -11,12 +11,13 @@ from bokeh.models import (ColumnDataSource, Column, Row, Div) from bokeh.models.widgets import Panel, Tabs from bokeh.plotting.helpers import _known_tools as known_tools +from bokeh.util.serialization import convert_datetime_array from ...core import (OrderedDict, Store, AdjointLayout, NdLayout, Layout, Empty, GridSpace, HoloMap, Element, DynamicMap) from ...core.options import SkipRendering from ...core.util import (basestring, wrap_tuple, unique_iterator, - get_method_owner, wrap_tuple_streams) + get_method_owner, datetime_types, wrap_tuple_streams) from ...streams import Stream from ..links import Link from ..plot import (DimensionedPlot, GenericCompositePlot, GenericLayoutPlot, @@ -25,7 +26,7 @@ from .callbacks import LinkCallback from .util import (layout_padding, pad_plots, filter_toolboxes, make_axis, update_shared_sources, empty_plot, decode_bytes, - theme_attr_json, cds_column_replace) + theme_attr_json, cds_column_replace, date_to_integer) TOOLS = {name: tool if isinstance(tool, basestring) else type(tool()) for name, tool in known_tools.items()} @@ -226,10 +227,26 @@ def _init_datasource(self, data): """ Initializes a data source to be passed into the bokeh glyph. """ - data = {k: decode_bytes(vs) for k, vs in data.items()} + data = self._postprocess_data(data) return ColumnDataSource(data=data) + def _postprocess_data(self, data): + """ + Applies necessary type transformation to the data before + it is set on a ColumnDataSource. + """ + new_data = {} + for k, values in data.items(): + values = decode_bytes(values) # Bytes need decoding to strings + + # Certain datetime types need to be converted + if len(values) and isinstance(values[0], datetime_types): + values = np.array([date_to_integer(v) for v in values]) + new_data[k] = values + return new_data + + def _update_datasource(self, source, data): """ Update datasource with data for a new frame. @@ -237,7 +254,7 @@ def _update_datasource(self, source, data): if not self.document: return - data = {k: decode_bytes(vs) for k, vs in data.items()} + data = self._postprocess_data(data) empty = all(len(v) == 0 for v in data.values()) if (self.streaming and self.streaming[0].data is self.current_frame.data and self._stream_data and not empty): diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py index 145bd9a75e..5c95c3f6dd 100644 --- a/holoviews/plotting/bokeh/util.py +++ b/holoviews/plotting/bokeh/util.py @@ -3,6 +3,7 @@ import re import time import sys +import calendar import datetime as dt from collections import defaultdict @@ -542,10 +543,10 @@ def date_to_integer(date): """ if isinstance(date, np.datetime64): date = dt64_to_dt(date) - if pd and isinstance(date, pd.Timestamp): - dt_int = date.timestamp()*1000 - elif isinstance(date, (dt.datetime, dt.date)): - dt_int = time.mktime(date.timetuple())*1000 + elif pd and isinstance(date, pd.Timestamp): + date = date.to_pydatetime() + if hasattr(date, 'timetuple'): + dt_int = calendar.timegm(date.timetuple())*1000 else: raise ValueError('Datetime type not recognized') return dt_int From 5ec1e7d9a82d30220b9a346f97313170b0b0f5e9 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 3 Dec 2018 18:27:11 +0000 Subject: [PATCH 02/14] Minor fix so NdMapping allows cftimes types as keys --- holoviews/core/ndmapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/core/ndmapping.py b/holoviews/core/ndmapping.py index 067b44068d..121899fde4 100644 --- a/holoviews/core/ndmapping.py +++ b/holoviews/core/ndmapping.py @@ -650,7 +650,7 @@ def __getitem__(self, indexslice): return self.data[()] elif indexslice in [Ellipsis, ()]: return self - elif Ellipsis in wrap_tuple(indexslice): + elif any(Ellipsis is sl for sl in wrap_tuple(indexslice)): indexslice = process_ellipses(self, indexslice) map_slice, data_slice = self._split_index(indexslice) From 88bebf399eca1b99e395f3bf8807b7163e6d78bc Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Wed, 5 Dec 2018 13:02:17 +0000 Subject: [PATCH 03/14] Add support for nc_time_axis in matplotlib backend --- holoviews/core/data/xarray.py | 8 +++++--- holoviews/plotting/mpl/util.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/holoviews/core/data/xarray.py b/holoviews/core/data/xarray.py index cac57018a7..03fa9f2284 100644 --- a/holoviews/core/data/xarray.py +++ b/holoviews/core/data/xarray.py @@ -14,14 +14,16 @@ try: import cftime - util.datetime_types += ( + cftime_types = ( cftime._cftime.DatetimeGregorian, cftime._cftime.Datetime360Day, cftime._cftime.DatetimeJulian, cftime._cftime.DatetimeNoLeap, - cftime._cftime.DatetimeProlepticGregorian) + cftime._cftime.DatetimeProlepticGregorian + ) + util.datetime_types += cftime_types except: - pass + cftime_types = () class XArrayInterface(GridInterface): diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py index 6b37961211..c4be4d6ea1 100644 --- a/holoviews/plotting/mpl/util.py +++ b/holoviews/plotting/mpl/util.py @@ -5,6 +5,7 @@ import numpy as np import matplotlib +from matplotlib import units as munits from matplotlib import ticker from matplotlib.colors import cnames from matplotlib.lines import Line2D @@ -14,6 +15,8 @@ from matplotlib.rcsetup import ( validate_capstyle, validate_fontsize, validate_fonttype, validate_hatch, validate_joinstyle) + +from ...core.data.xarray import cftime_types from ...core.util import LooseVersion, _getargspec, basestring, is_number from ...element import Raster, RGB, Polygons from ..util import COLOR_ALIASES, RGB_HEX_REGEX @@ -334,3 +337,34 @@ def polygons_to_path_patches(element): subpath.append(PathPatch(Path(vertices, codes))) mpl_paths.append(subpath) return mpl_paths + +try: + if cftime_types: + from nc_time_axis import NetCDFTimeConverter, CalendarDateTime + else: + import matplotlib.dates import DateConverter + NetCDFTimeConverter = DateConverter + nc_axis_available = True +except: + import matplotlib.dates import DateConverter + NetCDFTimeConverter = DateConverter + nc_axis_available = False + +class CFTimeConverter(NetCDFTimeConverter): + + @classmethod + def convert(cls, value, unit, axis): + if not nc_axis_available: + raise ValueError('In order to display cftime types with ' + 'matplotlib install the nc_time_axis ' + 'library using pip or from conda-forge ' + 'using:\n\tconda install -c conda-forge ' + 'nc_time_axis') + if isinstance(value, cftime_types): + value = CalendarDateTime(cftime.datetime(*value.timetuple()[0:6]), value.calendar) + elif isinstance(value, np.ndarray): + value = np.array([CalendarDateTime(cftime.datetime(*v.timetuple()[0:6]), v.calendar) for v in value]) + return super(CFTimeConverter, cls).convert(value, unit, axis) + +for cft in cftime_types: + munits.registry[cft] = CFTimeConverter() From 8b71267d2628d1b8f712862f208595d9fb6df160 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Wed, 5 Dec 2018 16:41:34 +0000 Subject: [PATCH 04/14] Fixed flakes --- holoviews/plotting/bokeh/plot.py | 1 - holoviews/plotting/mpl/util.py | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py index 66bcdb9bd1..640c3e43b1 100644 --- a/holoviews/plotting/bokeh/plot.py +++ b/holoviews/plotting/bokeh/plot.py @@ -11,7 +11,6 @@ from bokeh.models import (ColumnDataSource, Column, Row, Div) from bokeh.models.widgets import Panel, Tabs from bokeh.plotting.helpers import _known_tools as known_tools -from bokeh.util.serialization import convert_datetime_array from ...core import (OrderedDict, Store, AdjointLayout, NdLayout, Layout, Empty, GridSpace, HoloMap, Element, DynamicMap) diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py index c4be4d6ea1..1847cbec15 100644 --- a/holoviews/plotting/mpl/util.py +++ b/holoviews/plotting/mpl/util.py @@ -342,15 +342,19 @@ def polygons_to_path_patches(element): if cftime_types: from nc_time_axis import NetCDFTimeConverter, CalendarDateTime else: - import matplotlib.dates import DateConverter + from matplotlib.dates import DateConverter NetCDFTimeConverter = DateConverter nc_axis_available = True except: - import matplotlib.dates import DateConverter + from matplotlib.dates import DateConverter NetCDFTimeConverter = DateConverter nc_axis_available = False + class CFTimeConverter(NetCDFTimeConverter): + """ + Defines conversions for cftime types by extending nc_time_axis. + """ @classmethod def convert(cls, value, unit, axis): @@ -366,5 +370,6 @@ def convert(cls, value, unit, axis): value = np.array([CalendarDateTime(cftime.datetime(*v.timetuple()[0:6]), v.calendar) for v in value]) return super(CFTimeConverter, cls).convert(value, unit, axis) + for cft in cftime_types: munits.registry[cft] = CFTimeConverter() From eb3f0c8032da0b1ddc6eb707ed057e8d4555a708 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Wed, 5 Dec 2018 17:16:35 +0000 Subject: [PATCH 05/14] Further cftime fix --- holoviews/plotting/mpl/util.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py index 1847cbec15..45d612cc5d 100644 --- a/holoviews/plotting/mpl/util.py +++ b/holoviews/plotting/mpl/util.py @@ -339,11 +339,8 @@ def polygons_to_path_patches(element): return mpl_paths try: - if cftime_types: - from nc_time_axis import NetCDFTimeConverter, CalendarDateTime - else: - from matplotlib.dates import DateConverter - NetCDFTimeConverter = DateConverter + import cftime + from nc_time_axis import NetCDFTimeConverter, CalendarDateTime nc_axis_available = True except: from matplotlib.dates import DateConverter From 155cd82322e5a09b5e9950d4ca60599f6202711d Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 02:34:04 +0000 Subject: [PATCH 06/14] Improved handling of cftimes in bokeh --- holoviews/plotting/bokeh/element.py | 7 ++-- holoviews/plotting/bokeh/plot.py | 31 +++++++++++------ holoviews/plotting/bokeh/util.py | 52 ++++++++++++++++++++++++----- holoviews/plotting/mpl/element.py | 6 ++-- 4 files changed, 72 insertions(+), 24 deletions(-) diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py index 210243893b..39a1f2cfa8 100644 --- a/holoviews/plotting/bokeh/element.py +++ b/holoviews/plotting/bokeh/element.py @@ -25,6 +25,7 @@ from bokeh.plotting.helpers import _known_tools as known_tools from ...core import DynamicMap, CompositeOverlay, Element, Dimension +from ...core.data.xarray import cftime_types from ...core.options import abbreviated_exception, SkipRendering from ...core import util from ...element import Graph, VectorField, Path, Contours @@ -606,7 +607,9 @@ def _update_ranges(self, element, ranges): def _update_range(self, axis_range, low, high, factors, invert, shared, log, streaming=False): if isinstance(axis_range, (Range1d, DataRange1d)) and self.apply_ranges: - if (low == high and low is not None): + if isinstance(low, cftime_types): + pass + elif (low == high and low is not None): if isinstance(low, util.datetime_types): offset = np.timedelta64(500, 'ms') low -= offset @@ -634,7 +637,7 @@ def _update_range(self, axis_range, low, high, factors, invert, shared, log, str if reset_supported: updates['reset_end'] = updates['end'] for k, (old, new) in updates.items(): - if isinstance(new, util.datetime_types): + if isinstance(new, cftime_types): new = date_to_integer(new) axis_range.update(**{k:new}) if streaming and not k.startswith('reset_'): diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py index 640c3e43b1..3659a5f4ad 100644 --- a/holoviews/plotting/bokeh/plot.py +++ b/holoviews/plotting/bokeh/plot.py @@ -12,20 +12,29 @@ from bokeh.models.widgets import Panel, Tabs from bokeh.plotting.helpers import _known_tools as known_tools -from ...core import (OrderedDict, Store, AdjointLayout, NdLayout, Layout, - Empty, GridSpace, HoloMap, Element, DynamicMap) +from ...core import ( + OrderedDict, Store, AdjointLayout, NdLayout, Layout, Empty, + GridSpace, HoloMap, Element, DynamicMap +) +from ...core.data.xarray import cftime_types from ...core.options import SkipRendering -from ...core.util import (basestring, wrap_tuple, unique_iterator, - get_method_owner, datetime_types, wrap_tuple_streams) +from ...core.util import ( + basestring, wrap_tuple, unique_iterator, get_method_owner, + datetime_types, wrap_tuple_streams) + from ...streams import Stream from ..links import Link -from ..plot import (DimensionedPlot, GenericCompositePlot, GenericLayoutPlot, - GenericElementPlot, GenericOverlayPlot) +from ..plot import ( + DimensionedPlot, GenericCompositePlot, GenericLayoutPlot, + GenericElementPlot, GenericOverlayPlot +) from ..util import attach_streams, displayable, collate from .callbacks import LinkCallback -from .util import (layout_padding, pad_plots, filter_toolboxes, make_axis, - update_shared_sources, empty_plot, decode_bytes, - theme_attr_json, cds_column_replace, date_to_integer) +from .util import ( + layout_padding, pad_plots, filter_toolboxes, make_axis, + update_shared_sources, empty_plot, decode_bytes, theme_attr_json, + cds_column_replace, cftime_to_timestamp +) TOOLS = {name: tool if isinstance(tool, basestring) else type(tool()) for name, tool in known_tools.items()} @@ -240,8 +249,8 @@ def _postprocess_data(self, data): values = decode_bytes(values) # Bytes need decoding to strings # Certain datetime types need to be converted - if len(values) and isinstance(values[0], datetime_types): - values = np.array([date_to_integer(v) for v in values]) + if len(values) and isinstance(values[0], cftime_types): + values = cftime_to_timestamp(values) new_data[k] = values return new_data diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py index 5c95c3f6dd..9e224cb75e 100644 --- a/holoviews/plotting/bokeh/util.py +++ b/holoviews/plotting/bokeh/util.py @@ -31,6 +31,7 @@ except: Chart = type(None) # Create stub for isinstance check +from ...core.data.xarray import cftime_types from ...core.overlay import Overlay from ...core.util import (LooseVersion, _getargspec, basestring, callable_name, dt64_to_dt, pd, unique_array) @@ -209,10 +210,6 @@ def make_axis(axis, size, factors, dim, flip=False, rotation=0, return p -def convert_datetime(time): - return time.astype('datetime64[s]').astype(float)*1000 - - def hsv_to_rgb(hsv): """ Vectorized HSV to RGB conversion, adapted from: @@ -537,14 +534,51 @@ def append_refresh(dmap): return plot.hmap.traverse(append_refresh, [DynamicMap]) -def date_to_integer(date): +def cftime_to_timestamp(date): + """Converts cftime to timestamp since epoch in milliseconds + + Non-standard calendars (e.g. Julian or no leap calendars) + are converted to standard Gregorian calendar. This can cause + extra space to be added for dates that don't exist in the original + calendar. In order to handle these dates correctly a custom bokeh + model with support for other calendars would have to be defind. + + Args: + date: cftime datetime object (or array) + + Returns: + Milliseconds since 1970-01-01 00:00:00 """ - Converts datetime types to bokeh's integer format. + import cftime + utime = cftime.utime('days since 1970-01-01') + return utime.date2num(date)*86400000 + + +def date_to_integer(date): + """Converts support date types to milliseconds since epoch + + Attempts highest precision conversion of different datetime + formats to milliseconds since the epoch (1970-01-01 00:00:00). + If datetime is a cftime with a non-standard calendar the + caveats describd in cftime_to_timestamp apply. + + Args: + date: Date- or datetime-like object + + Returns: + Milliseconds since 1970-01-01 00:00:00 """ + if pd and isinstance(date, pd.Timestamp): + try: + date = date.to_datetime64() + except: + date = date.to_datetime() + if isinstance(date, np.datetime64): - date = dt64_to_dt(date) - elif pd and isinstance(date, pd.Timestamp): - date = date.to_pydatetime() + return time.astype('datetime64[ms]').astype(float) + elif isinstance(date, cftime_types): + return cftime_to_timestamp(date) + if hasattr(date, 'timetuple'): dt_int = calendar.timegm(date.timetuple())*1000 else: diff --git a/holoviews/plotting/mpl/element.py b/holoviews/plotting/mpl/element.py index 0e59114239..ab7548e9ba 100644 --- a/holoviews/plotting/mpl/element.py +++ b/holoviews/plotting/mpl/element.py @@ -14,6 +14,7 @@ from ...core import util from ...core import (OrderedDict, NdOverlay, DynamicMap, Dataset, CompositeOverlay, Element3D, Element) +from ...core.data.xarray import cftime_types from ...core.options import abbreviated_exception from ...element import Graph, Path, Contours from ...util.transform import dim @@ -331,7 +332,8 @@ def _set_axis_limits(self, axis, view, subplots, ranges): if self.invert_xaxis or any(p.invert_xaxis for p in subplots): r, l = l, r - if l != r: + + if isinstance(l, cftime_types) or l != r: lims = {} if valid_lim(l): lims['left'] = l @@ -343,7 +345,7 @@ def _set_axis_limits(self, axis, view, subplots, ranges): axis.set_xlim(**lims) if self.invert_yaxis or any(p.invert_yaxis for p in subplots): t, b = b, t - if b != t: + if isinstance(b, cftime_types) or b != t: lims = {} if valid_lim(b): lims['bottom'] = b From 180117ae8bb88feca9b7d9ae3718875c539aaae8 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 03:00:00 +0000 Subject: [PATCH 07/14] Reorganized code --- holoviews/core/data/xarray.py | 13 --------- holoviews/core/util.py | 45 +++++++++++++++++++++++++++-- holoviews/plotting/bokeh/element.py | 5 ++-- holoviews/plotting/bokeh/plot.py | 8 ++--- holoviews/plotting/bokeh/util.py | 26 ++--------------- holoviews/plotting/mpl/element.py | 5 ++-- holoviews/plotting/mpl/util.py | 22 +++++++------- 7 files changed, 64 insertions(+), 60 deletions(-) diff --git a/holoviews/core/data/xarray.py b/holoviews/core/data/xarray.py index 03fa9f2284..ae7394f33c 100644 --- a/holoviews/core/data/xarray.py +++ b/holoviews/core/data/xarray.py @@ -12,19 +12,6 @@ from .grid import GridInterface from .interface import Interface, DataError, dask_array_module -try: - import cftime - cftime_types = ( - cftime._cftime.DatetimeGregorian, - cftime._cftime.Datetime360Day, - cftime._cftime.DatetimeJulian, - cftime._cftime.DatetimeNoLeap, - cftime._cftime.DatetimeProlepticGregorian - ) - util.datetime_types += cftime_types -except: - cftime_types = () - class XArrayInterface(GridInterface): diff --git a/holoviews/core/util.py b/holoviews/core/util.py index c85c2096b7..c5e7d21f60 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -92,6 +92,19 @@ def __cmp__(self, other): except ImportError: pd = None +try: + import cftime + cftime_types = ( + cftime._cftime.DatetimeGregorian, + cftime._cftime.Datetime360Day, + cftime._cftime.DatetimeJulian, + cftime._cftime.DatetimeNoLeap, + cftime._cftime.DatetimeProlepticGregorian + ) + datetime_types += cftime_types +except: + cftime_types = () + class VersionError(Exception): "Raised when there is a library version mismatch." @@ -1805,8 +1818,12 @@ def dt_to_int(value, time_unit='us'): if isinstance(value, pd.Period): value = value.to_timestamp() if isinstance(value, pd.Timestamp): - value = value.to_pydatetime() - value = np.datetime64(value) + try: + value = value.to_datetime64() + except: + value = np.datetime64(value.to_pydatetime()) + elif isinstance(value, cftime_types): + return cftime_to_timestamp(value, time_unit) if isinstance(value, np.datetime64): value = np.datetime64(value, 'ns') @@ -1833,6 +1850,30 @@ def dt_to_int(value, time_unit='us'): return (time.mktime(value.timetuple()) + value.microsecond / 1e6) * tscale +def cftime_to_timestamp(date, time_unit='us'): + """Converts cftime to timestamp since epoch in milliseconds + + Non-standard calendars (e.g. Julian or no leap calendars) + are converted to standard Gregorian calendar. This can cause + extra space to be added for dates that don't exist in the original + calendar. In order to handle these dates correctly a custom bokeh + model with support for other calendars would have to be defind. + + Args: + date: cftime datetime object (or array) + + Returns: + Milliseconds since 1970-01-01 00:00:00 + """ + import cftime + utime = cftime.utime('microseconds since 1970-01-01') + if time_unit == 'us': + tscale = 1 + else: + tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'us')) * 1000. + return utime.date2num(date)*tscale + + def search_indices(values, source): """ Given a set of values returns the indices of each of those values diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py index 39a1f2cfa8..f944c1bd50 100644 --- a/holoviews/plotting/bokeh/element.py +++ b/holoviews/plotting/bokeh/element.py @@ -25,7 +25,6 @@ from bokeh.plotting.helpers import _known_tools as known_tools from ...core import DynamicMap, CompositeOverlay, Element, Dimension -from ...core.data.xarray import cftime_types from ...core.options import abbreviated_exception, SkipRendering from ...core import util from ...element import Graph, VectorField, Path, Contours @@ -607,7 +606,7 @@ def _update_ranges(self, element, ranges): def _update_range(self, axis_range, low, high, factors, invert, shared, log, streaming=False): if isinstance(axis_range, (Range1d, DataRange1d)) and self.apply_ranges: - if isinstance(low, cftime_types): + if isinstance(low, util.cftime_types): pass elif (low == high and low is not None): if isinstance(low, util.datetime_types): @@ -637,7 +636,7 @@ def _update_range(self, axis_range, low, high, factors, invert, shared, log, str if reset_supported: updates['reset_end'] = updates['end'] for k, (old, new) in updates.items(): - if isinstance(new, cftime_types): + if isinstance(new, util.cftime_types): new = date_to_integer(new) axis_range.update(**{k:new}) if streaming and not k.startswith('reset_'): diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py index 3659a5f4ad..3eebf86692 100644 --- a/holoviews/plotting/bokeh/plot.py +++ b/holoviews/plotting/bokeh/plot.py @@ -16,12 +16,10 @@ OrderedDict, Store, AdjointLayout, NdLayout, Layout, Empty, GridSpace, HoloMap, Element, DynamicMap ) -from ...core.data.xarray import cftime_types from ...core.options import SkipRendering from ...core.util import ( - basestring, wrap_tuple, unique_iterator, get_method_owner, - datetime_types, wrap_tuple_streams) - + basestring, cftime_to_timestamp, cftime_types, datetime_types, + get_method_owner, unique_iterator, wrap_tuple, wrap_tuple_streams) from ...streams import Stream from ..links import Link from ..plot import ( @@ -33,7 +31,7 @@ from .util import ( layout_padding, pad_plots, filter_toolboxes, make_axis, update_shared_sources, empty_plot, decode_bytes, theme_attr_json, - cds_column_replace, cftime_to_timestamp + cds_column_replace ) TOOLS = {name: tool if isinstance(tool, basestring) else type(tool()) diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py index 9e224cb75e..1e1ebbe50a 100644 --- a/holoviews/plotting/bokeh/util.py +++ b/holoviews/plotting/bokeh/util.py @@ -31,10 +31,10 @@ except: Chart = type(None) # Create stub for isinstance check -from ...core.data.xarray import cftime_types from ...core.overlay import Overlay -from ...core.util import (LooseVersion, _getargspec, basestring, - callable_name, dt64_to_dt, pd, unique_array) +from ...core.util import ( + LooseVersion, _getargspec, basestring, callable_name, cftime_types, + cftime_to_timestamp, dt64_to_dt, pd, unique_array) from ...core.spaces import get_nested_dmaps, DynamicMap from ..util import dim_axis_label @@ -534,26 +534,6 @@ def append_refresh(dmap): return plot.hmap.traverse(append_refresh, [DynamicMap]) -def cftime_to_timestamp(date): - """Converts cftime to timestamp since epoch in milliseconds - - Non-standard calendars (e.g. Julian or no leap calendars) - are converted to standard Gregorian calendar. This can cause - extra space to be added for dates that don't exist in the original - calendar. In order to handle these dates correctly a custom bokeh - model with support for other calendars would have to be defind. - - Args: - date: cftime datetime object (or array) - - Returns: - Milliseconds since 1970-01-01 00:00:00 - """ - import cftime - utime = cftime.utime('days since 1970-01-01') - return utime.date2num(date)*86400000 - - def date_to_integer(date): """Converts support date types to milliseconds since epoch diff --git a/holoviews/plotting/mpl/element.py b/holoviews/plotting/mpl/element.py index ab7548e9ba..87689a7924 100644 --- a/holoviews/plotting/mpl/element.py +++ b/holoviews/plotting/mpl/element.py @@ -14,7 +14,6 @@ from ...core import util from ...core import (OrderedDict, NdOverlay, DynamicMap, Dataset, CompositeOverlay, Element3D, Element) -from ...core.data.xarray import cftime_types from ...core.options import abbreviated_exception from ...element import Graph, Path, Contours from ...util.transform import dim @@ -333,7 +332,7 @@ def _set_axis_limits(self, axis, view, subplots, ranges): if self.invert_xaxis or any(p.invert_xaxis for p in subplots): r, l = l, r - if isinstance(l, cftime_types) or l != r: + if isinstance(l, util.cftime_types) or l != r: lims = {} if valid_lim(l): lims['left'] = l @@ -345,7 +344,7 @@ def _set_axis_limits(self, axis, view, subplots, ranges): axis.set_xlim(**lims) if self.invert_yaxis or any(p.invert_yaxis for p in subplots): t, b = b, t - if isinstance(b, cftime_types) or b != t: + if isinstance(b, util.cftime_types) or b != t: lims = {} if valid_lim(b): lims['bottom'] = b diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py index 45d612cc5d..d1a71191d8 100644 --- a/holoviews/plotting/mpl/util.py +++ b/holoviews/plotting/mpl/util.py @@ -16,8 +16,17 @@ validate_capstyle, validate_fontsize, validate_fonttype, validate_hatch, validate_joinstyle) -from ...core.data.xarray import cftime_types -from ...core.util import LooseVersion, _getargspec, basestring, is_number +try: + import cftime + from nc_time_axis import NetCDFTimeConverter, CalendarDateTime + nc_axis_available = True +except: + from matplotlib.dates import DateConverter + NetCDFTimeConverter = DateConverter + nc_axis_available = False + +from ...core.util import ( + LooseVersion, _getargspec, basestring, cftime_types, is_number) from ...element import Raster, RGB, Polygons from ..util import COLOR_ALIASES, RGB_HEX_REGEX @@ -338,15 +347,6 @@ def polygons_to_path_patches(element): mpl_paths.append(subpath) return mpl_paths -try: - import cftime - from nc_time_axis import NetCDFTimeConverter, CalendarDateTime - nc_axis_available = True -except: - from matplotlib.dates import DateConverter - NetCDFTimeConverter = DateConverter - nc_axis_available = False - class CFTimeConverter(NetCDFTimeConverter): """ From 5aad6f2023461ba8895c11404962b29abf03d008 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 13:33:17 +0000 Subject: [PATCH 08/14] Various datetime handling fixes --- holoviews/core/util.py | 12 +++--------- holoviews/plotting/bokeh/element.py | 4 ++-- holoviews/plotting/bokeh/plot.py | 2 +- holoviews/plotting/bokeh/util.py | 4 ++-- holoviews/plotting/mpl/util.py | 2 +- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/holoviews/core/util.py b/holoviews/core/util.py index c5e7d21f60..40883e033b 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -94,13 +94,7 @@ def __cmp__(self, other): try: import cftime - cftime_types = ( - cftime._cftime.DatetimeGregorian, - cftime._cftime.Datetime360Day, - cftime._cftime.DatetimeJulian, - cftime._cftime.DatetimeNoLeap, - cftime._cftime.DatetimeProlepticGregorian - ) + cftime_types = (cftime.datetime,) datetime_types += cftime_types except: cftime_types = () @@ -1857,7 +1851,7 @@ def cftime_to_timestamp(date, time_unit='us'): are converted to standard Gregorian calendar. This can cause extra space to be added for dates that don't exist in the original calendar. In order to handle these dates correctly a custom bokeh - model with support for other calendars would have to be defind. + model with support for other calendars would have to be defined. Args: date: cftime datetime object (or array) @@ -1870,7 +1864,7 @@ def cftime_to_timestamp(date, time_unit='us'): if time_unit == 'us': tscale = 1 else: - tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'us')) * 1000. + tscale = (np.timedelta64(1, 'us')/np.timedelta64(1, time_unit)) return utime.date2num(date)*tscale diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py index f944c1bd50..f0b3f488c9 100644 --- a/holoviews/plotting/bokeh/element.py +++ b/holoviews/plotting/bokeh/element.py @@ -301,7 +301,7 @@ def _axes_props(self, plots, subplots, element, ranges): xtype = el.nodes.get_dimension_type(xdims[0]) else: xtype = el.get_dimension_type(xdims[0]) - if ((xtype is np.object_ and type(l) in util.datetime_types) or + if ((xtype is np.object_ and issubclass(type(l), util.datetime_types)) or xtype in util.datetime_types): x_axis_type = 'datetime' @@ -315,7 +315,7 @@ def _axes_props(self, plots, subplots, element, ranges): ytype = el.nodes.get_dimension_type(ydims[0]) else: ytype = el.get_dimension_type(ydims[0]) - if ((ytype is np.object_ and type(b) in util.datetime_types) + if ((ytype is np.object_ and issubclass(type(b), util.datetime_types)) or ytype in util.datetime_types): y_axis_type = 'datetime' diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py index 3eebf86692..1ee2f9c211 100644 --- a/holoviews/plotting/bokeh/plot.py +++ b/holoviews/plotting/bokeh/plot.py @@ -248,7 +248,7 @@ def _postprocess_data(self, data): # Certain datetime types need to be converted if len(values) and isinstance(values[0], cftime_types): - values = cftime_to_timestamp(values) + values = cftime_to_timestamp(values, 'ms') new_data[k] = values return new_data diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py index 1e1ebbe50a..ca443c0b69 100644 --- a/holoviews/plotting/bokeh/util.py +++ b/holoviews/plotting/bokeh/util.py @@ -540,7 +540,7 @@ def date_to_integer(date): Attempts highest precision conversion of different datetime formats to milliseconds since the epoch (1970-01-01 00:00:00). If datetime is a cftime with a non-standard calendar the - caveats describd in cftime_to_timestamp apply. + caveats described in hv.core.util.cftime_to_timestamp apply. Args: date: Date- or datetime-like object @@ -557,7 +557,7 @@ def date_to_integer(date): if isinstance(date, np.datetime64): return time.astype('datetime64[ms]').astype(float) elif isinstance(date, cftime_types): - return cftime_to_timestamp(date) + return cftime_to_timestamp(date, 'ms') if hasattr(date, 'timetuple'): dt_int = calendar.timegm(date.timetuple())*1000 diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py index d1a71191d8..da2fd72d0a 100644 --- a/holoviews/plotting/mpl/util.py +++ b/holoviews/plotting/mpl/util.py @@ -364,7 +364,7 @@ def convert(cls, value, unit, axis): if isinstance(value, cftime_types): value = CalendarDateTime(cftime.datetime(*value.timetuple()[0:6]), value.calendar) elif isinstance(value, np.ndarray): - value = np.array([CalendarDateTime(cftime.datetime(*v.timetuple()[0:6]), v.calendar) for v in value]) + value = np.array([CalendarDateTime(v.datetime, v.calendar) for v in value]) return super(CFTimeConverter, cls).convert(value, unit, axis) From bef18f65be53b0cc9079bda4e5b9f3100ceb48e7 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 14:10:26 +0000 Subject: [PATCH 09/14] Overhauled datetime handling --- holoviews/core/util.py | 7 +++-- holoviews/operation/datashader.py | 47 ++++++++++++++++++------------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/holoviews/core/util.py b/holoviews/core/util.py index 40883e033b..4dbdebb377 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -1824,18 +1824,19 @@ def dt_to_int(value, time_unit='us'): if time_unit == 'ns': tscale = 1 else: - tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'ns')) * 1000. + tscale = (np.timedelta64(1, 'ns')/np.timedelta64(1, time_unit)) elif time_unit == 'ns': - tscale = 1000. + tscale = 1e9 else: tscale = 1./np.timedelta64(1, time_unit).tolist().total_seconds() if isinstance(value, np.datetime64): value = value.tolist() + if isinstance(value, (int, long)): # Handle special case of nanosecond precision which cannot be # represented by python datetime - return value * 10**-(np.log10(tscale)-3) + return value * 10**-(np.log10(tscale)) try: # Handle python3 return int(value.timestamp() * tscale) diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py index af671ec549..760f0fea44 100644 --- a/holoviews/operation/datashader.py +++ b/holoviews/operation/datashader.py @@ -23,7 +23,9 @@ CompositeOverlay, Dataset, Overlay) from ..core.data import PandasInterface, XArrayInterface from ..core.sheetcoords import BoundingBox -from ..core.util import LooseVersion, get_param_values, basestring, datetime_types, dt_to_int +from ..core.util import ( + LooseVersion, basestring, cftime_types, cftime_to_timestamp, + datetime_types, dt_to_int, get_param_values) from ..element import (Image, Path, Curve, RGB, Graph, TriMesh, QuadMesh, Contours) from ..streams import RangeXY, PlotSize @@ -321,11 +323,18 @@ def get_agg_data(cls, obj, category=None): if category and df[category].dtype.name != 'category': df[category] = df[category].astype('category') - if any(df[d.name].dtype.kind == 'M' for d in (x, y)): + if any(df[d.name].dtype.kind == 'M' or isinstance(df[d.name].values[0], cftime_types) + for d in (x, y)): df = df.copy() for d in (x, y): - if df[d.name].dtype.kind == 'M': - df[d.name] = df[d.name].astype('datetime64[ns]').astype('int64') * 1000. + vals = df[d.name].values + if len(vals) and isinstance(vals[0], cftime_types): + vals = cftime_to_timestamp(vals, 'ns') + elif df[d.name].dtype.kind == 'M': + vals = vals.astype('datetime64[ns]') + else: + continue + df[d.name] = vals.astype('int64') return x, y, Dataset(df, kdims=kdims, vdims=vdims), glyph @@ -344,9 +353,9 @@ def _aggregate_ndoverlay(self, element, agg_fn): info = self._get_sampling(element, x, y) (x_range, y_range), (xs, ys), (width, height), (xtype, ytype) = info if xtype == 'datetime': - x_range = tuple((np.array(x_range)/10e5).astype('datetime64[us]')) + x_range = tuple((np.array(x_range)/1e3).astype('datetime64[us]')) if ytype == 'datetime': - y_range = tuple((np.array(y_range)/10e5).astype('datetime64[us]')) + y_range = tuple((np.array(y_range)/1e3).astype('datetime64[us]')) agg_params = dict({k: v for k, v in dict(self.get_param_values(), **self.p).items() if k in aggregate.params()}, x_range=x_range, y_range=y_range) @@ -433,11 +442,11 @@ def _process(self, element, key=None): (x0, x1), (y0, y1) = x_range, y_range if xtype == 'datetime': - x0, x1 = (np.array([x0, x1])/10e5).astype('datetime64[us]') - xs = (xs/10e5).astype('datetime64[us]') + x0, x1 = (np.array([x0, x1])/1e3).astype('datetime64[us]') + xs = (xs/1e3).astype('datetime64[us]') if ytype == 'datetime': - y0, y1 = (np.array([y0, y1])/10e5).astype('datetime64[us]') - ys = (ys/10e5).astype('datetime64[us]') + y0, y1 = (np.array([y0, y1])/1e3).astype('datetime64[us]') + ys = (ys/1e3).astype('datetime64[us]') bounds = (x0, y0, x1, y1) params = dict(get_param_values(element), kdims=[x, y], datatype=['xarray'], bounds=bounds) @@ -483,9 +492,9 @@ def _process(self, element, key=None): if 'x_axis' in agg.coords and 'y_axis' in agg.coords: agg = agg.rename({'x_axis': x, 'y_axis': y}) if xtype == 'datetime': - agg[x.name] = (agg[x.name]/10e5).astype('datetime64[us]') + agg[x.name] = (agg[x.name]/1e3).astype('datetime64[us]') if ytype == 'datetime': - agg[y.name] = (agg[y.name]/10e5).astype('datetime64[us]') + agg[y.name] = (agg[y.name]/1e3).astype('datetime64[us]') if agg.ndim == 2: # Replacing x and y coordinates to avoid numerical precision issues @@ -606,11 +615,11 @@ def _process(self, element, key=None): # Compute bounds (converting datetimes) if xtype == 'datetime': - xstart, xend = (np.array([xstart, xend])/10e5).astype('datetime64[us]') - xs = (xs/10e5).astype('datetime64[us]') + xstart, xend = (np.array([xstart, xend])/1e3).astype('datetime64[us]') + xs = (xs/1e3).astype('datetime64[us]') if ytype == 'datetime': - ystart, yend = (np.array([ystart, yend])/10e5).astype('datetime64[us]') - ys = (ys/10e5).astype('datetime64[us]') + ystart, yend = (np.array([ystart, yend])/1e3).astype('datetime64[us]') + ys = (ys/1e3).astype('datetime64[us]') bbox = BoundingBox(points=[(xstart, ystart), (xend, yend)]) params = dict(bounds=bbox) @@ -632,9 +641,9 @@ def _process(self, element, key=None): # Convert datetime coordinates if xtype == "datetime": - rarray[x.name] = (rarray[x.name]/10e5).astype('datetime64[us]') + rarray[x.name] = (rarray[x.name]/1e3).astype('datetime64[us]') if ytype == "datetime": - rarray[y.name] = (rarray[y.name]/10e5).astype('datetime64[us]') + rarray[y.name] = (rarray[y.name]/1e3).astype('datetime64[us]') regridded[vd] = rarray regridded = xr.Dataset(regridded) @@ -957,7 +966,7 @@ def _process(self, element, key=None): for d in kdims: if array[d.name].dtype.kind == 'M': - array[d.name] = array[d.name].astype('datetime64[ns]').astype('int64') * 10e-4 + array[d.name] = array[d.name].astype('datetime64[us]').astype('int64') with warnings.catch_warnings(): warnings.filterwarnings('ignore', r'invalid value encountered in true_divide') From 5eee4b6e86ef960d4679aa53f5b6dca6d892cb98 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 14:19:11 +0000 Subject: [PATCH 10/14] Added warning about non-standard calendars --- holoviews/core/util.py | 1 + holoviews/plotting/bokeh/plot.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/holoviews/core/util.py b/holoviews/core/util.py index 4dbdebb377..ca37a5b05e 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -98,6 +98,7 @@ def __cmp__(self, other): datetime_types += cftime_types except: cftime_types = () +_STANDARD_CALENDARS = set(['standard', 'gregorian', 'proleptic_gregorian']) class VersionError(Exception): diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py index 1ee2f9c211..fb33142ebc 100644 --- a/holoviews/plotting/bokeh/plot.py +++ b/holoviews/plotting/bokeh/plot.py @@ -19,7 +19,8 @@ from ...core.options import SkipRendering from ...core.util import ( basestring, cftime_to_timestamp, cftime_types, datetime_types, - get_method_owner, unique_iterator, wrap_tuple, wrap_tuple_streams) + get_method_owner, unique_iterator, wrap_tuple, wrap_tuple_streams, + _STANDARD_CALENDARS) from ...streams import Stream from ..links import Link from ..plot import ( @@ -248,6 +249,13 @@ def _postprocess_data(self, data): # Certain datetime types need to be converted if len(values) and isinstance(values[0], cftime_types): + if any(v.calendar not in _STANDARD_CALENDARS for v in values): + self.param.warning( + 'Converting cftime.datetime from a non-standard ' + 'calendar (%s) to a standard calendar for plotting. ' + 'This may lead to subtle errors in formatting ' + 'dates, for accurate tick formatting switch to ' + 'the matplotlib backend.' % values[0].calendar) values = cftime_to_timestamp(values, 'ms') new_data[k] = values return new_data From 4da46b8285867cdc6d0d26dbc7c4c6e64e1d8377 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 14:24:41 +0000 Subject: [PATCH 11/14] Further fix --- holoviews/plotting/mpl/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py index da2fd72d0a..5b6aa18877 100644 --- a/holoviews/plotting/mpl/util.py +++ b/holoviews/plotting/mpl/util.py @@ -362,7 +362,7 @@ def convert(cls, value, unit, axis): 'using:\n\tconda install -c conda-forge ' 'nc_time_axis') if isinstance(value, cftime_types): - value = CalendarDateTime(cftime.datetime(*value.timetuple()[0:6]), value.calendar) + value = CalendarDateTime(v.datetime, value.calendar) elif isinstance(value, np.ndarray): value = np.array([CalendarDateTime(v.datetime, v.calendar) for v in value]) return super(CFTimeConverter, cls).convert(value, unit, axis) From e35461218a07fe5a33c1081bf7b81d7fb626af92 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 14:33:39 +0000 Subject: [PATCH 12/14] Added unit tests for cftime conversions --- .../tests/plotting/bokeh/testelementplot.py | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/holoviews/tests/plotting/bokeh/testelementplot.py b/holoviews/tests/plotting/bokeh/testelementplot.py index 62112e4bad..ac8bece0fe 100644 --- a/holoviews/tests/plotting/bokeh/testelementplot.py +++ b/holoviews/tests/plotting/bokeh/testelementplot.py @@ -10,6 +10,7 @@ from holoviews.plotting.util import process_cmap from .testplot import TestBokehPlot, bokeh_renderer +from ...utils import LoggingComparisonTestCase try: from bokeh.document import Document @@ -19,7 +20,7 @@ -class TestElementPlot(TestBokehPlot): +class TestElementPlot(LoggingComparisonTestCase, TestBokehPlot): def test_element_show_frame_disabled(self): curve = Curve(range(10)).opts(plot=dict(show_frame=False)) @@ -287,7 +288,6 @@ def test_categorical_axis_fontsize(self): curve = Curve([('A', 1), ('B', 2)]).options(fontsize={'minor_xticks': '6pt', 'xticks': 18}) plot = bokeh_renderer.get_plot(curve) xaxis = plot.handles['xaxis'] - print(xaxis.properties_with_values()) self.assertEqual(xaxis.major_label_text_font_size, '6pt') self.assertEqual(xaxis.group_text_font_size, {'value': '18pt'}) @@ -298,6 +298,39 @@ def test_categorical_axis_fontsize_both(self): self.assertEqual(xaxis.major_label_text_font_size, {'value': '18pt'}) self.assertEqual(xaxis.group_text_font_size, {'value': '18pt'}) + def test_cftime_transform_gregorian_no_warn(self): + try: + import cftime + except: + raise SkipTest('Test requires cftime library') + gregorian_dates = [cftime.DatetimeGregorian(2000, 2, 28), + cftime.DatetimeGregorian(2000, 3, 1), + cftime.DatetimeGregorian(2000, 3, 2)] + curve = Curve((gregorian_dates, [1, 2, 3])) + plot = bokeh_renderer.get_plot(curve) + xs = plot.handles['cds'].data['x'] + self.assertEqual(xs.astype('int'), + np.array([951696000000, 951868800000, 951955200000])) + + def test_cftime_transform_noleap_warn(self): + try: + import cftime + except: + raise SkipTest('Test requires cftime library') + gregorian_dates = [cftime.DatetimeNoLeap(2000, 2, 28), + cftime.DatetimeNoLeap(2000, 3, 1), + cftime.DatetimeNoLeap(2000, 3, 2)] + curve = Curve((gregorian_dates, [1, 2, 3])) + plot = bokeh_renderer.get_plot(curve) + xs = plot.handles['cds'].data['x'] + self.assertEqual(xs.astype('int'), + np.array([951696000000, 951868800000, 951955200000])) + substr = ( + "Converting cftime.datetime from a non-standard calendar " + "(noleap) to a standard calendar for plotting. This may " + "lead to subtle errors in formatting dates, for accurate " + "tick formatting switch to the matplotlib backend.") + self.log_handler.assertEndsWith('WARNING', substr) class TestColorbarPlot(TestBokehPlot): From 49e3b769c1aea48b57dc56ba02f6af5efed272c5 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 16:34:55 +0000 Subject: [PATCH 13/14] Further fixes for datetime handling --- holoviews/core/util.py | 23 ++++++------ holoviews/operation/element.py | 6 ++-- holoviews/tests/core/testutils.py | 6 ++-- holoviews/tests/operation/testoperation.py | 42 ++++++++++++++-------- 4 files changed, 43 insertions(+), 34 deletions(-) diff --git a/holoviews/core/util.py b/holoviews/core/util.py index ca37a5b05e..29573568f4 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -1820,24 +1820,21 @@ def dt_to_int(value, time_unit='us'): elif isinstance(value, cftime_types): return cftime_to_timestamp(value, time_unit) + # Handle datetime64 separately if isinstance(value, np.datetime64): - value = np.datetime64(value, 'ns') - if time_unit == 'ns': - tscale = 1 - else: - tscale = (np.timedelta64(1, 'ns')/np.timedelta64(1, time_unit)) - elif time_unit == 'ns': + try: + value = np.datetime64(value, 'ns') + tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'ns')) + return value.tolist()/tscale + except: + # If it can't handle ns precision fall back to datetime + value = value.tolist() + + if time_unit == 'ns': tscale = 1e9 else: tscale = 1./np.timedelta64(1, time_unit).tolist().total_seconds() - if isinstance(value, np.datetime64): - value = value.tolist() - - if isinstance(value, (int, long)): - # Handle special case of nanosecond precision which cannot be - # represented by python datetime - return value * 10**-(np.log10(tscale)) try: # Handle python3 return int(value.timestamp() * tscale) diff --git a/holoviews/operation/element.py b/holoviews/operation/element.py index 5bb20b10ae..80a9d90685 100644 --- a/holoviews/operation/element.py +++ b/holoviews/operation/element.py @@ -589,9 +589,9 @@ def _process(self, element, key=None): if data.dtype.kind == 'M' or (data.dtype.kind == 'O' and isinstance(data[0], datetime_types)): start, end = dt_to_int(start, 'ns'), dt_to_int(end, 'ns') datetimes = True - data = data.astype('datetime64[ns]').astype('int64') * 1000. + data = data.astype('datetime64[ns]').astype('int64') if bins is not None: - bins = bins.astype('datetime64[ns]').astype('int64') * 1000. + bins = bins.astype('datetime64[ns]').astype('int64') else: hist_range = start, end @@ -622,7 +622,7 @@ def _process(self, element, key=None): hist = np.zeros(self.p.num_bins) hist[np.isnan(hist)] = 0 if datetimes: - edges = (edges/10e5).astype('datetime64[us]') + edges = (edges/1e3).astype('datetime64[us]') params = {} if self.p.weight_dimension: diff --git a/holoviews/tests/core/testutils.py b/holoviews/tests/core/testutils.py index bd37d5d93f..097a1dceb3 100644 --- a/holoviews/tests/core/testutils.py +++ b/holoviews/tests/core/testutils.py @@ -584,15 +584,15 @@ def test_datetime_to_us_int(self): def test_datetime64_s_to_ns_int(self): dt = np.datetime64(datetime.datetime(2017, 1, 1), 's') - self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000000.0) + self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000.0) def test_datetime64_us_to_ns_int(self): dt = np.datetime64(datetime.datetime(2017, 1, 1), 'us') - self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000000.0) + self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000.0) def test_datetime64_to_ns_int(self): dt = np.datetime64(datetime.datetime(2017, 1, 1)) - self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000000.0) + self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000.0) def test_datetime64_us_to_us_int(self): dt = np.datetime64(datetime.datetime(2017, 1, 1), 'us') diff --git a/holoviews/tests/operation/testoperation.py b/holoviews/tests/operation/testoperation.py index 1f1ed744ca..e76770f5e2 100644 --- a/holoviews/tests/operation/testoperation.py +++ b/holoviews/tests/operation/testoperation.py @@ -168,22 +168,30 @@ def test_points_histogram_not_normed(self): def test_histogram_operation_datetime(self): dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]) op_hist = histogram(Dataset(dates, 'Date'), num_bins=4) - hist_data = {'Date': np.array(['2017-01-01T00:00:00.000000', '2017-01-01T17:59:59.999999', - '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000', - '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'), - 'Date_frequency': np.array([ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, - 3.85802469e-18])} + hist_data = { + 'Date': np.array([ + '2017-01-01T00:00:00.000000', '2017-01-01T18:00:00.000000', + '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000', + '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'), + 'Date_frequency': np.array([ + 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, + 3.85802469e-18]) + } hist = Histogram(hist_data, kdims='Date', vdims=('Date_frequency', 'Frequency')) self.assertEqual(op_hist, hist) def test_histogram_operation_datetime64(self): dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]).astype('M') op_hist = histogram(Dataset(dates, 'Date'), num_bins=4) - hist_data = {'Date': np.array(['2017-01-01T00:00:00.000000', '2017-01-01T17:59:59.999999', - '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000', - '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'), - 'Date_frequency': np.array([ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, - 3.85802469e-18])} + hist_data = { + 'Date': np.array([ + '2017-01-01T00:00:00.000000', '2017-01-01T18:00:00.000000', + '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000', + '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'), + 'Date_frequency': np.array([ + 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, + 3.85802469e-18]) + } hist = Histogram(hist_data, kdims='Date', vdims=('Date_frequency', 'Frequency')) self.assertEqual(op_hist, hist) @@ -191,11 +199,15 @@ def test_histogram_operation_datetime64(self): def test_histogram_operation_pd_period(self): dates = pd.date_range('2017-01-01', '2017-01-04', freq='D').to_period('D') op_hist = histogram(Dataset(dates, 'Date'), num_bins=4) - hist_data = {'Date': np.array(['2017-01-01T00:00:00.000000', '2017-01-01T17:59:59.999999', - '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000', - '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'), - 'Date_frequency': np.array([ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, - 3.85802469e-18])} + hist_data = { + 'Date': np.array([ + '2017-01-01T00:00:00.000000', '2017-01-01T18:00:00.000000', + '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000', + '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'), + 'Date_frequency': np.array([ + 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, + 3.85802469e-18]) + } hist = Histogram(hist_data, kdims='Date', vdims=('Date_frequency', 'Frequency')) self.assertEqual(op_hist, hist) From 23fabdd8c362f2e0207c8f85cc3f0faffa4e6aaa Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Thu, 6 Dec 2018 17:24:06 +0000 Subject: [PATCH 14/14] Fixed flakes --- holoviews/plotting/bokeh/plot.py | 5 ++--- holoviews/plotting/bokeh/util.py | 2 +- holoviews/plotting/mpl/util.py | 3 +-- .../tests/plotting/bokeh/testhistogramplot.py | 20 ++++++++++++------- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py index fb33142ebc..f7c745ee88 100644 --- a/holoviews/plotting/bokeh/plot.py +++ b/holoviews/plotting/bokeh/plot.py @@ -18,9 +18,8 @@ ) from ...core.options import SkipRendering from ...core.util import ( - basestring, cftime_to_timestamp, cftime_types, datetime_types, - get_method_owner, unique_iterator, wrap_tuple, wrap_tuple_streams, - _STANDARD_CALENDARS) + basestring, cftime_to_timestamp, cftime_types, get_method_owner, + unique_iterator, wrap_tuple, wrap_tuple_streams, _STANDARD_CALENDARS) from ...streams import Stream from ..links import Link from ..plot import ( diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py index ca443c0b69..94063666d9 100644 --- a/holoviews/plotting/bokeh/util.py +++ b/holoviews/plotting/bokeh/util.py @@ -34,7 +34,7 @@ from ...core.overlay import Overlay from ...core.util import ( LooseVersion, _getargspec, basestring, callable_name, cftime_types, - cftime_to_timestamp, dt64_to_dt, pd, unique_array) + cftime_to_timestamp, pd, unique_array) from ...core.spaces import get_nested_dmaps, DynamicMap from ..util import dim_axis_label diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py index 5b6aa18877..701744afa6 100644 --- a/holoviews/plotting/mpl/util.py +++ b/holoviews/plotting/mpl/util.py @@ -17,7 +17,6 @@ validate_joinstyle) try: - import cftime from nc_time_axis import NetCDFTimeConverter, CalendarDateTime nc_axis_available = True except: @@ -362,7 +361,7 @@ def convert(cls, value, unit, axis): 'using:\n\tconda install -c conda-forge ' 'nc_time_axis') if isinstance(value, cftime_types): - value = CalendarDateTime(v.datetime, value.calendar) + value = CalendarDateTime(value.datetime, value.calendar) elif isinstance(value, np.ndarray): value = np.array([CalendarDateTime(v.datetime, v.calendar) for v in value]) return super(CFTimeConverter, cls).convert(value, unit, axis) diff --git a/holoviews/tests/plotting/bokeh/testhistogramplot.py b/holoviews/tests/plotting/bokeh/testhistogramplot.py index c1f5a28f64..e405daa790 100644 --- a/holoviews/tests/plotting/bokeh/testhistogramplot.py +++ b/holoviews/tests/plotting/bokeh/testhistogramplot.py @@ -59,13 +59,19 @@ def test_histogram_datetime64_plot(self): hist = histogram(Dataset(dates, 'Date'), num_bins=4) plot = bokeh_renderer.get_plot(hist) source = plot.handles['source'] - data = {'top': np.array([ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, 3.85802469e-18]), - 'left': np.array(['2017-01-01T00:00:00.000000', '2017-01-01T17:59:59.999999', - '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000'], - dtype='datetime64[us]'), - 'right': np.array(['2017-01-01T17:59:59.999999', '2017-01-02T12:00:00.000000', - '2017-01-03T06:00:00.000000', '2017-01-04T00:00:00.000000'], - dtype='datetime64[us]')} + print(source.data) + data = { + 'top': np.array([ + 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, 3.85802469e-18]), + 'left': np.array([ + '2017-01-01T00:00:00.000000', '2017-01-01T18:00:00.000000', + '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000'], + dtype='datetime64[us]'), + 'right': np.array([ + '2017-01-01T18:00:00.000000', '2017-01-02T12:00:00.000000', + '2017-01-03T06:00:00.000000', '2017-01-04T00:00:00.000000'], + dtype='datetime64[us]') + } for k, v in data.items(): self.assertEqual(source.data[k], v) xaxis = plot.handles['xaxis']