From b7d37b856f9240688fe11b65f72e2c9cc42b1267 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Fri, 25 May 2018 02:50:26 +0100
Subject: [PATCH 01/14] Added support for cftime types
---
holoviews/core/data/xarray.py | 11 +++++++++++
holoviews/plotting/bokeh/element.py | 4 +++-
holoviews/plotting/bokeh/plot.py | 25 +++++++++++++++++++++----
holoviews/plotting/bokeh/util.py | 9 +++++----
4 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/holoviews/core/data/xarray.py b/holoviews/core/data/xarray.py
index ae7394f33c..cac57018a7 100644
--- a/holoviews/core/data/xarray.py
+++ b/holoviews/core/data/xarray.py
@@ -12,6 +12,17 @@
from .grid import GridInterface
from .interface import Interface, DataError, dask_array_module
+try:
+ import cftime
+ util.datetime_types += (
+ cftime._cftime.DatetimeGregorian,
+ cftime._cftime.Datetime360Day,
+ cftime._cftime.DatetimeJulian,
+ cftime._cftime.DatetimeNoLeap,
+ cftime._cftime.DatetimeProlepticGregorian)
+except:
+ pass
+
class XArrayInterface(GridInterface):
diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py
index b0e3364771..210243893b 100644
--- a/holoviews/plotting/bokeh/element.py
+++ b/holoviews/plotting/bokeh/element.py
@@ -40,7 +40,7 @@
from .util import (
bokeh_version, decode_bytes, get_tab_title, glyph_order,
py2js_tickformatter, recursive_model_update, theme_attr_json,
- cds_column_replace, hold_policy, match_dim_specs
+ cds_column_replace, hold_policy, match_dim_specs, date_to_integer
)
@@ -634,6 +634,8 @@ def _update_range(self, axis_range, low, high, factors, invert, shared, log, str
if reset_supported:
updates['reset_end'] = updates['end']
for k, (old, new) in updates.items():
+ if isinstance(new, util.datetime_types):
+ new = date_to_integer(new)
axis_range.update(**{k:new})
if streaming and not k.startswith('reset_'):
axis_range.trigger(k, old, new)
diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py
index ab1d675d6d..66bcdb9bd1 100644
--- a/holoviews/plotting/bokeh/plot.py
+++ b/holoviews/plotting/bokeh/plot.py
@@ -11,12 +11,13 @@
from bokeh.models import (ColumnDataSource, Column, Row, Div)
from bokeh.models.widgets import Panel, Tabs
from bokeh.plotting.helpers import _known_tools as known_tools
+from bokeh.util.serialization import convert_datetime_array
from ...core import (OrderedDict, Store, AdjointLayout, NdLayout, Layout,
Empty, GridSpace, HoloMap, Element, DynamicMap)
from ...core.options import SkipRendering
from ...core.util import (basestring, wrap_tuple, unique_iterator,
- get_method_owner, wrap_tuple_streams)
+ get_method_owner, datetime_types, wrap_tuple_streams)
from ...streams import Stream
from ..links import Link
from ..plot import (DimensionedPlot, GenericCompositePlot, GenericLayoutPlot,
@@ -25,7 +26,7 @@
from .callbacks import LinkCallback
from .util import (layout_padding, pad_plots, filter_toolboxes, make_axis,
update_shared_sources, empty_plot, decode_bytes,
- theme_attr_json, cds_column_replace)
+ theme_attr_json, cds_column_replace, date_to_integer)
TOOLS = {name: tool if isinstance(tool, basestring) else type(tool())
for name, tool in known_tools.items()}
@@ -226,10 +227,26 @@ def _init_datasource(self, data):
"""
Initializes a data source to be passed into the bokeh glyph.
"""
- data = {k: decode_bytes(vs) for k, vs in data.items()}
+ data = self._postprocess_data(data)
return ColumnDataSource(data=data)
+ def _postprocess_data(self, data):
+ """
+ Applies necessary type transformation to the data before
+ it is set on a ColumnDataSource.
+ """
+ new_data = {}
+ for k, values in data.items():
+ values = decode_bytes(values) # Bytes need decoding to strings
+
+ # Certain datetime types need to be converted
+ if len(values) and isinstance(values[0], datetime_types):
+ values = np.array([date_to_integer(v) for v in values])
+ new_data[k] = values
+ return new_data
+
+
def _update_datasource(self, source, data):
"""
Update datasource with data for a new frame.
@@ -237,7 +254,7 @@ def _update_datasource(self, source, data):
if not self.document:
return
- data = {k: decode_bytes(vs) for k, vs in data.items()}
+ data = self._postprocess_data(data)
empty = all(len(v) == 0 for v in data.values())
if (self.streaming and self.streaming[0].data is self.current_frame.data
and self._stream_data and not empty):
diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py
index 145bd9a75e..5c95c3f6dd 100644
--- a/holoviews/plotting/bokeh/util.py
+++ b/holoviews/plotting/bokeh/util.py
@@ -3,6 +3,7 @@
import re
import time
import sys
+import calendar
import datetime as dt
from collections import defaultdict
@@ -542,10 +543,10 @@ def date_to_integer(date):
"""
if isinstance(date, np.datetime64):
date = dt64_to_dt(date)
- if pd and isinstance(date, pd.Timestamp):
- dt_int = date.timestamp()*1000
- elif isinstance(date, (dt.datetime, dt.date)):
- dt_int = time.mktime(date.timetuple())*1000
+ elif pd and isinstance(date, pd.Timestamp):
+ date = date.to_pydatetime()
+ if hasattr(date, 'timetuple'):
+ dt_int = calendar.timegm(date.timetuple())*1000
else:
raise ValueError('Datetime type not recognized')
return dt_int
From 5ec1e7d9a82d30220b9a346f97313170b0b0f5e9 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Mon, 3 Dec 2018 18:27:11 +0000
Subject: [PATCH 02/14] Minor fix so NdMapping allows cftimes types as keys
---
holoviews/core/ndmapping.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/holoviews/core/ndmapping.py b/holoviews/core/ndmapping.py
index 067b44068d..121899fde4 100644
--- a/holoviews/core/ndmapping.py
+++ b/holoviews/core/ndmapping.py
@@ -650,7 +650,7 @@ def __getitem__(self, indexslice):
return self.data[()]
elif indexslice in [Ellipsis, ()]:
return self
- elif Ellipsis in wrap_tuple(indexslice):
+ elif any(Ellipsis is sl for sl in wrap_tuple(indexslice)):
indexslice = process_ellipses(self, indexslice)
map_slice, data_slice = self._split_index(indexslice)
From 88bebf399eca1b99e395f3bf8807b7163e6d78bc Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Wed, 5 Dec 2018 13:02:17 +0000
Subject: [PATCH 03/14] Add support for nc_time_axis in matplotlib backend
---
holoviews/core/data/xarray.py | 8 +++++---
holoviews/plotting/mpl/util.py | 34 ++++++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 3 deletions(-)
diff --git a/holoviews/core/data/xarray.py b/holoviews/core/data/xarray.py
index cac57018a7..03fa9f2284 100644
--- a/holoviews/core/data/xarray.py
+++ b/holoviews/core/data/xarray.py
@@ -14,14 +14,16 @@
try:
import cftime
- util.datetime_types += (
+ cftime_types = (
cftime._cftime.DatetimeGregorian,
cftime._cftime.Datetime360Day,
cftime._cftime.DatetimeJulian,
cftime._cftime.DatetimeNoLeap,
- cftime._cftime.DatetimeProlepticGregorian)
+ cftime._cftime.DatetimeProlepticGregorian
+ )
+ util.datetime_types += cftime_types
except:
- pass
+ cftime_types = ()
class XArrayInterface(GridInterface):
diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py
index 6b37961211..c4be4d6ea1 100644
--- a/holoviews/plotting/mpl/util.py
+++ b/holoviews/plotting/mpl/util.py
@@ -5,6 +5,7 @@
import numpy as np
import matplotlib
+from matplotlib import units as munits
from matplotlib import ticker
from matplotlib.colors import cnames
from matplotlib.lines import Line2D
@@ -14,6 +15,8 @@
from matplotlib.rcsetup import (
validate_capstyle, validate_fontsize, validate_fonttype, validate_hatch,
validate_joinstyle)
+
+from ...core.data.xarray import cftime_types
from ...core.util import LooseVersion, _getargspec, basestring, is_number
from ...element import Raster, RGB, Polygons
from ..util import COLOR_ALIASES, RGB_HEX_REGEX
@@ -334,3 +337,34 @@ def polygons_to_path_patches(element):
subpath.append(PathPatch(Path(vertices, codes)))
mpl_paths.append(subpath)
return mpl_paths
+
+try:
+ if cftime_types:
+ from nc_time_axis import NetCDFTimeConverter, CalendarDateTime
+ else:
+ import matplotlib.dates import DateConverter
+ NetCDFTimeConverter = DateConverter
+ nc_axis_available = True
+except:
+ import matplotlib.dates import DateConverter
+ NetCDFTimeConverter = DateConverter
+ nc_axis_available = False
+
+class CFTimeConverter(NetCDFTimeConverter):
+
+ @classmethod
+ def convert(cls, value, unit, axis):
+ if not nc_axis_available:
+ raise ValueError('In order to display cftime types with '
+ 'matplotlib install the nc_time_axis '
+ 'library using pip or from conda-forge '
+ 'using:\n\tconda install -c conda-forge '
+ 'nc_time_axis')
+ if isinstance(value, cftime_types):
+ value = CalendarDateTime(cftime.datetime(*value.timetuple()[0:6]), value.calendar)
+ elif isinstance(value, np.ndarray):
+ value = np.array([CalendarDateTime(cftime.datetime(*v.timetuple()[0:6]), v.calendar) for v in value])
+ return super(CFTimeConverter, cls).convert(value, unit, axis)
+
+for cft in cftime_types:
+ munits.registry[cft] = CFTimeConverter()
From 8b71267d2628d1b8f712862f208595d9fb6df160 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Wed, 5 Dec 2018 16:41:34 +0000
Subject: [PATCH 04/14] Fixed flakes
---
holoviews/plotting/bokeh/plot.py | 1 -
holoviews/plotting/mpl/util.py | 9 +++++++--
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py
index 66bcdb9bd1..640c3e43b1 100644
--- a/holoviews/plotting/bokeh/plot.py
+++ b/holoviews/plotting/bokeh/plot.py
@@ -11,7 +11,6 @@
from bokeh.models import (ColumnDataSource, Column, Row, Div)
from bokeh.models.widgets import Panel, Tabs
from bokeh.plotting.helpers import _known_tools as known_tools
-from bokeh.util.serialization import convert_datetime_array
from ...core import (OrderedDict, Store, AdjointLayout, NdLayout, Layout,
Empty, GridSpace, HoloMap, Element, DynamicMap)
diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py
index c4be4d6ea1..1847cbec15 100644
--- a/holoviews/plotting/mpl/util.py
+++ b/holoviews/plotting/mpl/util.py
@@ -342,15 +342,19 @@ def polygons_to_path_patches(element):
if cftime_types:
from nc_time_axis import NetCDFTimeConverter, CalendarDateTime
else:
- import matplotlib.dates import DateConverter
+ from matplotlib.dates import DateConverter
NetCDFTimeConverter = DateConverter
nc_axis_available = True
except:
- import matplotlib.dates import DateConverter
+ from matplotlib.dates import DateConverter
NetCDFTimeConverter = DateConverter
nc_axis_available = False
+
class CFTimeConverter(NetCDFTimeConverter):
+ """
+ Defines conversions for cftime types by extending nc_time_axis.
+ """
@classmethod
def convert(cls, value, unit, axis):
@@ -366,5 +370,6 @@ def convert(cls, value, unit, axis):
value = np.array([CalendarDateTime(cftime.datetime(*v.timetuple()[0:6]), v.calendar) for v in value])
return super(CFTimeConverter, cls).convert(value, unit, axis)
+
for cft in cftime_types:
munits.registry[cft] = CFTimeConverter()
From eb3f0c8032da0b1ddc6eb707ed057e8d4555a708 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Wed, 5 Dec 2018 17:16:35 +0000
Subject: [PATCH 05/14] Further cftime fix
---
holoviews/plotting/mpl/util.py | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py
index 1847cbec15..45d612cc5d 100644
--- a/holoviews/plotting/mpl/util.py
+++ b/holoviews/plotting/mpl/util.py
@@ -339,11 +339,8 @@ def polygons_to_path_patches(element):
return mpl_paths
try:
- if cftime_types:
- from nc_time_axis import NetCDFTimeConverter, CalendarDateTime
- else:
- from matplotlib.dates import DateConverter
- NetCDFTimeConverter = DateConverter
+ import cftime
+ from nc_time_axis import NetCDFTimeConverter, CalendarDateTime
nc_axis_available = True
except:
from matplotlib.dates import DateConverter
From 155cd82322e5a09b5e9950d4ca60599f6202711d Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 02:34:04 +0000
Subject: [PATCH 06/14] Improved handling of cftimes in bokeh
---
holoviews/plotting/bokeh/element.py | 7 ++--
holoviews/plotting/bokeh/plot.py | 31 +++++++++++------
holoviews/plotting/bokeh/util.py | 52 ++++++++++++++++++++++++-----
holoviews/plotting/mpl/element.py | 6 ++--
4 files changed, 72 insertions(+), 24 deletions(-)
diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py
index 210243893b..39a1f2cfa8 100644
--- a/holoviews/plotting/bokeh/element.py
+++ b/holoviews/plotting/bokeh/element.py
@@ -25,6 +25,7 @@
from bokeh.plotting.helpers import _known_tools as known_tools
from ...core import DynamicMap, CompositeOverlay, Element, Dimension
+from ...core.data.xarray import cftime_types
from ...core.options import abbreviated_exception, SkipRendering
from ...core import util
from ...element import Graph, VectorField, Path, Contours
@@ -606,7 +607,9 @@ def _update_ranges(self, element, ranges):
def _update_range(self, axis_range, low, high, factors, invert, shared, log, streaming=False):
if isinstance(axis_range, (Range1d, DataRange1d)) and self.apply_ranges:
- if (low == high and low is not None):
+ if isinstance(low, cftime_types):
+ pass
+ elif (low == high and low is not None):
if isinstance(low, util.datetime_types):
offset = np.timedelta64(500, 'ms')
low -= offset
@@ -634,7 +637,7 @@ def _update_range(self, axis_range, low, high, factors, invert, shared, log, str
if reset_supported:
updates['reset_end'] = updates['end']
for k, (old, new) in updates.items():
- if isinstance(new, util.datetime_types):
+ if isinstance(new, cftime_types):
new = date_to_integer(new)
axis_range.update(**{k:new})
if streaming and not k.startswith('reset_'):
diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py
index 640c3e43b1..3659a5f4ad 100644
--- a/holoviews/plotting/bokeh/plot.py
+++ b/holoviews/plotting/bokeh/plot.py
@@ -12,20 +12,29 @@
from bokeh.models.widgets import Panel, Tabs
from bokeh.plotting.helpers import _known_tools as known_tools
-from ...core import (OrderedDict, Store, AdjointLayout, NdLayout, Layout,
- Empty, GridSpace, HoloMap, Element, DynamicMap)
+from ...core import (
+ OrderedDict, Store, AdjointLayout, NdLayout, Layout, Empty,
+ GridSpace, HoloMap, Element, DynamicMap
+)
+from ...core.data.xarray import cftime_types
from ...core.options import SkipRendering
-from ...core.util import (basestring, wrap_tuple, unique_iterator,
- get_method_owner, datetime_types, wrap_tuple_streams)
+from ...core.util import (
+ basestring, wrap_tuple, unique_iterator, get_method_owner,
+ datetime_types, wrap_tuple_streams)
+
from ...streams import Stream
from ..links import Link
-from ..plot import (DimensionedPlot, GenericCompositePlot, GenericLayoutPlot,
- GenericElementPlot, GenericOverlayPlot)
+from ..plot import (
+ DimensionedPlot, GenericCompositePlot, GenericLayoutPlot,
+ GenericElementPlot, GenericOverlayPlot
+)
from ..util import attach_streams, displayable, collate
from .callbacks import LinkCallback
-from .util import (layout_padding, pad_plots, filter_toolboxes, make_axis,
- update_shared_sources, empty_plot, decode_bytes,
- theme_attr_json, cds_column_replace, date_to_integer)
+from .util import (
+ layout_padding, pad_plots, filter_toolboxes, make_axis,
+ update_shared_sources, empty_plot, decode_bytes, theme_attr_json,
+ cds_column_replace, cftime_to_timestamp
+)
TOOLS = {name: tool if isinstance(tool, basestring) else type(tool())
for name, tool in known_tools.items()}
@@ -240,8 +249,8 @@ def _postprocess_data(self, data):
values = decode_bytes(values) # Bytes need decoding to strings
# Certain datetime types need to be converted
- if len(values) and isinstance(values[0], datetime_types):
- values = np.array([date_to_integer(v) for v in values])
+ if len(values) and isinstance(values[0], cftime_types):
+ values = cftime_to_timestamp(values)
new_data[k] = values
return new_data
diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py
index 5c95c3f6dd..9e224cb75e 100644
--- a/holoviews/plotting/bokeh/util.py
+++ b/holoviews/plotting/bokeh/util.py
@@ -31,6 +31,7 @@
except:
Chart = type(None) # Create stub for isinstance check
+from ...core.data.xarray import cftime_types
from ...core.overlay import Overlay
from ...core.util import (LooseVersion, _getargspec, basestring,
callable_name, dt64_to_dt, pd, unique_array)
@@ -209,10 +210,6 @@ def make_axis(axis, size, factors, dim, flip=False, rotation=0,
return p
-def convert_datetime(time):
- return time.astype('datetime64[s]').astype(float)*1000
-
-
def hsv_to_rgb(hsv):
"""
Vectorized HSV to RGB conversion, adapted from:
@@ -537,14 +534,51 @@ def append_refresh(dmap):
return plot.hmap.traverse(append_refresh, [DynamicMap])
-def date_to_integer(date):
+def cftime_to_timestamp(date):
+ """Converts cftime to timestamp since epoch in milliseconds
+
+ Non-standard calendars (e.g. Julian or no leap calendars)
+ are converted to standard Gregorian calendar. This can cause
+ extra space to be added for dates that don't exist in the original
+ calendar. In order to handle these dates correctly a custom bokeh
+ model with support for other calendars would have to be defind.
+
+ Args:
+ date: cftime datetime object (or array)
+
+ Returns:
+ Milliseconds since 1970-01-01 00:00:00
"""
- Converts datetime types to bokeh's integer format.
+ import cftime
+ utime = cftime.utime('days since 1970-01-01')
+ return utime.date2num(date)*86400000
+
+
+def date_to_integer(date):
+ """Converts support date types to milliseconds since epoch
+
+ Attempts highest precision conversion of different datetime
+ formats to milliseconds since the epoch (1970-01-01 00:00:00).
+ If datetime is a cftime with a non-standard calendar the
+ caveats describd in cftime_to_timestamp apply.
+
+ Args:
+ date: Date- or datetime-like object
+
+ Returns:
+ Milliseconds since 1970-01-01 00:00:00
"""
+ if pd and isinstance(date, pd.Timestamp):
+ try:
+ date = date.to_datetime64()
+ except:
+ date = date.to_datetime()
+
if isinstance(date, np.datetime64):
- date = dt64_to_dt(date)
- elif pd and isinstance(date, pd.Timestamp):
- date = date.to_pydatetime()
+ return time.astype('datetime64[ms]').astype(float)
+ elif isinstance(date, cftime_types):
+ return cftime_to_timestamp(date)
+
if hasattr(date, 'timetuple'):
dt_int = calendar.timegm(date.timetuple())*1000
else:
diff --git a/holoviews/plotting/mpl/element.py b/holoviews/plotting/mpl/element.py
index 0e59114239..ab7548e9ba 100644
--- a/holoviews/plotting/mpl/element.py
+++ b/holoviews/plotting/mpl/element.py
@@ -14,6 +14,7 @@
from ...core import util
from ...core import (OrderedDict, NdOverlay, DynamicMap, Dataset,
CompositeOverlay, Element3D, Element)
+from ...core.data.xarray import cftime_types
from ...core.options import abbreviated_exception
from ...element import Graph, Path, Contours
from ...util.transform import dim
@@ -331,7 +332,8 @@ def _set_axis_limits(self, axis, view, subplots, ranges):
if self.invert_xaxis or any(p.invert_xaxis for p in subplots):
r, l = l, r
- if l != r:
+
+ if isinstance(l, cftime_types) or l != r:
lims = {}
if valid_lim(l):
lims['left'] = l
@@ -343,7 +345,7 @@ def _set_axis_limits(self, axis, view, subplots, ranges):
axis.set_xlim(**lims)
if self.invert_yaxis or any(p.invert_yaxis for p in subplots):
t, b = b, t
- if b != t:
+ if isinstance(b, cftime_types) or b != t:
lims = {}
if valid_lim(b):
lims['bottom'] = b
From 180117ae8bb88feca9b7d9ae3718875c539aaae8 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 03:00:00 +0000
Subject: [PATCH 07/14] Reorganized code
---
holoviews/core/data/xarray.py | 13 ---------
holoviews/core/util.py | 45 +++++++++++++++++++++++++++--
holoviews/plotting/bokeh/element.py | 5 ++--
holoviews/plotting/bokeh/plot.py | 8 ++---
holoviews/plotting/bokeh/util.py | 26 ++---------------
holoviews/plotting/mpl/element.py | 5 ++--
holoviews/plotting/mpl/util.py | 22 +++++++-------
7 files changed, 64 insertions(+), 60 deletions(-)
diff --git a/holoviews/core/data/xarray.py b/holoviews/core/data/xarray.py
index 03fa9f2284..ae7394f33c 100644
--- a/holoviews/core/data/xarray.py
+++ b/holoviews/core/data/xarray.py
@@ -12,19 +12,6 @@
from .grid import GridInterface
from .interface import Interface, DataError, dask_array_module
-try:
- import cftime
- cftime_types = (
- cftime._cftime.DatetimeGregorian,
- cftime._cftime.Datetime360Day,
- cftime._cftime.DatetimeJulian,
- cftime._cftime.DatetimeNoLeap,
- cftime._cftime.DatetimeProlepticGregorian
- )
- util.datetime_types += cftime_types
-except:
- cftime_types = ()
-
class XArrayInterface(GridInterface):
diff --git a/holoviews/core/util.py b/holoviews/core/util.py
index c85c2096b7..c5e7d21f60 100644
--- a/holoviews/core/util.py
+++ b/holoviews/core/util.py
@@ -92,6 +92,19 @@ def __cmp__(self, other):
except ImportError:
pd = None
+try:
+ import cftime
+ cftime_types = (
+ cftime._cftime.DatetimeGregorian,
+ cftime._cftime.Datetime360Day,
+ cftime._cftime.DatetimeJulian,
+ cftime._cftime.DatetimeNoLeap,
+ cftime._cftime.DatetimeProlepticGregorian
+ )
+ datetime_types += cftime_types
+except:
+ cftime_types = ()
+
class VersionError(Exception):
"Raised when there is a library version mismatch."
@@ -1805,8 +1818,12 @@ def dt_to_int(value, time_unit='us'):
if isinstance(value, pd.Period):
value = value.to_timestamp()
if isinstance(value, pd.Timestamp):
- value = value.to_pydatetime()
- value = np.datetime64(value)
+ try:
+ value = value.to_datetime64()
+ except:
+ value = np.datetime64(value.to_pydatetime())
+ elif isinstance(value, cftime_types):
+ return cftime_to_timestamp(value, time_unit)
if isinstance(value, np.datetime64):
value = np.datetime64(value, 'ns')
@@ -1833,6 +1850,30 @@ def dt_to_int(value, time_unit='us'):
return (time.mktime(value.timetuple()) + value.microsecond / 1e6) * tscale
+def cftime_to_timestamp(date, time_unit='us'):
+ """Converts cftime to timestamp since epoch in milliseconds
+
+ Non-standard calendars (e.g. Julian or no leap calendars)
+ are converted to standard Gregorian calendar. This can cause
+ extra space to be added for dates that don't exist in the original
+ calendar. In order to handle these dates correctly a custom bokeh
+ model with support for other calendars would have to be defind.
+
+ Args:
+ date: cftime datetime object (or array)
+
+ Returns:
+ Milliseconds since 1970-01-01 00:00:00
+ """
+ import cftime
+ utime = cftime.utime('microseconds since 1970-01-01')
+ if time_unit == 'us':
+ tscale = 1
+ else:
+ tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'us')) * 1000.
+ return utime.date2num(date)*tscale
+
+
def search_indices(values, source):
"""
Given a set of values returns the indices of each of those values
diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py
index 39a1f2cfa8..f944c1bd50 100644
--- a/holoviews/plotting/bokeh/element.py
+++ b/holoviews/plotting/bokeh/element.py
@@ -25,7 +25,6 @@
from bokeh.plotting.helpers import _known_tools as known_tools
from ...core import DynamicMap, CompositeOverlay, Element, Dimension
-from ...core.data.xarray import cftime_types
from ...core.options import abbreviated_exception, SkipRendering
from ...core import util
from ...element import Graph, VectorField, Path, Contours
@@ -607,7 +606,7 @@ def _update_ranges(self, element, ranges):
def _update_range(self, axis_range, low, high, factors, invert, shared, log, streaming=False):
if isinstance(axis_range, (Range1d, DataRange1d)) and self.apply_ranges:
- if isinstance(low, cftime_types):
+ if isinstance(low, util.cftime_types):
pass
elif (low == high and low is not None):
if isinstance(low, util.datetime_types):
@@ -637,7 +636,7 @@ def _update_range(self, axis_range, low, high, factors, invert, shared, log, str
if reset_supported:
updates['reset_end'] = updates['end']
for k, (old, new) in updates.items():
- if isinstance(new, cftime_types):
+ if isinstance(new, util.cftime_types):
new = date_to_integer(new)
axis_range.update(**{k:new})
if streaming and not k.startswith('reset_'):
diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py
index 3659a5f4ad..3eebf86692 100644
--- a/holoviews/plotting/bokeh/plot.py
+++ b/holoviews/plotting/bokeh/plot.py
@@ -16,12 +16,10 @@
OrderedDict, Store, AdjointLayout, NdLayout, Layout, Empty,
GridSpace, HoloMap, Element, DynamicMap
)
-from ...core.data.xarray import cftime_types
from ...core.options import SkipRendering
from ...core.util import (
- basestring, wrap_tuple, unique_iterator, get_method_owner,
- datetime_types, wrap_tuple_streams)
-
+ basestring, cftime_to_timestamp, cftime_types, datetime_types,
+ get_method_owner, unique_iterator, wrap_tuple, wrap_tuple_streams)
from ...streams import Stream
from ..links import Link
from ..plot import (
@@ -33,7 +31,7 @@
from .util import (
layout_padding, pad_plots, filter_toolboxes, make_axis,
update_shared_sources, empty_plot, decode_bytes, theme_attr_json,
- cds_column_replace, cftime_to_timestamp
+ cds_column_replace
)
TOOLS = {name: tool if isinstance(tool, basestring) else type(tool())
diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py
index 9e224cb75e..1e1ebbe50a 100644
--- a/holoviews/plotting/bokeh/util.py
+++ b/holoviews/plotting/bokeh/util.py
@@ -31,10 +31,10 @@
except:
Chart = type(None) # Create stub for isinstance check
-from ...core.data.xarray import cftime_types
from ...core.overlay import Overlay
-from ...core.util import (LooseVersion, _getargspec, basestring,
- callable_name, dt64_to_dt, pd, unique_array)
+from ...core.util import (
+ LooseVersion, _getargspec, basestring, callable_name, cftime_types,
+ cftime_to_timestamp, dt64_to_dt, pd, unique_array)
from ...core.spaces import get_nested_dmaps, DynamicMap
from ..util import dim_axis_label
@@ -534,26 +534,6 @@ def append_refresh(dmap):
return plot.hmap.traverse(append_refresh, [DynamicMap])
-def cftime_to_timestamp(date):
- """Converts cftime to timestamp since epoch in milliseconds
-
- Non-standard calendars (e.g. Julian or no leap calendars)
- are converted to standard Gregorian calendar. This can cause
- extra space to be added for dates that don't exist in the original
- calendar. In order to handle these dates correctly a custom bokeh
- model with support for other calendars would have to be defind.
-
- Args:
- date: cftime datetime object (or array)
-
- Returns:
- Milliseconds since 1970-01-01 00:00:00
- """
- import cftime
- utime = cftime.utime('days since 1970-01-01')
- return utime.date2num(date)*86400000
-
-
def date_to_integer(date):
"""Converts support date types to milliseconds since epoch
diff --git a/holoviews/plotting/mpl/element.py b/holoviews/plotting/mpl/element.py
index ab7548e9ba..87689a7924 100644
--- a/holoviews/plotting/mpl/element.py
+++ b/holoviews/plotting/mpl/element.py
@@ -14,7 +14,6 @@
from ...core import util
from ...core import (OrderedDict, NdOverlay, DynamicMap, Dataset,
CompositeOverlay, Element3D, Element)
-from ...core.data.xarray import cftime_types
from ...core.options import abbreviated_exception
from ...element import Graph, Path, Contours
from ...util.transform import dim
@@ -333,7 +332,7 @@ def _set_axis_limits(self, axis, view, subplots, ranges):
if self.invert_xaxis or any(p.invert_xaxis for p in subplots):
r, l = l, r
- if isinstance(l, cftime_types) or l != r:
+ if isinstance(l, util.cftime_types) or l != r:
lims = {}
if valid_lim(l):
lims['left'] = l
@@ -345,7 +344,7 @@ def _set_axis_limits(self, axis, view, subplots, ranges):
axis.set_xlim(**lims)
if self.invert_yaxis or any(p.invert_yaxis for p in subplots):
t, b = b, t
- if isinstance(b, cftime_types) or b != t:
+ if isinstance(b, util.cftime_types) or b != t:
lims = {}
if valid_lim(b):
lims['bottom'] = b
diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py
index 45d612cc5d..d1a71191d8 100644
--- a/holoviews/plotting/mpl/util.py
+++ b/holoviews/plotting/mpl/util.py
@@ -16,8 +16,17 @@
validate_capstyle, validate_fontsize, validate_fonttype, validate_hatch,
validate_joinstyle)
-from ...core.data.xarray import cftime_types
-from ...core.util import LooseVersion, _getargspec, basestring, is_number
+try:
+ import cftime
+ from nc_time_axis import NetCDFTimeConverter, CalendarDateTime
+ nc_axis_available = True
+except:
+ from matplotlib.dates import DateConverter
+ NetCDFTimeConverter = DateConverter
+ nc_axis_available = False
+
+from ...core.util import (
+ LooseVersion, _getargspec, basestring, cftime_types, is_number)
from ...element import Raster, RGB, Polygons
from ..util import COLOR_ALIASES, RGB_HEX_REGEX
@@ -338,15 +347,6 @@ def polygons_to_path_patches(element):
mpl_paths.append(subpath)
return mpl_paths
-try:
- import cftime
- from nc_time_axis import NetCDFTimeConverter, CalendarDateTime
- nc_axis_available = True
-except:
- from matplotlib.dates import DateConverter
- NetCDFTimeConverter = DateConverter
- nc_axis_available = False
-
class CFTimeConverter(NetCDFTimeConverter):
"""
From 5aad6f2023461ba8895c11404962b29abf03d008 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 13:33:17 +0000
Subject: [PATCH 08/14] Various datetime handling fixes
---
holoviews/core/util.py | 12 +++---------
holoviews/plotting/bokeh/element.py | 4 ++--
holoviews/plotting/bokeh/plot.py | 2 +-
holoviews/plotting/bokeh/util.py | 4 ++--
holoviews/plotting/mpl/util.py | 2 +-
5 files changed, 9 insertions(+), 15 deletions(-)
diff --git a/holoviews/core/util.py b/holoviews/core/util.py
index c5e7d21f60..40883e033b 100644
--- a/holoviews/core/util.py
+++ b/holoviews/core/util.py
@@ -94,13 +94,7 @@ def __cmp__(self, other):
try:
import cftime
- cftime_types = (
- cftime._cftime.DatetimeGregorian,
- cftime._cftime.Datetime360Day,
- cftime._cftime.DatetimeJulian,
- cftime._cftime.DatetimeNoLeap,
- cftime._cftime.DatetimeProlepticGregorian
- )
+ cftime_types = (cftime.datetime,)
datetime_types += cftime_types
except:
cftime_types = ()
@@ -1857,7 +1851,7 @@ def cftime_to_timestamp(date, time_unit='us'):
are converted to standard Gregorian calendar. This can cause
extra space to be added for dates that don't exist in the original
calendar. In order to handle these dates correctly a custom bokeh
- model with support for other calendars would have to be defind.
+ model with support for other calendars would have to be defined.
Args:
date: cftime datetime object (or array)
@@ -1870,7 +1864,7 @@ def cftime_to_timestamp(date, time_unit='us'):
if time_unit == 'us':
tscale = 1
else:
- tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'us')) * 1000.
+ tscale = (np.timedelta64(1, 'us')/np.timedelta64(1, time_unit))
return utime.date2num(date)*tscale
diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py
index f944c1bd50..f0b3f488c9 100644
--- a/holoviews/plotting/bokeh/element.py
+++ b/holoviews/plotting/bokeh/element.py
@@ -301,7 +301,7 @@ def _axes_props(self, plots, subplots, element, ranges):
xtype = el.nodes.get_dimension_type(xdims[0])
else:
xtype = el.get_dimension_type(xdims[0])
- if ((xtype is np.object_ and type(l) in util.datetime_types) or
+ if ((xtype is np.object_ and issubclass(type(l), util.datetime_types)) or
xtype in util.datetime_types):
x_axis_type = 'datetime'
@@ -315,7 +315,7 @@ def _axes_props(self, plots, subplots, element, ranges):
ytype = el.nodes.get_dimension_type(ydims[0])
else:
ytype = el.get_dimension_type(ydims[0])
- if ((ytype is np.object_ and type(b) in util.datetime_types)
+ if ((ytype is np.object_ and issubclass(type(b), util.datetime_types))
or ytype in util.datetime_types):
y_axis_type = 'datetime'
diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py
index 3eebf86692..1ee2f9c211 100644
--- a/holoviews/plotting/bokeh/plot.py
+++ b/holoviews/plotting/bokeh/plot.py
@@ -248,7 +248,7 @@ def _postprocess_data(self, data):
# Certain datetime types need to be converted
if len(values) and isinstance(values[0], cftime_types):
- values = cftime_to_timestamp(values)
+ values = cftime_to_timestamp(values, 'ms')
new_data[k] = values
return new_data
diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py
index 1e1ebbe50a..ca443c0b69 100644
--- a/holoviews/plotting/bokeh/util.py
+++ b/holoviews/plotting/bokeh/util.py
@@ -540,7 +540,7 @@ def date_to_integer(date):
Attempts highest precision conversion of different datetime
formats to milliseconds since the epoch (1970-01-01 00:00:00).
If datetime is a cftime with a non-standard calendar the
- caveats describd in cftime_to_timestamp apply.
+ caveats described in hv.core.util.cftime_to_timestamp apply.
Args:
date: Date- or datetime-like object
@@ -557,7 +557,7 @@ def date_to_integer(date):
if isinstance(date, np.datetime64):
return time.astype('datetime64[ms]').astype(float)
elif isinstance(date, cftime_types):
- return cftime_to_timestamp(date)
+ return cftime_to_timestamp(date, 'ms')
if hasattr(date, 'timetuple'):
dt_int = calendar.timegm(date.timetuple())*1000
diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py
index d1a71191d8..da2fd72d0a 100644
--- a/holoviews/plotting/mpl/util.py
+++ b/holoviews/plotting/mpl/util.py
@@ -364,7 +364,7 @@ def convert(cls, value, unit, axis):
if isinstance(value, cftime_types):
value = CalendarDateTime(cftime.datetime(*value.timetuple()[0:6]), value.calendar)
elif isinstance(value, np.ndarray):
- value = np.array([CalendarDateTime(cftime.datetime(*v.timetuple()[0:6]), v.calendar) for v in value])
+ value = np.array([CalendarDateTime(v.datetime, v.calendar) for v in value])
return super(CFTimeConverter, cls).convert(value, unit, axis)
From bef18f65be53b0cc9079bda4e5b9f3100ceb48e7 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 14:10:26 +0000
Subject: [PATCH 09/14] Overhauled datetime handling
---
holoviews/core/util.py | 7 +++--
holoviews/operation/datashader.py | 47 ++++++++++++++++++-------------
2 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/holoviews/core/util.py b/holoviews/core/util.py
index 40883e033b..4dbdebb377 100644
--- a/holoviews/core/util.py
+++ b/holoviews/core/util.py
@@ -1824,18 +1824,19 @@ def dt_to_int(value, time_unit='us'):
if time_unit == 'ns':
tscale = 1
else:
- tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'ns')) * 1000.
+ tscale = (np.timedelta64(1, 'ns')/np.timedelta64(1, time_unit))
elif time_unit == 'ns':
- tscale = 1000.
+ tscale = 1e9
else:
tscale = 1./np.timedelta64(1, time_unit).tolist().total_seconds()
if isinstance(value, np.datetime64):
value = value.tolist()
+
if isinstance(value, (int, long)):
# Handle special case of nanosecond precision which cannot be
# represented by python datetime
- return value * 10**-(np.log10(tscale)-3)
+ return value * 10**-(np.log10(tscale))
try:
# Handle python3
return int(value.timestamp() * tscale)
diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py
index af671ec549..760f0fea44 100644
--- a/holoviews/operation/datashader.py
+++ b/holoviews/operation/datashader.py
@@ -23,7 +23,9 @@
CompositeOverlay, Dataset, Overlay)
from ..core.data import PandasInterface, XArrayInterface
from ..core.sheetcoords import BoundingBox
-from ..core.util import LooseVersion, get_param_values, basestring, datetime_types, dt_to_int
+from ..core.util import (
+ LooseVersion, basestring, cftime_types, cftime_to_timestamp,
+ datetime_types, dt_to_int, get_param_values)
from ..element import (Image, Path, Curve, RGB, Graph, TriMesh, QuadMesh, Contours)
from ..streams import RangeXY, PlotSize
@@ -321,11 +323,18 @@ def get_agg_data(cls, obj, category=None):
if category and df[category].dtype.name != 'category':
df[category] = df[category].astype('category')
- if any(df[d.name].dtype.kind == 'M' for d in (x, y)):
+ if any(df[d.name].dtype.kind == 'M' or isinstance(df[d.name].values[0], cftime_types)
+ for d in (x, y)):
df = df.copy()
for d in (x, y):
- if df[d.name].dtype.kind == 'M':
- df[d.name] = df[d.name].astype('datetime64[ns]').astype('int64') * 1000.
+ vals = df[d.name].values
+ if len(vals) and isinstance(vals[0], cftime_types):
+ vals = cftime_to_timestamp(vals, 'ns')
+ elif df[d.name].dtype.kind == 'M':
+ vals = vals.astype('datetime64[ns]')
+ else:
+ continue
+ df[d.name] = vals.astype('int64')
return x, y, Dataset(df, kdims=kdims, vdims=vdims), glyph
@@ -344,9 +353,9 @@ def _aggregate_ndoverlay(self, element, agg_fn):
info = self._get_sampling(element, x, y)
(x_range, y_range), (xs, ys), (width, height), (xtype, ytype) = info
if xtype == 'datetime':
- x_range = tuple((np.array(x_range)/10e5).astype('datetime64[us]'))
+ x_range = tuple((np.array(x_range)/1e3).astype('datetime64[us]'))
if ytype == 'datetime':
- y_range = tuple((np.array(y_range)/10e5).astype('datetime64[us]'))
+ y_range = tuple((np.array(y_range)/1e3).astype('datetime64[us]'))
agg_params = dict({k: v for k, v in dict(self.get_param_values(), **self.p).items()
if k in aggregate.params()},
x_range=x_range, y_range=y_range)
@@ -433,11 +442,11 @@ def _process(self, element, key=None):
(x0, x1), (y0, y1) = x_range, y_range
if xtype == 'datetime':
- x0, x1 = (np.array([x0, x1])/10e5).astype('datetime64[us]')
- xs = (xs/10e5).astype('datetime64[us]')
+ x0, x1 = (np.array([x0, x1])/1e3).astype('datetime64[us]')
+ xs = (xs/1e3).astype('datetime64[us]')
if ytype == 'datetime':
- y0, y1 = (np.array([y0, y1])/10e5).astype('datetime64[us]')
- ys = (ys/10e5).astype('datetime64[us]')
+ y0, y1 = (np.array([y0, y1])/1e3).astype('datetime64[us]')
+ ys = (ys/1e3).astype('datetime64[us]')
bounds = (x0, y0, x1, y1)
params = dict(get_param_values(element), kdims=[x, y],
datatype=['xarray'], bounds=bounds)
@@ -483,9 +492,9 @@ def _process(self, element, key=None):
if 'x_axis' in agg.coords and 'y_axis' in agg.coords:
agg = agg.rename({'x_axis': x, 'y_axis': y})
if xtype == 'datetime':
- agg[x.name] = (agg[x.name]/10e5).astype('datetime64[us]')
+ agg[x.name] = (agg[x.name]/1e3).astype('datetime64[us]')
if ytype == 'datetime':
- agg[y.name] = (agg[y.name]/10e5).astype('datetime64[us]')
+ agg[y.name] = (agg[y.name]/1e3).astype('datetime64[us]')
if agg.ndim == 2:
# Replacing x and y coordinates to avoid numerical precision issues
@@ -606,11 +615,11 @@ def _process(self, element, key=None):
# Compute bounds (converting datetimes)
if xtype == 'datetime':
- xstart, xend = (np.array([xstart, xend])/10e5).astype('datetime64[us]')
- xs = (xs/10e5).astype('datetime64[us]')
+ xstart, xend = (np.array([xstart, xend])/1e3).astype('datetime64[us]')
+ xs = (xs/1e3).astype('datetime64[us]')
if ytype == 'datetime':
- ystart, yend = (np.array([ystart, yend])/10e5).astype('datetime64[us]')
- ys = (ys/10e5).astype('datetime64[us]')
+ ystart, yend = (np.array([ystart, yend])/1e3).astype('datetime64[us]')
+ ys = (ys/1e3).astype('datetime64[us]')
bbox = BoundingBox(points=[(xstart, ystart), (xend, yend)])
params = dict(bounds=bbox)
@@ -632,9 +641,9 @@ def _process(self, element, key=None):
# Convert datetime coordinates
if xtype == "datetime":
- rarray[x.name] = (rarray[x.name]/10e5).astype('datetime64[us]')
+ rarray[x.name] = (rarray[x.name]/1e3).astype('datetime64[us]')
if ytype == "datetime":
- rarray[y.name] = (rarray[y.name]/10e5).astype('datetime64[us]')
+ rarray[y.name] = (rarray[y.name]/1e3).astype('datetime64[us]')
regridded[vd] = rarray
regridded = xr.Dataset(regridded)
@@ -957,7 +966,7 @@ def _process(self, element, key=None):
for d in kdims:
if array[d.name].dtype.kind == 'M':
- array[d.name] = array[d.name].astype('datetime64[ns]').astype('int64') * 10e-4
+ array[d.name] = array[d.name].astype('datetime64[us]').astype('int64')
with warnings.catch_warnings():
warnings.filterwarnings('ignore', r'invalid value encountered in true_divide')
From 5eee4b6e86ef960d4679aa53f5b6dca6d892cb98 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 14:19:11 +0000
Subject: [PATCH 10/14] Added warning about non-standard calendars
---
holoviews/core/util.py | 1 +
holoviews/plotting/bokeh/plot.py | 10 +++++++++-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/holoviews/core/util.py b/holoviews/core/util.py
index 4dbdebb377..ca37a5b05e 100644
--- a/holoviews/core/util.py
+++ b/holoviews/core/util.py
@@ -98,6 +98,7 @@ def __cmp__(self, other):
datetime_types += cftime_types
except:
cftime_types = ()
+_STANDARD_CALENDARS = set(['standard', 'gregorian', 'proleptic_gregorian'])
class VersionError(Exception):
diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py
index 1ee2f9c211..fb33142ebc 100644
--- a/holoviews/plotting/bokeh/plot.py
+++ b/holoviews/plotting/bokeh/plot.py
@@ -19,7 +19,8 @@
from ...core.options import SkipRendering
from ...core.util import (
basestring, cftime_to_timestamp, cftime_types, datetime_types,
- get_method_owner, unique_iterator, wrap_tuple, wrap_tuple_streams)
+ get_method_owner, unique_iterator, wrap_tuple, wrap_tuple_streams,
+ _STANDARD_CALENDARS)
from ...streams import Stream
from ..links import Link
from ..plot import (
@@ -248,6 +249,13 @@ def _postprocess_data(self, data):
# Certain datetime types need to be converted
if len(values) and isinstance(values[0], cftime_types):
+ if any(v.calendar not in _STANDARD_CALENDARS for v in values):
+ self.param.warning(
+ 'Converting cftime.datetime from a non-standard '
+ 'calendar (%s) to a standard calendar for plotting. '
+ 'This may lead to subtle errors in formatting '
+ 'dates, for accurate tick formatting switch to '
+ 'the matplotlib backend.' % values[0].calendar)
values = cftime_to_timestamp(values, 'ms')
new_data[k] = values
return new_data
From 4da46b8285867cdc6d0d26dbc7c4c6e64e1d8377 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 14:24:41 +0000
Subject: [PATCH 11/14] Further fix
---
holoviews/plotting/mpl/util.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py
index da2fd72d0a..5b6aa18877 100644
--- a/holoviews/plotting/mpl/util.py
+++ b/holoviews/plotting/mpl/util.py
@@ -362,7 +362,7 @@ def convert(cls, value, unit, axis):
'using:\n\tconda install -c conda-forge '
'nc_time_axis')
if isinstance(value, cftime_types):
- value = CalendarDateTime(cftime.datetime(*value.timetuple()[0:6]), value.calendar)
+ value = CalendarDateTime(v.datetime, value.calendar)
elif isinstance(value, np.ndarray):
value = np.array([CalendarDateTime(v.datetime, v.calendar) for v in value])
return super(CFTimeConverter, cls).convert(value, unit, axis)
From e35461218a07fe5a33c1081bf7b81d7fb626af92 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 14:33:39 +0000
Subject: [PATCH 12/14] Added unit tests for cftime conversions
---
.../tests/plotting/bokeh/testelementplot.py | 37 ++++++++++++++++++-
1 file changed, 35 insertions(+), 2 deletions(-)
diff --git a/holoviews/tests/plotting/bokeh/testelementplot.py b/holoviews/tests/plotting/bokeh/testelementplot.py
index 62112e4bad..ac8bece0fe 100644
--- a/holoviews/tests/plotting/bokeh/testelementplot.py
+++ b/holoviews/tests/plotting/bokeh/testelementplot.py
@@ -10,6 +10,7 @@
from holoviews.plotting.util import process_cmap
from .testplot import TestBokehPlot, bokeh_renderer
+from ...utils import LoggingComparisonTestCase
try:
from bokeh.document import Document
@@ -19,7 +20,7 @@
-class TestElementPlot(TestBokehPlot):
+class TestElementPlot(LoggingComparisonTestCase, TestBokehPlot):
def test_element_show_frame_disabled(self):
curve = Curve(range(10)).opts(plot=dict(show_frame=False))
@@ -287,7 +288,6 @@ def test_categorical_axis_fontsize(self):
curve = Curve([('A', 1), ('B', 2)]).options(fontsize={'minor_xticks': '6pt', 'xticks': 18})
plot = bokeh_renderer.get_plot(curve)
xaxis = plot.handles['xaxis']
- print(xaxis.properties_with_values())
self.assertEqual(xaxis.major_label_text_font_size, '6pt')
self.assertEqual(xaxis.group_text_font_size, {'value': '18pt'})
@@ -298,6 +298,39 @@ def test_categorical_axis_fontsize_both(self):
self.assertEqual(xaxis.major_label_text_font_size, {'value': '18pt'})
self.assertEqual(xaxis.group_text_font_size, {'value': '18pt'})
+ def test_cftime_transform_gregorian_no_warn(self):
+ try:
+ import cftime
+ except:
+ raise SkipTest('Test requires cftime library')
+ gregorian_dates = [cftime.DatetimeGregorian(2000, 2, 28),
+ cftime.DatetimeGregorian(2000, 3, 1),
+ cftime.DatetimeGregorian(2000, 3, 2)]
+ curve = Curve((gregorian_dates, [1, 2, 3]))
+ plot = bokeh_renderer.get_plot(curve)
+ xs = plot.handles['cds'].data['x']
+ self.assertEqual(xs.astype('int'),
+ np.array([951696000000, 951868800000, 951955200000]))
+
+ def test_cftime_transform_noleap_warn(self):
+ try:
+ import cftime
+ except:
+ raise SkipTest('Test requires cftime library')
+ gregorian_dates = [cftime.DatetimeNoLeap(2000, 2, 28),
+ cftime.DatetimeNoLeap(2000, 3, 1),
+ cftime.DatetimeNoLeap(2000, 3, 2)]
+ curve = Curve((gregorian_dates, [1, 2, 3]))
+ plot = bokeh_renderer.get_plot(curve)
+ xs = plot.handles['cds'].data['x']
+ self.assertEqual(xs.astype('int'),
+ np.array([951696000000, 951868800000, 951955200000]))
+ substr = (
+ "Converting cftime.datetime from a non-standard calendar "
+ "(noleap) to a standard calendar for plotting. This may "
+ "lead to subtle errors in formatting dates, for accurate "
+ "tick formatting switch to the matplotlib backend.")
+ self.log_handler.assertEndsWith('WARNING', substr)
class TestColorbarPlot(TestBokehPlot):
From 49e3b769c1aea48b57dc56ba02f6af5efed272c5 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 16:34:55 +0000
Subject: [PATCH 13/14] Further fixes for datetime handling
---
holoviews/core/util.py | 23 ++++++------
holoviews/operation/element.py | 6 ++--
holoviews/tests/core/testutils.py | 6 ++--
holoviews/tests/operation/testoperation.py | 42 ++++++++++++++--------
4 files changed, 43 insertions(+), 34 deletions(-)
diff --git a/holoviews/core/util.py b/holoviews/core/util.py
index ca37a5b05e..29573568f4 100644
--- a/holoviews/core/util.py
+++ b/holoviews/core/util.py
@@ -1820,24 +1820,21 @@ def dt_to_int(value, time_unit='us'):
elif isinstance(value, cftime_types):
return cftime_to_timestamp(value, time_unit)
+ # Handle datetime64 separately
if isinstance(value, np.datetime64):
- value = np.datetime64(value, 'ns')
- if time_unit == 'ns':
- tscale = 1
- else:
- tscale = (np.timedelta64(1, 'ns')/np.timedelta64(1, time_unit))
- elif time_unit == 'ns':
+ try:
+ value = np.datetime64(value, 'ns')
+ tscale = (np.timedelta64(1, time_unit)/np.timedelta64(1, 'ns'))
+ return value.tolist()/tscale
+ except:
+ # If it can't handle ns precision fall back to datetime
+ value = value.tolist()
+
+ if time_unit == 'ns':
tscale = 1e9
else:
tscale = 1./np.timedelta64(1, time_unit).tolist().total_seconds()
- if isinstance(value, np.datetime64):
- value = value.tolist()
-
- if isinstance(value, (int, long)):
- # Handle special case of nanosecond precision which cannot be
- # represented by python datetime
- return value * 10**-(np.log10(tscale))
try:
# Handle python3
return int(value.timestamp() * tscale)
diff --git a/holoviews/operation/element.py b/holoviews/operation/element.py
index 5bb20b10ae..80a9d90685 100644
--- a/holoviews/operation/element.py
+++ b/holoviews/operation/element.py
@@ -589,9 +589,9 @@ def _process(self, element, key=None):
if data.dtype.kind == 'M' or (data.dtype.kind == 'O' and isinstance(data[0], datetime_types)):
start, end = dt_to_int(start, 'ns'), dt_to_int(end, 'ns')
datetimes = True
- data = data.astype('datetime64[ns]').astype('int64') * 1000.
+ data = data.astype('datetime64[ns]').astype('int64')
if bins is not None:
- bins = bins.astype('datetime64[ns]').astype('int64') * 1000.
+ bins = bins.astype('datetime64[ns]').astype('int64')
else:
hist_range = start, end
@@ -622,7 +622,7 @@ def _process(self, element, key=None):
hist = np.zeros(self.p.num_bins)
hist[np.isnan(hist)] = 0
if datetimes:
- edges = (edges/10e5).astype('datetime64[us]')
+ edges = (edges/1e3).astype('datetime64[us]')
params = {}
if self.p.weight_dimension:
diff --git a/holoviews/tests/core/testutils.py b/holoviews/tests/core/testutils.py
index bd37d5d93f..097a1dceb3 100644
--- a/holoviews/tests/core/testutils.py
+++ b/holoviews/tests/core/testutils.py
@@ -584,15 +584,15 @@ def test_datetime_to_us_int(self):
def test_datetime64_s_to_ns_int(self):
dt = np.datetime64(datetime.datetime(2017, 1, 1), 's')
- self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000000.0)
+ self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000.0)
def test_datetime64_us_to_ns_int(self):
dt = np.datetime64(datetime.datetime(2017, 1, 1), 'us')
- self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000000.0)
+ self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000.0)
def test_datetime64_to_ns_int(self):
dt = np.datetime64(datetime.datetime(2017, 1, 1))
- self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000000.0)
+ self.assertEqual(dt_to_int(dt, 'ns'), 1483228800000000000.0)
def test_datetime64_us_to_us_int(self):
dt = np.datetime64(datetime.datetime(2017, 1, 1), 'us')
diff --git a/holoviews/tests/operation/testoperation.py b/holoviews/tests/operation/testoperation.py
index 1f1ed744ca..e76770f5e2 100644
--- a/holoviews/tests/operation/testoperation.py
+++ b/holoviews/tests/operation/testoperation.py
@@ -168,22 +168,30 @@ def test_points_histogram_not_normed(self):
def test_histogram_operation_datetime(self):
dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)])
op_hist = histogram(Dataset(dates, 'Date'), num_bins=4)
- hist_data = {'Date': np.array(['2017-01-01T00:00:00.000000', '2017-01-01T17:59:59.999999',
- '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000',
- '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'),
- 'Date_frequency': np.array([ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18,
- 3.85802469e-18])}
+ hist_data = {
+ 'Date': np.array([
+ '2017-01-01T00:00:00.000000', '2017-01-01T18:00:00.000000',
+ '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000',
+ '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'),
+ 'Date_frequency': np.array([
+ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18,
+ 3.85802469e-18])
+ }
hist = Histogram(hist_data, kdims='Date', vdims=('Date_frequency', 'Frequency'))
self.assertEqual(op_hist, hist)
def test_histogram_operation_datetime64(self):
dates = np.array([dt.datetime(2017, 1, i) for i in range(1, 5)]).astype('M')
op_hist = histogram(Dataset(dates, 'Date'), num_bins=4)
- hist_data = {'Date': np.array(['2017-01-01T00:00:00.000000', '2017-01-01T17:59:59.999999',
- '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000',
- '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'),
- 'Date_frequency': np.array([ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18,
- 3.85802469e-18])}
+ hist_data = {
+ 'Date': np.array([
+ '2017-01-01T00:00:00.000000', '2017-01-01T18:00:00.000000',
+ '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000',
+ '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'),
+ 'Date_frequency': np.array([
+ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18,
+ 3.85802469e-18])
+ }
hist = Histogram(hist_data, kdims='Date', vdims=('Date_frequency', 'Frequency'))
self.assertEqual(op_hist, hist)
@@ -191,11 +199,15 @@ def test_histogram_operation_datetime64(self):
def test_histogram_operation_pd_period(self):
dates = pd.date_range('2017-01-01', '2017-01-04', freq='D').to_period('D')
op_hist = histogram(Dataset(dates, 'Date'), num_bins=4)
- hist_data = {'Date': np.array(['2017-01-01T00:00:00.000000', '2017-01-01T17:59:59.999999',
- '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000',
- '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'),
- 'Date_frequency': np.array([ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18,
- 3.85802469e-18])}
+ hist_data = {
+ 'Date': np.array([
+ '2017-01-01T00:00:00.000000', '2017-01-01T18:00:00.000000',
+ '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000',
+ '2017-01-04T00:00:00.000000'], dtype='datetime64[us]'),
+ 'Date_frequency': np.array([
+ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18,
+ 3.85802469e-18])
+ }
hist = Histogram(hist_data, kdims='Date', vdims=('Date_frequency', 'Frequency'))
self.assertEqual(op_hist, hist)
From 23fabdd8c362f2e0207c8f85cc3f0faffa4e6aaa Mon Sep 17 00:00:00 2001
From: Philipp Rudiger
Date: Thu, 6 Dec 2018 17:24:06 +0000
Subject: [PATCH 14/14] Fixed flakes
---
holoviews/plotting/bokeh/plot.py | 5 ++---
holoviews/plotting/bokeh/util.py | 2 +-
holoviews/plotting/mpl/util.py | 3 +--
.../tests/plotting/bokeh/testhistogramplot.py | 20 ++++++++++++-------
4 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/holoviews/plotting/bokeh/plot.py b/holoviews/plotting/bokeh/plot.py
index fb33142ebc..f7c745ee88 100644
--- a/holoviews/plotting/bokeh/plot.py
+++ b/holoviews/plotting/bokeh/plot.py
@@ -18,9 +18,8 @@
)
from ...core.options import SkipRendering
from ...core.util import (
- basestring, cftime_to_timestamp, cftime_types, datetime_types,
- get_method_owner, unique_iterator, wrap_tuple, wrap_tuple_streams,
- _STANDARD_CALENDARS)
+ basestring, cftime_to_timestamp, cftime_types, get_method_owner,
+ unique_iterator, wrap_tuple, wrap_tuple_streams, _STANDARD_CALENDARS)
from ...streams import Stream
from ..links import Link
from ..plot import (
diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py
index ca443c0b69..94063666d9 100644
--- a/holoviews/plotting/bokeh/util.py
+++ b/holoviews/plotting/bokeh/util.py
@@ -34,7 +34,7 @@
from ...core.overlay import Overlay
from ...core.util import (
LooseVersion, _getargspec, basestring, callable_name, cftime_types,
- cftime_to_timestamp, dt64_to_dt, pd, unique_array)
+ cftime_to_timestamp, pd, unique_array)
from ...core.spaces import get_nested_dmaps, DynamicMap
from ..util import dim_axis_label
diff --git a/holoviews/plotting/mpl/util.py b/holoviews/plotting/mpl/util.py
index 5b6aa18877..701744afa6 100644
--- a/holoviews/plotting/mpl/util.py
+++ b/holoviews/plotting/mpl/util.py
@@ -17,7 +17,6 @@
validate_joinstyle)
try:
- import cftime
from nc_time_axis import NetCDFTimeConverter, CalendarDateTime
nc_axis_available = True
except:
@@ -362,7 +361,7 @@ def convert(cls, value, unit, axis):
'using:\n\tconda install -c conda-forge '
'nc_time_axis')
if isinstance(value, cftime_types):
- value = CalendarDateTime(v.datetime, value.calendar)
+ value = CalendarDateTime(value.datetime, value.calendar)
elif isinstance(value, np.ndarray):
value = np.array([CalendarDateTime(v.datetime, v.calendar) for v in value])
return super(CFTimeConverter, cls).convert(value, unit, axis)
diff --git a/holoviews/tests/plotting/bokeh/testhistogramplot.py b/holoviews/tests/plotting/bokeh/testhistogramplot.py
index c1f5a28f64..e405daa790 100644
--- a/holoviews/tests/plotting/bokeh/testhistogramplot.py
+++ b/holoviews/tests/plotting/bokeh/testhistogramplot.py
@@ -59,13 +59,19 @@ def test_histogram_datetime64_plot(self):
hist = histogram(Dataset(dates, 'Date'), num_bins=4)
plot = bokeh_renderer.get_plot(hist)
source = plot.handles['source']
- data = {'top': np.array([ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, 3.85802469e-18]),
- 'left': np.array(['2017-01-01T00:00:00.000000', '2017-01-01T17:59:59.999999',
- '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000'],
- dtype='datetime64[us]'),
- 'right': np.array(['2017-01-01T17:59:59.999999', '2017-01-02T12:00:00.000000',
- '2017-01-03T06:00:00.000000', '2017-01-04T00:00:00.000000'],
- dtype='datetime64[us]')}
+ print(source.data)
+ data = {
+ 'top': np.array([
+ 3.85802469e-18, 3.85802469e-18, 3.85802469e-18, 3.85802469e-18]),
+ 'left': np.array([
+ '2017-01-01T00:00:00.000000', '2017-01-01T18:00:00.000000',
+ '2017-01-02T12:00:00.000000', '2017-01-03T06:00:00.000000'],
+ dtype='datetime64[us]'),
+ 'right': np.array([
+ '2017-01-01T18:00:00.000000', '2017-01-02T12:00:00.000000',
+ '2017-01-03T06:00:00.000000', '2017-01-04T00:00:00.000000'],
+ dtype='datetime64[us]')
+ }
for k, v in data.items():
self.assertEqual(source.data[k], v)
xaxis = plot.handles['xaxis']