From efea3752a751d6a4d2de9386fe17483b04405cc7 Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 27 Sep 2016 12:06:09 -0700 Subject: [PATCH 1/5] Monitoring - Look for specific messages in retries --- system_tests/monitoring.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/system_tests/monitoring.py b/system_tests/monitoring.py index 7305ae2f61bf..dc833924f433 100644 --- a/system_tests/monitoring.py +++ b/system_tests/monitoring.py @@ -207,9 +207,14 @@ def _query_timeseries_with_retries(): def _has_timeseries(result): return len(list(result)) > 0 + def _unknown_metric(result): + return ('The provided filter doesn\'t refer to any known ' + 'metric.'in result.message) + retry_result = RetryResult(_has_timeseries, max_tries=MAX_RETRIES)(client.query) - return RetryErrors(BadRequest, max_tries=MAX_RETRIES)(retry_result) + return RetryErrors(BadRequest, _unknown_metric, + max_tries=MAX_RETRIES)(retry_result) query = _query_timeseries_with_retries()(METRIC_TYPE, minutes=5) timeseries_list = list(query) From 08ca167cff4f9b46aaa6313189e3495b7b8885bd Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 27 Sep 2016 15:11:40 -0700 Subject: [PATCH 2/5] review comments --- system_tests/monitoring.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/system_tests/monitoring.py b/system_tests/monitoring.py index dc833924f433..78972431dace 100644 --- a/system_tests/monitoring.py +++ b/system_tests/monitoring.py @@ -29,6 +29,21 @@ retry_500 = RetryErrors(InternalServerError) retry_503 = RetryErrors(ServiceUnavailable) +# Retry predicates + + +def _has_timeseries(result): + """Return True if a time series query has non-empty results.""" + return len(list(result)) > 0 + +UNKNOWN_METRIC_ERROR = ('The provided filter doesn\'t refer to any known ' + 'metric.') + + +def _unknown_metric(result): + """Return True if the error describes writing to an unknown metric..""" + return UNKNOWN_METRIC_ERROR in result.message + class TestMonitoring(unittest.TestCase): @@ -203,17 +218,9 @@ def test_write_point(self): def _query_timeseries_with_retries(): MAX_RETRIES = 7 - - def _has_timeseries(result): - return len(list(result)) > 0 - - def _unknown_metric(result): - return ('The provided filter doesn\'t refer to any known ' - 'metric.'in result.message) - retry_result = RetryResult(_has_timeseries, max_tries=MAX_RETRIES)(client.query) - return RetryErrors(BadRequest, _unknown_metric, + return RetryErrors(BadRequest, error_predicate=_unknown_metric, max_tries=MAX_RETRIES)(retry_result) query = _query_timeseries_with_retries()(METRIC_TYPE, minutes=5) From 4206d2dbe1f9cd4c6807906fb23dff3e994e2b1c Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 27 Sep 2016 15:51:54 -0700 Subject: [PATCH 3/5] review comments --- system_tests/monitoring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/monitoring.py b/system_tests/monitoring.py index 78972431dace..16d342bfaa4b 100644 --- a/system_tests/monitoring.py +++ b/system_tests/monitoring.py @@ -29,8 +29,6 @@ retry_500 = RetryErrors(InternalServerError) retry_503 = RetryErrors(ServiceUnavailable) -# Retry predicates - def _has_timeseries(result): """Return True if a time series query has non-empty results.""" @@ -42,6 +40,8 @@ def _has_timeseries(result): def _unknown_metric(result): """Return True if the error describes writing to an unknown metric..""" + import sys + sys.stderr.write("WOW %s " % result.args) return UNKNOWN_METRIC_ERROR in result.message From e651d3e8830b7671dad3b8ec400d33f6fcbc196e Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 27 Sep 2016 15:52:42 -0700 Subject: [PATCH 4/5] Fix messup --- system_tests/monitoring.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/system_tests/monitoring.py b/system_tests/monitoring.py index 16d342bfaa4b..0a41744b9376 100644 --- a/system_tests/monitoring.py +++ b/system_tests/monitoring.py @@ -29,7 +29,6 @@ retry_500 = RetryErrors(InternalServerError) retry_503 = RetryErrors(ServiceUnavailable) - def _has_timeseries(result): """Return True if a time series query has non-empty results.""" return len(list(result)) > 0 @@ -37,11 +36,8 @@ def _has_timeseries(result): UNKNOWN_METRIC_ERROR = ('The provided filter doesn\'t refer to any known ' 'metric.') - def _unknown_metric(result): """Return True if the error describes writing to an unknown metric..""" - import sys - sys.stderr.write("WOW %s " % result.args) return UNKNOWN_METRIC_ERROR in result.message From 539449bbabab9f0e36a292e1a535124638e21751 Mon Sep 17 00:00:00 2001 From: Bill Prin Date: Tue, 27 Sep 2016 16:24:47 -0700 Subject: [PATCH 5/5] Actually needed those newlines --- system_tests/monitoring.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/system_tests/monitoring.py b/system_tests/monitoring.py index 0a41744b9376..816307e43cd5 100644 --- a/system_tests/monitoring.py +++ b/system_tests/monitoring.py @@ -29,6 +29,7 @@ retry_500 = RetryErrors(InternalServerError) retry_503 = RetryErrors(ServiceUnavailable) + def _has_timeseries(result): """Return True if a time series query has non-empty results.""" return len(list(result)) > 0 @@ -36,6 +37,7 @@ def _has_timeseries(result): UNKNOWN_METRIC_ERROR = ('The provided filter doesn\'t refer to any known ' 'metric.') + def _unknown_metric(result): """Return True if the error describes writing to an unknown metric..""" return UNKNOWN_METRIC_ERROR in result.message