diff --git a/experimental/CHANGELOG.md b/experimental/CHANGELOG.md index c8bddd915c..e11fb70b01 100644 --- a/experimental/CHANGELOG.md +++ b/experimental/CHANGELOG.md @@ -72,6 +72,7 @@ All notable changes to experimental packages in this project will be documented * deps: remove unused proto-loader dependencies and update grpc-js and proto-loader versions [#3337](https://github.com/open-telemetry/opentelemetry-js/pull/3337) @seemk * feat(metrics-exporters): configure temporality via environment variable [#3305](https://github.com/open-telemetry/opentelemetry-js/pull/3305) @pichlermarc * feat(console-metric-exporter): add temporality configuration [#3387](https://github.com/open-telemetry/opentelemetry-js/pull/3387) @pichlermarc +* feat(otlp-exporter-base): add retries [#3207](https://github.com/open-telemetry/opentelemetry-js/pull/3207) @svetlanabrennan ### :bug: (Bug Fix) diff --git a/experimental/packages/exporter-trace-otlp-http/README.md b/experimental/packages/exporter-trace-otlp-http/README.md index ef4259fcd1..f5944a047b 100644 --- a/experimental/packages/exporter-trace-otlp-http/README.md +++ b/experimental/packages/exporter-trace-otlp-http/README.md @@ -143,6 +143,21 @@ To override the default timeout duration, use the following options: > Providing `timeoutMillis` with `collectorOptions` takes precedence and overrides timeout set with environment variables. +## OTLP Exporter Retry + +OTLP requires that transient errors be handled with a [retry strategy](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry). + +This retry policy has the following configuration, which there is currently no way to customize. + ++ `DEFAULT_EXPORT_MAX_ATTEMPTS`: The maximum number of attempts, including the original request. Defaults to 5. ++ `DEFAULT_EXPORT_INITIAL_BACKOFF`: The initial backoff duration. Defaults to 1 second. ++ `DEFAULT_EXPORT_MAX_BACKOFF`: The maximum backoff duration. Defaults to 5 seconds. ++ `DEFAULT_EXPORT_BACKOFF_MULTIPLIER`: The backoff multiplier. Defaults to 1.5. + +This retry policy first checks if the response has a `'Retry-After'` header. If there is a `'Retry-After'` header, the exporter will wait the amount specified in the `'Retry-After'` header before retrying. If there is no `'Retry-After'` header, the exporter will use an exponential backoff with jitter retry strategy. + + > The exporter will retry exporting within the [exporter timeout configuration](#Exporter-Timeout-Configuration) time. + ## Running opentelemetry-collector locally to see the traces 1. Go to `examples/otlp-exporter-node` diff --git a/experimental/packages/exporter-trace-otlp-http/test/browser/CollectorTraceExporter.test.ts b/experimental/packages/exporter-trace-otlp-http/test/browser/CollectorTraceExporter.test.ts index 81bc6c6b49..4e8bc1d6e1 100644 --- a/experimental/packages/exporter-trace-otlp-http/test/browser/CollectorTraceExporter.test.ts +++ b/experimental/packages/exporter-trace-otlp-http/test/browser/CollectorTraceExporter.test.ts @@ -582,3 +582,123 @@ describe('when configuring via environment', () => { envSource.OTEL_EXPORTER_OTLP_HEADERS = ''; }); }); + +describe('export with retry - real http request destroyed', () => { + let server: any; + let collectorTraceExporter: OTLPTraceExporter; + let collectorExporterConfig: OTLPExporterConfigBase; + let spans: ReadableSpan[]; + + beforeEach(() => { + server = sinon.fakeServer.create({ + autoRespond: true, + }); + collectorExporterConfig = { + timeoutMillis: 1500, + }; + }); + + afterEach(() => { + server.restore(); + }); + + describe('when "sendBeacon" is NOT available', () => { + beforeEach(() => { + (window.navigator as any).sendBeacon = false; + collectorTraceExporter = new OTLPTraceExporter(collectorExporterConfig); + }); + it('should log the timeout request error message when retrying with exponential backoff with jitter', done => { + spans = []; + spans.push(Object.assign({}, mockedReadableSpan)); + + let retry = 0; + server.respondWith( + 'http://localhost:4318/v1/traces', + function (xhr: any) { + retry++; + xhr.respond(503); + } + ); + + collectorTraceExporter.export(spans, result => { + assert.strictEqual(result.code, core.ExportResultCode.FAILED); + const error = result.error as OTLPExporterError; + assert.ok(error !== undefined); + assert.strictEqual(error.message, 'Request Timeout'); + assert.strictEqual(retry, 1); + done(); + }); + }).timeout(3000); + + it('should log the timeout request error message when retry-after header is set to 3 seconds', done => { + spans = []; + spans.push(Object.assign({}, mockedReadableSpan)); + + let retry = 0; + server.respondWith( + 'http://localhost:4318/v1/traces', + function (xhr: any) { + retry++; + xhr.respond(503, { 'Retry-After': 3 }); + } + ); + + collectorTraceExporter.export(spans, result => { + assert.strictEqual(result.code, core.ExportResultCode.FAILED); + const error = result.error as OTLPExporterError; + assert.ok(error !== undefined); + assert.strictEqual(error.message, 'Request Timeout'); + assert.strictEqual(retry, 1); + done(); + }); + }).timeout(3000); + it('should log the timeout request error message when retry-after header is a date', done => { + spans = []; + spans.push(Object.assign({}, mockedReadableSpan)); + + let retry = 0; + server.respondWith( + 'http://localhost:4318/v1/traces', + function (xhr: any) { + retry++; + const d = new Date(); + d.setSeconds(d.getSeconds() + 1); + xhr.respond(503, { 'Retry-After': d }); + } + ); + + collectorTraceExporter.export(spans, result => { + assert.strictEqual(result.code, core.ExportResultCode.FAILED); + const error = result.error as OTLPExporterError; + assert.ok(error !== undefined); + assert.strictEqual(error.message, 'Request Timeout'); + assert.strictEqual(retry, 2); + done(); + }); + }).timeout(3000); + it('should log the timeout request error message when retry-after header is a date with long delay', done => { + spans = []; + spans.push(Object.assign({}, mockedReadableSpan)); + + let retry = 0; + server.respondWith( + 'http://localhost:4318/v1/traces', + function (xhr: any) { + retry++; + const d = new Date(); + d.setSeconds(d.getSeconds() + 120); + xhr.respond(503, { 'Retry-After': d }); + } + ); + + collectorTraceExporter.export(spans, result => { + assert.strictEqual(result.code, core.ExportResultCode.FAILED); + const error = result.error as OTLPExporterError; + assert.ok(error !== undefined); + assert.strictEqual(error.message, 'Request Timeout'); + assert.strictEqual(retry, 1); + done(); + }); + }).timeout(3000); + }); +}); diff --git a/experimental/packages/exporter-trace-otlp-http/test/node/CollectorTraceExporter.test.ts b/experimental/packages/exporter-trace-otlp-http/test/node/CollectorTraceExporter.test.ts index e7006dbda5..426aed4431 100644 --- a/experimental/packages/exporter-trace-otlp-http/test/node/CollectorTraceExporter.test.ts +++ b/experimental/packages/exporter-trace-otlp-http/test/node/CollectorTraceExporter.test.ts @@ -551,38 +551,3 @@ describe('export - real http request destroyed before response received', () => }, 0); }); }); - -describe('export - real http request destroyed after response received', () => { - let collectorExporter: OTLPTraceExporter; - let collectorExporterConfig: OTLPExporterNodeConfigBase; - let spans: ReadableSpan[]; - - const server = http.createServer((_, res) => { - res.write('writing something'); - }); - before(done => { - server.listen(8081, done); - }); - after(done => { - server.close(done); - }); - it('should log the timeout request error message', done => { - collectorExporterConfig = { - url: 'http://localhost:8081', - timeoutMillis: 300, - }; - collectorExporter = new OTLPTraceExporter(collectorExporterConfig); - spans = []; - spans.push(Object.assign({}, mockedReadableSpan)); - - setTimeout(() => { - collectorExporter.export(spans, result => { - assert.strictEqual(result.code, core.ExportResultCode.FAILED); - const error = result.error as OTLPExporterError; - assert.ok(error !== undefined); - assert.strictEqual(error.message, 'Request Timeout'); - done(); - }); - }, 0); - }); -}); diff --git a/experimental/packages/exporter-trace-otlp-proto/README.md b/experimental/packages/exporter-trace-otlp-proto/README.md index 0338b4cd93..efd22d2abf 100644 --- a/experimental/packages/exporter-trace-otlp-proto/README.md +++ b/experimental/packages/exporter-trace-otlp-proto/README.md @@ -72,6 +72,21 @@ To override the default timeout duration, use the following options: > Providing `timeoutMillis` with `collectorOptions` takes precedence and overrides timeout set with environment variables. +## OTLP Exporter Retry + +OTLP requires that transient errors be handled with a [retry strategy](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry). + +This retry policy has the following configuration, which there is currently no way to customize. + ++ `DEFAULT_EXPORT_MAX_ATTEMPTS`: The maximum number of attempts, including the original request. Defaults to 5. ++ `DEFAULT_EXPORT_INITIAL_BACKOFF`: The initial backoff duration. Defaults to 1 second. ++ `DEFAULT_EXPORT_MAX_BACKOFF`: The maximum backoff duration. Defaults to 5 seconds. ++ `DEFAULT_EXPORT_BACKOFF_MULTIPLIER`: The backoff multiplier. Defaults to 1.5. + +This retry policy first checks if the response has a `'Retry-After'` header. If there is a `'Retry-After'` header, the exporter will wait the amount specified in the `'Retry-After'` header before retrying. If there is no `'Retry-After'` header, the exporter will use an exponential backoff with jitter retry strategy. + + > The exporter will retry exporting within the [exporter timeout configuration](#Exporter-Timeout-Configuration) time. + ## Running opentelemetry-collector locally to see the traces 1. Go to examples/otlp-exporter-node diff --git a/experimental/packages/otlp-exporter-base/src/platform/browser/util.ts b/experimental/packages/otlp-exporter-base/src/platform/browser/util.ts index a271a3bf5f..fade4afa88 100644 --- a/experimental/packages/otlp-exporter-base/src/platform/browser/util.ts +++ b/experimental/packages/otlp-exporter-base/src/platform/browser/util.ts @@ -15,6 +15,14 @@ */ import { diag } from '@opentelemetry/api'; import { OTLPExporterError } from '../../types'; +import { + DEFAULT_EXPORT_MAX_ATTEMPTS, + DEFAULT_EXPORT_INITIAL_BACKOFF, + DEFAULT_EXPORT_BACKOFF_MULTIPLIER, + DEFAULT_EXPORT_MAX_BACKOFF, + isExportRetryable, + parseRetryAfterToMills, +} from '../../util'; /** * Send metrics/spans using browser navigator.sendBeacon @@ -57,47 +65,99 @@ export function sendWithXhr( onSuccess: () => void, onError: (error: OTLPExporterError) => void ): void { - let reqIsDestroyed: boolean; + let retryTimer: ReturnType; + let xhr: XMLHttpRequest; + let reqIsDestroyed = false; const exporterTimer = setTimeout(() => { + clearTimeout(retryTimer); reqIsDestroyed = true; - xhr.abort(); + + if (xhr.readyState === XMLHttpRequest.DONE) { + const err = new OTLPExporterError('Request Timeout'); + onError(err); + } else { + xhr.abort(); + } }, exporterTimeout); - const xhr = new XMLHttpRequest(); - xhr.open('POST', url); + const sendWithRetry = ( + retries = DEFAULT_EXPORT_MAX_ATTEMPTS, + minDelay = DEFAULT_EXPORT_INITIAL_BACKOFF + ) => { + xhr = new XMLHttpRequest(); + xhr.open('POST', url); - const defaultHeaders = { - Accept: 'application/json', - 'Content-Type': 'application/json', - }; + const defaultHeaders = { + Accept: 'application/json', + 'Content-Type': 'application/json', + }; - Object.entries({ - ...defaultHeaders, - ...headers, - }).forEach(([k, v]) => { - xhr.setRequestHeader(k, v); - }); + Object.entries({ + ...defaultHeaders, + ...headers, + }).forEach(([k, v]) => { + xhr.setRequestHeader(k, v); + }); - xhr.send(body); + xhr.send(body); - xhr.onreadystatechange = () => { - if (xhr.readyState === XMLHttpRequest.DONE) { - if (xhr.status >= 200 && xhr.status <= 299) { - clearTimeout(exporterTimer); - diag.debug('xhr success', body); - onSuccess(); - } else if (reqIsDestroyed) { - const error = new OTLPExporterError('Request Timeout', xhr.status); - onError(error); - } else { - const error = new OTLPExporterError( - `Failed to export with XHR (status: ${xhr.status})`, - xhr.status - ); - clearTimeout(exporterTimer); - onError(error); + xhr.onreadystatechange = () => { + if (xhr.readyState === XMLHttpRequest.DONE && reqIsDestroyed === false) { + if (xhr.status >= 200 && xhr.status <= 299) { + diag.debug('xhr success', body); + onSuccess(); + clearTimeout(exporterTimer); + clearTimeout(retryTimer); + } else if (xhr.status && isExportRetryable(xhr.status) && retries > 0) { + let retryTime: number; + minDelay = DEFAULT_EXPORT_BACKOFF_MULTIPLIER * minDelay; + + // retry after interval specified in Retry-After header + if (xhr.getResponseHeader('Retry-After')) { + retryTime = parseRetryAfterToMills( + xhr.getResponseHeader('Retry-After')! + ); + } else { + // exponential backoff with jitter + retryTime = Math.round( + Math.random() * (DEFAULT_EXPORT_MAX_BACKOFF - minDelay) + minDelay + ); + } + + retryTimer = setTimeout(() => { + sendWithRetry(retries - 1, minDelay); + }, retryTime); + } else { + const error = new OTLPExporterError( + `Failed to export with XHR (status: ${xhr.status})`, + xhr.status + ); + onError(error); + clearTimeout(exporterTimer); + clearTimeout(retryTimer); + } } - } + }; + + xhr.onabort = () => { + if (reqIsDestroyed) { + const err = new OTLPExporterError('Request Timeout'); + onError(err); + } + clearTimeout(exporterTimer); + clearTimeout(retryTimer); + }; + + xhr.onerror = () => { + if (reqIsDestroyed) { + const err = new OTLPExporterError('Request Timeout'); + onError(err); + } + clearTimeout(exporterTimer); + clearTimeout(retryTimer); + }; }; + + sendWithRetry(); } diff --git a/experimental/packages/otlp-exporter-base/src/platform/node/util.ts b/experimental/packages/otlp-exporter-base/src/platform/node/util.ts index d5636d14e6..fd40981e85 100644 --- a/experimental/packages/otlp-exporter-base/src/platform/node/util.ts +++ b/experimental/packages/otlp-exporter-base/src/platform/node/util.ts @@ -24,6 +24,14 @@ import { diag } from '@opentelemetry/api'; import { CompressionAlgorithm } from './types'; import { getEnv } from '@opentelemetry/core'; import { OTLPExporterError } from '../../types'; +import { + DEFAULT_EXPORT_MAX_ATTEMPTS, + DEFAULT_EXPORT_INITIAL_BACKOFF, + DEFAULT_EXPORT_BACKOFF_MULTIPLIER, + DEFAULT_EXPORT_MAX_BACKOFF, + isExportRetryable, + parseRetryAfterToMills, +} from '../../util'; /** * Sends data using http @@ -42,16 +50,21 @@ export function sendWithHttp( ): void { const exporterTimeout = collector.timeoutMillis; const parsedUrl = new url.URL(collector.url); - let reqIsDestroyed: boolean; const nodeVersion = Number(process.versions.node.split('.')[0]); + let retryTimer: ReturnType; + let req: http.ClientRequest; + let reqIsDestroyed = false; const exporterTimer = setTimeout(() => { + clearTimeout(retryTimer); reqIsDestroyed = true; - // req.abort() was deprecated since v14 - if (nodeVersion >= 14) { - req.destroy(); + + if (req.destroyed) { + const err = new OTLPExporterError('Request Timeout'); + onError(err); } else { - req.abort(); + // req.abort() was deprecated since v14 + nodeVersion >= 14 ? req.destroy() : req.abort(); } }, exporterTimeout); @@ -69,61 +82,104 @@ export function sendWithHttp( const request = parsedUrl.protocol === 'http:' ? http.request : https.request; - const req = request(options, (res: http.IncomingMessage) => { - let responseData = ''; - res.on('data', chunk => (responseData += chunk)); + const sendWithRetry = ( + retries = DEFAULT_EXPORT_MAX_ATTEMPTS, + minDelay = DEFAULT_EXPORT_INITIAL_BACKOFF + ) => { + req = request(options, (res: http.IncomingMessage) => { + let responseData = ''; + res.on('data', chunk => (responseData += chunk)); + + res.on('aborted', () => { + if (reqIsDestroyed) { + const err = new OTLPExporterError('Request Timeout'); + onError(err); + } + }); + + res.on('end', () => { + if (reqIsDestroyed === false) { + if (res.statusCode && res.statusCode < 299) { + diag.debug(`statusCode: ${res.statusCode}`, responseData); + onSuccess(); + // clear all timers since request was completed and promise was resolved + clearTimeout(exporterTimer); + clearTimeout(retryTimer); + } else if ( + res.statusCode && + isExportRetryable(res.statusCode) && + retries > 0 + ) { + let retryTime: number; + minDelay = DEFAULT_EXPORT_BACKOFF_MULTIPLIER * minDelay; + + // retry after interval specified in Retry-After header + if (res.headers['retry-after']) { + retryTime = parseRetryAfterToMills(res.headers['retry-after']!); + } else { + // exponential backoff with jitter + retryTime = Math.round( + Math.random() * (DEFAULT_EXPORT_MAX_BACKOFF - minDelay) + + minDelay + ); + } + + retryTimer = setTimeout(() => { + sendWithRetry(retries - 1, minDelay); + }, retryTime); + } else { + const error = new OTLPExporterError( + res.statusMessage, + res.statusCode, + responseData + ); + onError(error); + // clear all timers since request was completed and promise was resolved + clearTimeout(exporterTimer); + clearTimeout(retryTimer); + } + } + }); + }); - res.on('aborted', () => { + req.on('error', (error: Error | any) => { if (reqIsDestroyed) { - const err = new OTLPExporterError('Request Timeout'); + const err = new OTLPExporterError('Request Timeout', error.code); onError(err); + } else { + onError(error); } + clearTimeout(exporterTimer); + clearTimeout(retryTimer); }); - res.on('end', () => { - if (!reqIsDestroyed) { - if (res.statusCode && res.statusCode < 299) { - diag.debug(`statusCode: ${res.statusCode}`, responseData); - onSuccess(); - } else { - const error = new OTLPExporterError( - res.statusMessage, - res.statusCode, - responseData - ); - onError(error); - } - clearTimeout(exporterTimer); + req.on('abort', () => { + if (reqIsDestroyed) { + const err = new OTLPExporterError('Request Timeout'); + onError(err); } + clearTimeout(exporterTimer); + clearTimeout(retryTimer); }); - }); - req.on('error', (error: Error | any) => { - if (reqIsDestroyed) { - const err = new OTLPExporterError('Request Timeout', error.code); - onError(err); - } else { - clearTimeout(exporterTimer); - onError(error); - } - }); - - switch (collector.compression) { - case CompressionAlgorithm.GZIP: { - req.setHeader('Content-Encoding', 'gzip'); - const dataStream = readableFromBuffer(data); - dataStream - .on('error', onError) - .pipe(zlib.createGzip()) - .on('error', onError) - .pipe(req); - - break; + switch (collector.compression) { + case CompressionAlgorithm.GZIP: { + req.setHeader('Content-Encoding', 'gzip'); + const dataStream = readableFromBuffer(data); + dataStream + .on('error', onError) + .pipe(zlib.createGzip()) + .on('error', onError) + .pipe(req); + + break; + } + default: + req.end(data); + break; } - default: - req.end(data); - break; - } + }; + sendWithRetry(); } function readableFromBuffer(buff: string | Buffer): Readable { diff --git a/experimental/packages/otlp-exporter-base/src/util.ts b/experimental/packages/otlp-exporter-base/src/util.ts index 99a9f6e333..f5dc70c9e8 100644 --- a/experimental/packages/otlp-exporter-base/src/util.ts +++ b/experimental/packages/otlp-exporter-base/src/util.ts @@ -18,6 +18,10 @@ import { diag } from '@opentelemetry/api'; import { getEnv } from '@opentelemetry/core'; const DEFAULT_TRACE_TIMEOUT = 10000; +export const DEFAULT_EXPORT_MAX_ATTEMPTS = 5; +export const DEFAULT_EXPORT_INITIAL_BACKOFF = 1000; +export const DEFAULT_EXPORT_MAX_BACKOFF = 5000; +export const DEFAULT_EXPORT_BACKOFF_MULTIPLIER = 1.5; /** * Parses headers from config leaving only those that have defined values @@ -110,3 +114,26 @@ export function invalidTimeout( return defaultTimeout; } + +export function isExportRetryable(statusCode: number): boolean { + const retryCodes = [429, 502, 503, 504]; + + return retryCodes.includes(statusCode); +} + +export function parseRetryAfterToMills(retryAfter?: string | null): number { + if (retryAfter == null) { + return -1; + } + const seconds = Number.parseInt(retryAfter, 10); + if (Number.isInteger(seconds)) { + return seconds > 0 ? seconds * 1000 : -1; + } + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After#directives + const delay = new Date(retryAfter).getTime() - Date.now(); + + if (delay >= 0) { + return delay; + } + return 0; +} diff --git a/experimental/packages/otlp-exporter-base/test/common/util.test.ts b/experimental/packages/otlp-exporter-base/test/common/util.test.ts index d78b719faa..b00d1f36a5 100644 --- a/experimental/packages/otlp-exporter-base/test/common/util.test.ts +++ b/experimental/packages/otlp-exporter-base/test/common/util.test.ts @@ -21,6 +21,7 @@ import { parseHeaders, appendResourcePathToUrl, appendRootPathToUrlIfNeeded, + parseRetryAfterToMills, } from '../../src/util'; describe('utils', () => { @@ -117,3 +118,30 @@ describe('utils', () => { }); }); }); + +describe('parseRetryAfterToMills', () => { + // now: 2023-01-20T00:00:00.000Z + const tests = [ + [null, -1], + // duration + ['-100', -1], + ['1000', 1000 * 1000], + // future timestamp + ['Fri, 20 Jan 2023 00:00:01 GMT', 1000], + // Past timestamp + ['Fri, 19 Jan 2023 23:59:59 GMT', 0], + ] as [string | null, number][]; + + afterEach(() => { + sinon.restore(); + }); + + for (const [value, expect] of tests) { + it(`test ${value}`, () => { + sinon.useFakeTimers({ + now: new Date('2023-01-20T00:00:00.000Z'), + }); + assert.strictEqual(parseRetryAfterToMills(value), expect); + }); + } +}); diff --git a/experimental/packages/otlp-exporter-base/test/node/util.test.ts b/experimental/packages/otlp-exporter-base/test/node/util.test.ts index 7d5b03d3eb..86c8df40ab 100644 --- a/experimental/packages/otlp-exporter-base/test/node/util.test.ts +++ b/experimental/packages/otlp-exporter-base/test/node/util.test.ts @@ -225,6 +225,9 @@ describe('sendWithHttp', () => { assert.strictEqual(requestData, data); }); + // use fake timers to replace setTimeout in sendWithHttp function + const clock = sinon.useFakeTimers(); + sendWithHttp( exporter, data, @@ -237,6 +240,8 @@ describe('sendWithHttp', () => { assert.fail(err); } ); + + clock.restore(); }); it('should send with gzip compression if configured to do so', () => { @@ -255,6 +260,9 @@ describe('sendWithHttp', () => { assert(Buffer.concat(buffers).equals(compressedData)); }); + // use fake timers to replace setTimeout in sendWithHttp function + const clock = sinon.useFakeTimers(); + sendWithHttp( exporter, data, @@ -267,6 +275,8 @@ describe('sendWithHttp', () => { assert.fail(err); } ); + + clock.restore(); }); it('should work with gzip compression enabled even after multiple requests', () => { @@ -297,6 +307,9 @@ describe('sendWithHttp', () => { assert(Buffer.concat(buffers).equals(compressedData)); }); + // use fake timers to replace setTimeout in sendWithHttp function + const clock = sinon.useFakeTimers(); + sendWithHttp( exporter, data, @@ -309,6 +322,8 @@ describe('sendWithHttp', () => { assert.fail(err); } ); + + clock.restore(); } }); });