From d8e12b90477facc1d75126f6ba536a1325ad2fb7 Mon Sep 17 00:00:00 2001 From: "commit-queue@webkit.org" Date: Mon, 9 Oct 2017 22:17:05 +0000 Subject: [PATCH] Add quota to cache API https://bugs.webkit.org/show_bug.cgi?id=177552 Patch by Youenn Fablet on 2017-10-09 Reviewed by Alex Christensen. Source/WebCore: Tests: http/wpt/cache-storage/cache-quota.any.html Storing padded opaque response body sizes within FetchResponse and CacheStorageConnection. See https://github.com/whatwg/storage/issues/31 for the rationale about this padding. Storing in CacheStorageConnection is needed for handling cloned network fetched created responses. Storing in FetchResponse is needed for handling cloned cache-storage created opaque responses. Adding internals to query and set the fuzzed size of a response. * Modules/cache/CacheStorageConnection.cpp: (WebCore::computeRealBodySize): (WebCore::CacheStorageConnection::computeRecordBodySize): (WebCore::CacheStorageConnection::setResponseBodySizeWithPadding): (WebCore::CacheStorageConnection::responseBodySizeWithPadding const): * Modules/cache/CacheStorageConnection.h: * Modules/cache/DOMCache.cpp: (WebCore::DOMCache::toConnectionRecord): (WebCore::DOMCache::updateRecords): * Modules/cache/DOMCache.h: * Modules/cache/DOMCacheEngine.cpp: (WebCore::DOMCacheEngine::errorToException): (WebCore::DOMCacheEngine::Record::copy const): * Modules/cache/DOMCacheEngine.h: * Modules/cache/WorkerCacheStorageConnection.cpp: (WebCore::toCrossThreadRecordData): (WebCore::fromCrossThreadRecordData): * Modules/fetch/FetchResponse.cpp: (WebCore::FetchResponse::clone): (WebCore::FetchResponse::BodyLoader::didReceiveResponse): * Modules/fetch/FetchResponse.h: * Modules/fetch/FetchResponse.idl: * testing/Internals.cpp: (WebCore::Internals::setResponseSizeWithPadding): (WebCore::Internals::responseSizeWithPadding const): * testing/Internals.h: * testing/Internals.idl: Source/WebKit: Adding support for quota checking in CacheStorage::Caches. It is passed to NetworkProcess at creation time. Default quota size is configured to 400Ko by origin per default. This value is suitable for testing. Future patch should raise this default value and allows configuring it. Quota is computed based on the response body size. This size is padded at WebCore for opaque responses. Size is stored persistently as opaque response padded size should remain stable. See https://github.com/whatwg/storage/issues/31 for the rationale about this padding. In case of putting several records at the same time, the size of all records is computed so that all records will be written or rejected together. Sending QuotaExceeded error when quota is exceeded. Future effort should allow asking UIProcess for quota extension. * NetworkProcess/NetworkProcess.cpp: (WebKit::NetworkProcess::cacheStoragePerOriginQuota const): * NetworkProcess/NetworkProcess.h: * NetworkProcess/NetworkProcessCreationParameters.cpp: (WebKit::NetworkProcessCreationParameters::encode const): (WebKit::NetworkProcessCreationParameters::decode): * NetworkProcess/NetworkProcessCreationParameters.h: * NetworkProcess/cache/CacheStorageEngine.cpp: (WebKit::CacheStorage::Engine::readCachesFromDisk): * NetworkProcess/cache/CacheStorageEngineCache.cpp: (WebKit::CacheStorage::Cache::toRecordInformation): (WebKit::CacheStorage::isolatedCopy): (WebKit::CacheStorage::Cache::open): (WebKit::CacheStorage::Cache::storeRecords): (WebKit::CacheStorage::Cache::put): (WebKit::CacheStorage::Cache::writeRecordToDisk): (WebKit::CacheStorage::Cache::updateRecordToDisk): (WebKit::CacheStorage::Cache::removeRecordFromDisk): (WebKit::CacheStorage::Cache::encode): (WebKit::CacheStorage::Cache::decodeRecordHeader): (WebKit::CacheStorage::Cache::decode): * NetworkProcess/cache/CacheStorageEngineCache.h: * NetworkProcess/cache/CacheStorageEngineCaches.cpp: (WebKit::CacheStorage::Caches::Caches): (WebKit::CacheStorage::Caches::initialize): (WebKit::CacheStorage::Caches::initializeSize): (WebKit::CacheStorage::Caches::requestSpace): (WebKit::CacheStorage::Caches::writeRecord): (WebKit::CacheStorage::Caches::removeRecord): (WebKit::CacheStorage::Caches::removeCacheEntry): * NetworkProcess/cache/CacheStorageEngineCaches.h: (WebKit::CacheStorage::Caches::create): (WebKit::CacheStorage::Caches::hasEnoughSpace const): * NetworkProcess/cache/NetworkCacheStorage.cpp: (WebKit::NetworkCache::Storage::traverse): * NetworkProcess/cocoa/NetworkProcessCocoa.mm: (WebKit::NetworkProcess::platformInitializeNetworkProcessCocoa): * Shared/WebCoreArgumentCoders.cpp: (IPC::ArgumentCoder::encode): (IPC::ArgumentCoder::decode): * UIProcess/API/APIProcessPoolConfiguration.cpp: (API::ProcessPoolConfiguration::copy): * UIProcess/API/APIProcessPoolConfiguration.h: * UIProcess/WebProcessPool.cpp: (WebKit::WebProcessPool::ensureNetworkProcess): LayoutTests: * http/wpt/cache-storage/cache-quota.https.any-expected.txt: Added. * http/wpt/cache-storage/cache-quota.https.any.html: Added. * http/wpt/cache-storage/cache-quota.https.any.js: Added. git-svn-id: http://svn.webkit.org/repository/webkit/trunk@223073 268f45cc-cd09-0410-ab3c-d52691b4dbfc --- LayoutTests/ChangeLog | 11 ++ .../cache-quota.any-expected.txt | 7 + .../wpt/cache-storage/cache-quota.any.html | 1 + .../http/wpt/cache-storage/cache-quota.any.js | 169 ++++++++++++++++++ Source/WebCore/ChangeLog | 44 +++++ .../Modules/cache/CacheStorageConnection.cpp | 34 ++++ .../Modules/cache/CacheStorageConnection.h | 4 + Source/WebCore/Modules/cache/DOMCache.cpp | 16 +- Source/WebCore/Modules/cache/DOMCache.h | 1 + .../WebCore/Modules/cache/DOMCacheEngine.cpp | 4 +- Source/WebCore/Modules/cache/DOMCacheEngine.h | 9 +- .../cache/WorkerCacheStorageConnection.cpp | 7 +- .../WebCore/Modules/fetch/FetchResponse.cpp | 10 +- Source/WebCore/Modules/fetch/FetchResponse.h | 9 +- .../WebCore/Modules/fetch/FetchResponse.idl | 1 + Source/WebCore/testing/Internals.cpp | 11 ++ Source/WebCore/testing/Internals.h | 3 + Source/WebCore/testing/Internals.idl | 2 + Source/WebKit/ChangeLog | 70 ++++++++ .../WebKit/NetworkProcess/NetworkProcess.cpp | 5 + Source/WebKit/NetworkProcess/NetworkProcess.h | 2 + .../NetworkProcessCreationParameters.cpp | 3 + .../NetworkProcessCreationParameters.h | 1 + .../cache/CacheStorageEngine.cpp | 2 +- .../cache/CacheStorageEngineCache.cpp | 103 ++++++++--- .../cache/CacheStorageEngineCache.h | 19 +- .../cache/CacheStorageEngineCaches.cpp | 56 +++++- .../cache/CacheStorageEngineCaches.h | 16 +- .../cache/NetworkCacheStorage.cpp | 2 +- .../cocoa/NetworkProcessCocoa.mm | 1 + .../WebKit/Shared/WebCoreArgumentCoders.cpp | 7 +- .../API/APIProcessPoolConfiguration.cpp | 3 +- .../API/APIProcessPoolConfiguration.h | 2 + Source/WebKit/UIProcess/WebProcessPool.cpp | 1 + 34 files changed, 574 insertions(+), 62 deletions(-) create mode 100644 LayoutTests/http/wpt/cache-storage/cache-quota.any-expected.txt create mode 100644 LayoutTests/http/wpt/cache-storage/cache-quota.any.html create mode 100644 LayoutTests/http/wpt/cache-storage/cache-quota.any.js diff --git a/LayoutTests/ChangeLog b/LayoutTests/ChangeLog index fc10c01f09141..e9c7a6ea3c83f 100644 --- a/LayoutTests/ChangeLog +++ b/LayoutTests/ChangeLog @@ -1,3 +1,14 @@ +2017-10-09 Youenn Fablet + + Add quota to cache API + https://bugs.webkit.org/show_bug.cgi?id=177552 + + Reviewed by Alex Christensen. + + * http/wpt/cache-storage/cache-quota.https.any-expected.txt: Added. + * http/wpt/cache-storage/cache-quota.https.any.html: Added. + * http/wpt/cache-storage/cache-quota.https.any.js: Added. + 2017-10-09 Matt Lewis Unskipped http/tests/cache/disk-cache/disk-cache-validation-no-body.html diff --git a/LayoutTests/http/wpt/cache-storage/cache-quota.any-expected.txt b/LayoutTests/http/wpt/cache-storage/cache-quota.any-expected.txt new file mode 100644 index 0000000000000..3af2626bfbab8 --- /dev/null +++ b/LayoutTests/http/wpt/cache-storage/cache-quota.any-expected.txt @@ -0,0 +1,7 @@ + +PASS Testing synthetic response body size padding +PASS Testing non opaque response body size padding +PASS Testing opaque response body size padding +PASS Hitting cache quota for non opaque responses +PASS Hitting cache quota for padded responses + diff --git a/LayoutTests/http/wpt/cache-storage/cache-quota.any.html b/LayoutTests/http/wpt/cache-storage/cache-quota.any.html new file mode 100644 index 0000000000000..2382913528e69 --- /dev/null +++ b/LayoutTests/http/wpt/cache-storage/cache-quota.any.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/LayoutTests/http/wpt/cache-storage/cache-quota.any.js b/LayoutTests/http/wpt/cache-storage/cache-quota.any.js new file mode 100644 index 0000000000000..5a94d5b78f3b6 --- /dev/null +++ b/LayoutTests/http/wpt/cache-storage/cache-quota.any.js @@ -0,0 +1,169 @@ +// META: script=/service-workers/cache-storage/resources/test-helpers.js +// META: script=/common/get-host-info.sub.js + +var test_url = 'https://example.com/foo'; +var test_body = 'Hello world!'; + +function getResponseBodySizeWithPadding(response) +{ + var cache; + var request = new Request("temp"); + var clone = response.clone(); + return self.caches.open("test").then((c) => { + cache = c; + return cache.put(request, clone); + }).then(() => { + return self.caches.delete("temp"); + }).then(() => { + return window.internals.responseSizeWithPadding(clone); + }); +} + +promise_test(() => { + if (!window.internals) + return Promise.reject("Test requires internals"); + + var response = new Response(""); + return getResponseBodySizeWithPadding(response).then(size => { + assert_equals(size, 0, "zero size synthetic response"); + return getResponseBodySizeWithPadding(response.clone()); + }).then((size) => { + assert_equals(size, 0, "zero size synthetic cloned response"); + + response = new Response("a"); + return getResponseBodySizeWithPadding(response); + }).then((size) => { + assert_equals(size, 1, "non zero size synthetic response"); + return getResponseBodySizeWithPadding(response.clone()); + }).then((size) => { + assert_equals(size, 1, "non zero size synthetic cloned response"); + }) +}, "Testing synthetic response body size padding"); + +promise_test(() => { + if (!window.internals) + return Promise.reject("Test requires internals"); + + var paddedSize; + var response, responseClone; + return fetch("").then(r => { + response = r; + responseClone = response.clone(); + return getResponseBodySizeWithPadding(response); + }).then((size) => { + paddedSize = size; + return response.arrayBuffer(); + }).then((buffer) => { + assert_equals(buffer.byteLength, paddedSize, "non opaque network response"); + return getResponseBodySizeWithPadding(responseClone); + }).then((size) => { + assert_equals(size, paddedSize, "non opaque network cloned response"); + }); +}, "Testing non opaque response body size padding"); + +promise_test(() => { + if (!window.internals) + return Promise.reject("Test requires internals"); + + var actualSize, paddedSize; + var response, responseClone1; + return fetch(get_host_info().HTTP_REMOTE_ORIGIN, {mode: "no-cors"}).then(r => { + response = r; + }).then(() => { + return fetch(get_host_info().HTTP_ORIGIN); + }).then(r => { + return r.arrayBuffer(); + }).then((buffer) => { + actualSize = buffer.byteLength; + }).then(() => { + responseClone1 = response.clone(); + return getResponseBodySizeWithPadding(response); + }).then((size) => { + paddedSize = size; + return getResponseBodySizeWithPadding(responseClone1); + }).then((size) => { + assert_not_equals(size, actualSize, "padded size should be different from actual size"); + assert_equals(size, paddedSize, "opaque network cloned response"); + }); +}, "Testing opaque response body size padding"); + +async function doCleanup() +{ + var cachesKeys = await self.caches.keys(); + for (let name of cachesKeys) { + let cache = await self.caches.open(name); + let keys = await cache.keys(); + for (let key of keys) + await cache.delete(key); + } +} + +promise_test((test) => { + var cache; + var response1ko = new Response(new ArrayBuffer(1 * 1024)); + var response399ko = new Response(new ArrayBuffer(399 * 1024)); + + return doCleanup().then(() => { + return self.caches.open("temp1"); + }).then((c) => { + cache = c; + return cache.put("399ko", response399ko.clone()); + }).then(() => { + return cache.put("1ko-v1", response1ko.clone()); + }).then(() => { + return cache.put("1ko-v2", response1ko.clone()).then(assert_unreached, (e) => { + assert_equals(e.name, "QuotaExceededError"); + }); + }).then(() => { + return cache.delete("1ko-v1"); + }).then(() => { + return cache.put("1ko-v2", response1ko.clone()); + }).then(() => { + return cache.delete("399ko"); + }).then(() => { + return cache.delete("1ko-v1"); + }).then(() => { + return cache.delete("1ko-v2"); + }); +}, 'Hitting cache quota for non opaque responses'); + +promise_test((test) => { + if (!window.internals) + return Promise.reject("Test requires internals"); + + var cache; + var response1ko = new Response(new ArrayBuffer(1 * 1024)); + var responsePadded = new Response(new ArrayBuffer(1 * 1024)); + var response200ko = new Response(new ArrayBuffer(200 * 1024)); + + return doCleanup().then(() => { + return self.caches.open("temp2"); + }).then((c) => { + cache = c; + return fetch(get_host_info().HTTP_REMOTE_ORIGIN, {mode: "no-cors"}); + }).then((r) => { + responsePadded = r; + internals.setResponseSizeWithPadding(responsePadded, 200 * 1024); + return cache.put("200ko", response200ko.clone()); + }).then(() => { + return cache.put("1ko-padded-to-200ko", responsePadded.clone()); + }).then(() => { + return cache.put("1ko", response1ko.clone()).then(assert_unreached, (e) => { + assert_equals(e.name, "QuotaExceededError"); + }); + }).then(() => { + return cache.delete("1ko-padded-to-200ko"); + }).then(() => { + return cache.put("1ko-v2", response1ko.clone()); + }).then(() => { + return cache.put("1ko-v3", response1ko.clone()); + }).then(() => { + return cache.delete("200ko"); + }).then(() => { + return cache.delete("1ko-v2"); + }).then(() => { + return cache.delete("1ko-v3"); + }); +}, 'Hitting cache quota for padded responses'); + +done(); diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog index 14040826eb56e..569877dbcb236 100644 --- a/Source/WebCore/ChangeLog +++ b/Source/WebCore/ChangeLog @@ -1,3 +1,47 @@ +2017-10-09 Youenn Fablet + + Add quota to cache API + https://bugs.webkit.org/show_bug.cgi?id=177552 + + Reviewed by Alex Christensen. + + Tests: http/wpt/cache-storage/cache-quota.any.html + + Storing padded opaque response body sizes within FetchResponse and CacheStorageConnection. + See https://github.com/whatwg/storage/issues/31 for the rationale about this padding. + Storing in CacheStorageConnection is needed for handling cloned network fetched created responses. + Storing in FetchResponse is needed for handling cloned cache-storage created opaque responses. + + Adding internals to query and set the fuzzed size of a response. + + * Modules/cache/CacheStorageConnection.cpp: + (WebCore::computeRealBodySize): + (WebCore::CacheStorageConnection::computeRecordBodySize): + (WebCore::CacheStorageConnection::setResponseBodySizeWithPadding): + (WebCore::CacheStorageConnection::responseBodySizeWithPadding const): + * Modules/cache/CacheStorageConnection.h: + * Modules/cache/DOMCache.cpp: + (WebCore::DOMCache::toConnectionRecord): + (WebCore::DOMCache::updateRecords): + * Modules/cache/DOMCache.h: + * Modules/cache/DOMCacheEngine.cpp: + (WebCore::DOMCacheEngine::errorToException): + (WebCore::DOMCacheEngine::Record::copy const): + * Modules/cache/DOMCacheEngine.h: + * Modules/cache/WorkerCacheStorageConnection.cpp: + (WebCore::toCrossThreadRecordData): + (WebCore::fromCrossThreadRecordData): + * Modules/fetch/FetchResponse.cpp: + (WebCore::FetchResponse::clone): + (WebCore::FetchResponse::BodyLoader::didReceiveResponse): + * Modules/fetch/FetchResponse.h: + * Modules/fetch/FetchResponse.idl: + * testing/Internals.cpp: + (WebCore::Internals::setResponseSizeWithPadding): + (WebCore::Internals::responseSizeWithPadding const): + * testing/Internals.h: + * testing/Internals.idl: + 2017-10-09 Zalan Bujtas Remove redundant RenderObject::virtualContinuation diff --git a/Source/WebCore/Modules/cache/CacheStorageConnection.cpp b/Source/WebCore/Modules/cache/CacheStorageConnection.cpp index c8d9d225f4619..e0eaf2c4c410e 100644 --- a/Source/WebCore/Modules/cache/CacheStorageConnection.cpp +++ b/Source/WebCore/Modules/cache/CacheStorageConnection.cpp @@ -27,6 +27,9 @@ #include "config.h" #include "CacheStorageConnection.h" +#include "FetchResponse.h" +#include + using namespace WebCore::DOMCacheEngine; namespace WebCore { @@ -71,6 +74,37 @@ void CacheStorageConnection::batchDeleteOperation(uint64_t cacheIdentifier, cons doBatchDeleteOperation(requestIdentifier, cacheIdentifier, request, WTFMove(options)); } +static inline uint64_t computeRealBodySize(const DOMCacheEngine::ResponseBody& body) +{ + uint64_t result = 0; + WTF::switchOn(body, [&] (const Ref& formData) { + result = formData->lengthInBytes(); + }, [&] (const Ref& buffer) { + result = buffer->size(); + }, [] (const std::nullptr_t&) { + }); + return result; +} + +uint64_t CacheStorageConnection::computeRecordBodySize(const FetchResponse& response, const DOMCacheEngine::ResponseBody& body, ResourceResponse::Tainting tainting) +{ + if (!response.opaqueLoadIdentifier()) { + ASSERT_UNUSED(tainting, tainting != ResourceResponse::Tainting::Opaque); + return computeRealBodySize(body); + } + + return m_opaqueResponseToSizeWithPaddingMap.ensure(response.opaqueLoadIdentifier(), [&] () { + uint64_t realSize = computeRealBodySize(body); + + // Padding the size as per https://github.com/whatwg/storage/issues/31. + uint64_t sizeWithPadding = realSize + static_cast(randomNumber() * 128000); + sizeWithPadding = ((sizeWithPadding / 32000) + 1) * 32000; + + m_opaqueResponseToSizeWithPaddingMap.set(response.opaqueLoadIdentifier(), sizeWithPadding); + return sizeWithPadding; + }).iterator->value; +} + void CacheStorageConnection::batchPutOperation(uint64_t cacheIdentifier, Vector&& records, RecordIdentifiersCallback&& callback) { uint64_t requestIdentifier = ++m_lastRequestIdentifier; diff --git a/Source/WebCore/Modules/cache/CacheStorageConnection.h b/Source/WebCore/Modules/cache/CacheStorageConnection.h index 466d5782b26f4..6ca7bfb914e05 100644 --- a/Source/WebCore/Modules/cache/CacheStorageConnection.h +++ b/Source/WebCore/Modules/cache/CacheStorageConnection.h @@ -32,6 +32,8 @@ namespace WebCore { +class FetchResponse; + class CacheStorageConnection : public ThreadSafeRefCounted { public: static Ref create() { return adoptRef(*new CacheStorageConnection()); } @@ -44,6 +46,7 @@ class CacheStorageConnection : public ThreadSafeRefCounted&&, DOMCacheEngine::RecordIdentifiersCallback&&); + uint64_t computeRecordBodySize(const FetchResponse&, const DOMCacheEngine::ResponseBody&, ResourceResponse::Tainting); virtual void reference(uint64_t /* cacheIdentifier */) { } virtual void dereference(uint64_t /* cacheIdentifier */) { } @@ -78,6 +81,7 @@ class CacheStorageConnection : public ThreadSafeRefCounted m_retrieveCachesPendingRequests; HashMap m_retrieveRecordsPendingRequests; HashMap m_batchDeleteAndPutPendingRequests; + HashMap m_opaqueResponseToSizeWithPaddingMap; uint64_t m_lastRequestIdentifier { 0 }; }; diff --git a/Source/WebCore/Modules/cache/DOMCache.cpp b/Source/WebCore/Modules/cache/DOMCache.cpp index 5d10fdd7a42cd..cb96cfc6f49c2 100644 --- a/Source/WebCore/Modules/cache/DOMCache.cpp +++ b/Source/WebCore/Modules/cache/DOMCache.cpp @@ -38,8 +38,6 @@ using namespace WebCore::DOMCacheEngine; namespace WebCore { -static Record toConnectionRecord(const FetchRequest&, FetchResponse&, ResponseBody&&); - DOMCache::DOMCache(ScriptExecutionContext& context, String&& name, uint64_t identifier, Ref&& connection) : ActiveDOMObject(&context) , m_name(WTFMove(name)) @@ -480,7 +478,7 @@ void DOMCache::batchDeleteOperation(const FetchRequest& request, CacheQueryOptio }); } -Record toConnectionRecord(const FetchRequest& request, FetchResponse& response, DOMCacheEngine::ResponseBody&& responseBody) +Record DOMCache::toConnectionRecord(const FetchRequest& request, FetchResponse& response, DOMCacheEngine::ResponseBody&& responseBody) { // FIXME: Add a setHTTPHeaderFields on ResourceResponseBase. ResourceResponse cachedResponse = response.resourceResponse(); @@ -493,9 +491,15 @@ Record toConnectionRecord(const FetchRequest& request, FetchResponse& response, ASSERT(!cachedRequest.isNull()); ASSERT(!cachedResponse.isNull()); + auto sizeWithPadding = response.bodySizeWithPadding(); + if (!sizeWithPadding) { + sizeWithPadding = m_connection->computeRecordBodySize(response, responseBody, cachedResponse.tainting()); + response.setBodySizeWithPadding(sizeWithPadding); + } + return { 0, 0, request.headers().guard(), WTFMove(cachedRequest), request.fetchOptions(), request.internalRequestReferrer(), - response.headers().guard(), WTFMove(cachedResponse), WTFMove(responseBody) + response.headers().guard(), WTFMove(cachedResponse), WTFMove(responseBody), sizeWithPadding }; } @@ -533,7 +537,7 @@ void DOMCache::updateRecords(Vector&& records) if (current.updateResponseCounter != record.updateResponseCounter) { auto responseHeaders = FetchHeaders::create(record.responseHeadersGuard, HTTPHeaderMap { record.response.httpHeaderFields() }); auto response = FetchResponse::create(*scriptExecutionContext(), std::nullopt, WTFMove(responseHeaders), WTFMove(record.response)); - response->setBodyData(WTFMove(record.responseBody)); + response->setBodyData(WTFMove(record.responseBody), record.responseBodySize); current.response = WTFMove(response); current.updateResponseCounter = record.updateResponseCounter; @@ -545,7 +549,7 @@ void DOMCache::updateRecords(Vector&& records) auto responseHeaders = FetchHeaders::create(record.responseHeadersGuard, HTTPHeaderMap { record.response.httpHeaderFields() }); auto response = FetchResponse::create(*scriptExecutionContext(), std::nullopt, WTFMove(responseHeaders), WTFMove(record.response)); - response->setBodyData(WTFMove(record.responseBody)); + response->setBodyData(WTFMove(record.responseBody), record.responseBodySize); newRecords.append(CacheStorageRecord { record.identifier, record.updateResponseCounter, WTFMove(request), WTFMove(response) }); } diff --git a/Source/WebCore/Modules/cache/DOMCache.h b/Source/WebCore/Modules/cache/DOMCache.h index cd222130ccac7..4f5adf86895a0 100644 --- a/Source/WebCore/Modules/cache/DOMCache.h +++ b/Source/WebCore/Modules/cache/DOMCache.h @@ -80,6 +80,7 @@ class DOMCache final : public RefCounted, public ActiveDOMObject { void updateRecords(Vector&&); Vector> cloneResponses(const Vector&); + DOMCacheEngine::Record toConnectionRecord(const FetchRequest&, FetchResponse&, DOMCacheEngine::ResponseBody&&); String m_name; uint64_t m_identifier; diff --git a/Source/WebCore/Modules/cache/DOMCacheEngine.cpp b/Source/WebCore/Modules/cache/DOMCacheEngine.cpp index dde2e89fb3baf..d2b4377f3dfb2 100644 --- a/Source/WebCore/Modules/cache/DOMCacheEngine.cpp +++ b/Source/WebCore/Modules/cache/DOMCacheEngine.cpp @@ -44,6 +44,8 @@ Exception errorToException(Error error) return Exception { TypeError, ASCIILiteral("Failed reading data from the file system") }; case Error::WriteDisk: return Exception { TypeError, ASCIILiteral("Failed writing data to the file system") }; + case Error::QuotaExceeded: + return Exception { QuotaExceededError, ASCIILiteral("Quota exceeded") }; default: return Exception { TypeError, ASCIILiteral("Internal error") }; } @@ -135,7 +137,7 @@ ResponseBody copyResponseBody(const ResponseBody& body) Record Record::copy() const { - return Record { identifier, updateResponseCounter, requestHeadersGuard, request, options, referrer, responseHeadersGuard, response, copyResponseBody(responseBody) }; + return Record { identifier, updateResponseCounter, requestHeadersGuard, request, options, referrer, responseHeadersGuard, response, copyResponseBody(responseBody), responseBodySize }; } static inline CacheInfo isolateCacheInfo(const CacheInfo& info) diff --git a/Source/WebCore/Modules/cache/DOMCacheEngine.h b/Source/WebCore/Modules/cache/DOMCacheEngine.h index eb555a59ca508..f664a2e65c4db 100644 --- a/Source/WebCore/Modules/cache/DOMCacheEngine.h +++ b/Source/WebCore/Modules/cache/DOMCacheEngine.h @@ -42,6 +42,7 @@ enum class Error { NotImplemented, ReadDisk, WriteDisk, + QuotaExceeded, Internal }; @@ -68,6 +69,7 @@ struct Record { FetchHeaders::Guard responseHeadersGuard; ResourceResponse response; ResponseBody responseBody; + uint64_t responseBodySize; }; struct CacheInfo { @@ -121,12 +123,12 @@ template inline std::optional CacheInfos::decode(Deco decoder >> infos; if (!infos) return std::nullopt; - + std::optional updateCounter; decoder >> updateCounter; if (!updateCounter) return std::nullopt; - + return {{ WTFMove(*infos), WTFMove(*updateCounter) }}; } @@ -142,7 +144,7 @@ template inline std::optional Cac decoder >> identifier; if (!identifier) return std::nullopt; - + std::optional hadStorageError; decoder >> hadStorageError; if (!hadStorageError) @@ -161,6 +163,7 @@ template<> struct EnumTraits { WebCore::DOMCacheEngine::Error::NotImplemented, WebCore::DOMCacheEngine::Error::ReadDisk, WebCore::DOMCacheEngine::Error::WriteDisk, + WebCore::DOMCacheEngine::Error::QuotaExceeded, WebCore::DOMCacheEngine::Error::Internal >; }; diff --git a/Source/WebCore/Modules/cache/WorkerCacheStorageConnection.cpp b/Source/WebCore/Modules/cache/WorkerCacheStorageConnection.cpp index afa07d2b1466b..d88760d42e9f7 100644 --- a/Source/WebCore/Modules/cache/WorkerCacheStorageConnection.cpp +++ b/Source/WebCore/Modules/cache/WorkerCacheStorageConnection.cpp @@ -53,6 +53,7 @@ struct CrossThreadRecordData { FetchHeaders::Guard responseHeadersGuard; ResourceResponse::CrossThreadData response; ResponseBody responseBody; + uint64_t responseBodySize; }; static CrossThreadRecordData toCrossThreadRecordData(const Record& record) @@ -66,7 +67,8 @@ static CrossThreadRecordData toCrossThreadRecordData(const Record& record) record.referrer.isolatedCopy(), record.responseHeadersGuard, record.response.crossThreadData(), - isolatedResponseBody(record.responseBody) + isolatedResponseBody(record.responseBody), + record.responseBodySize }; } @@ -81,7 +83,8 @@ static Record fromCrossThreadRecordData(CrossThreadRecordData&& data) WTFMove(data.referrer), data.responseHeadersGuard, ResourceResponse::fromCrossThreadData(WTFMove(data.response)), - WTFMove(data.responseBody) + WTFMove(data.responseBody), + data.responseBodySize }; } diff --git a/Source/WebCore/Modules/fetch/FetchResponse.cpp b/Source/WebCore/Modules/fetch/FetchResponse.cpp index 3f70e2cc80f22..e24209c7c487b 100644 --- a/Source/WebCore/Modules/fetch/FetchResponse.cpp +++ b/Source/WebCore/Modules/fetch/FetchResponse.cpp @@ -159,6 +159,8 @@ ExceptionOr> FetchResponse::clone(ScriptExecutionContext& con clone->cloneBody(*this); if (isBodyOpaque()) clone->setBodyAsOpaque(); + clone->m_opaqueLoadIdentifier = m_opaqueLoadIdentifier; + clone->m_bodySizeWithPadding = m_bodySizeWithPadding; return WTFMove(clone); } @@ -239,11 +241,14 @@ FetchResponse::BodyLoader::~BodyLoader() m_response.unsetPendingActivity(&m_response); } +static uint64_t nextOpaqueLoadIdentifier { 0 }; void FetchResponse::BodyLoader::didReceiveResponse(const ResourceResponse& resourceResponse) { m_response.m_response = ResourceResponseBase::filter(resourceResponse); - if (resourceResponse.tainting() == ResourceResponse::Tainting::Opaque) + if (resourceResponse.tainting() == ResourceResponse::Tainting::Opaque) { + m_response.m_opaqueLoadIdentifier = ++nextOpaqueLoadIdentifier; m_response.setBodyAsOpaque(); + } m_response.m_headers->filterAndFill(m_response.m_response.httpHeaderFields(), FetchHeaders::Guard::Response); m_response.updateContentType(); @@ -326,8 +331,9 @@ void FetchResponse::consumeBodyWhenLoaded(ConsumeDataCallback&& callback) m_bodyLoader->setConsumeDataCallback(WTFMove(callback)); } -void FetchResponse::setBodyData(ResponseData&& data) +void FetchResponse::setBodyData(ResponseData&& data, uint64_t bodySizeWithPadding) { + m_bodySizeWithPadding = bodySizeWithPadding; WTF::switchOn(data, [this](Ref& formData) { if (isBodyNull()) diff --git a/Source/WebCore/Modules/fetch/FetchResponse.h b/Source/WebCore/Modules/fetch/FetchResponse.h index 9f106c6f7d7d7..887d5d4f8957d 100644 --- a/Source/WebCore/Modules/fetch/FetchResponse.h +++ b/Source/WebCore/Modules/fetch/FetchResponse.h @@ -87,7 +87,7 @@ class FetchResponse final : public FetchBodyOwner { using ResponseData = Variant, Ref>; ResponseData consumeBody(); - void setBodyData(ResponseData&&); + void setBodyData(ResponseData&&, uint64_t bodySizeWithPadding); bool isLoading() const { return !!m_bodyLoader; } @@ -97,6 +97,10 @@ class FetchResponse final : public FetchBodyOwner { const ResourceResponse& resourceResponse() const { return m_response; } + uint64_t bodySizeWithPadding() const { return m_bodySizeWithPadding; } + void setBodySizeWithPadding(uint64_t size) { m_bodySizeWithPadding = size; } + uint64_t opaqueLoadIdentifier() const { return m_opaqueLoadIdentifier; } + private: FetchResponse(ScriptExecutionContext&, std::optional&&, Ref&&, ResourceResponse&&); @@ -138,6 +142,9 @@ class FetchResponse final : public FetchBodyOwner { ResourceResponse m_response; std::optional m_bodyLoader; mutable String m_responseURL; + // Opaque responses will padd their body size when used with Cache API. + uint64_t m_bodySizeWithPadding { 0 }; + uint64_t m_opaqueLoadIdentifier { 0 }; }; inline Ref FetchResponse::create(ScriptExecutionContext& context, std::optional&& body, Ref&& headers, ResourceResponse&& response) diff --git a/Source/WebCore/Modules/fetch/FetchResponse.idl b/Source/WebCore/Modules/fetch/FetchResponse.idl index e463fbbd75bdf..a063b2703a235 100644 --- a/Source/WebCore/Modules/fetch/FetchResponse.idl +++ b/Source/WebCore/Modules/fetch/FetchResponse.idl @@ -43,6 +43,7 @@ dictionary FetchResponseInit { ConstructorCallWith=ScriptExecutionContext, ConstructorMayThrowException, EnabledAtRuntime=FetchAPI, + ExportToWrappedFunction, Exposed=(Window,Worker), InterfaceName=Response, ] interface FetchResponse { diff --git a/Source/WebCore/testing/Internals.cpp b/Source/WebCore/testing/Internals.cpp index 8495edfff7feb..97258698a85e7 100644 --- a/Source/WebCore/testing/Internals.cpp +++ b/Source/WebCore/testing/Internals.cpp @@ -58,6 +58,7 @@ #include "Element.h" #include "EventHandler.h" #include "ExtensionStyleSheets.h" +#include "FetchResponse.h" #include "File.h" #include "FontCache.h" #include "FormController.h" @@ -4174,4 +4175,14 @@ void Internals::setConsoleMessageListener(RefPtr&& listener) contextDocument()->setConsoleMessageListener(WTFMove(listener)); } +void Internals::setResponseSizeWithPadding(FetchResponse& response, uint64_t size) +{ + response.setBodySizeWithPadding(size); +} + +uint64_t Internals::responseSizeWithPadding(FetchResponse& response) const +{ + return response.bodySizeWithPadding(); +} + } // namespace WebCore diff --git a/Source/WebCore/testing/Internals.h b/Source/WebCore/testing/Internals.h index 2ea5847c80666..16696682abba3 100644 --- a/Source/WebCore/testing/Internals.h +++ b/Source/WebCore/testing/Internals.h @@ -50,6 +50,7 @@ class DOMURL; class DOMWindow; class Document; class Element; +class FetchResponse; class File; class Frame; class GCObservation; @@ -600,6 +601,8 @@ class Internals final : public RefCounted, private ContextDestructio void clearCacheStorageMemoryRepresentation(DOMPromiseDeferred&&); void cacheStorageEngineRepresentation(DOMPromiseDeferred&&); + void setResponseSizeWithPadding(FetchResponse&, uint64_t size); + uint64_t responseSizeWithPadding(FetchResponse&) const; void setConsoleMessageListener(RefPtr&&); diff --git a/Source/WebCore/testing/Internals.idl b/Source/WebCore/testing/Internals.idl index c0c9cae54f4e3..82de510c6401f 100644 --- a/Source/WebCore/testing/Internals.idl +++ b/Source/WebCore/testing/Internals.idl @@ -547,6 +547,8 @@ enum EventThrottlingBehavior { Promise clearCacheStorageMemoryRepresentation(); Promise cacheStorageEngineRepresentation(); + void setResponseSizeWithPadding(FetchResponse response, unsigned long long size); + unsigned long long responseSizeWithPadding(FetchResponse response); void setConsoleMessageListener(StringCallback callback); diff --git a/Source/WebKit/ChangeLog b/Source/WebKit/ChangeLog index d796f894af120..2f6f06d55e74c 100644 --- a/Source/WebKit/ChangeLog +++ b/Source/WebKit/ChangeLog @@ -1,3 +1,73 @@ +2017-10-09 Youenn Fablet + + Add quota to cache API + https://bugs.webkit.org/show_bug.cgi?id=177552 + + Reviewed by Alex Christensen. + + Adding support for quota checking in CacheStorage::Caches. + It is passed to NetworkProcess at creation time. + Default quota size is configured to 400Ko by origin per default. + This value is suitable for testing. + Future patch should raise this default value and allows configuring it. + + Quota is computed based on the response body size. + This size is padded at WebCore for opaque responses. + Size is stored persistently as opaque response padded size should remain stable. + See https://github.com/whatwg/storage/issues/31 for the rationale about this padding. + + In case of putting several records at the same time, the size of all records + is computed so that all records will be written or rejected together. + + Sending QuotaExceeded error when quota is exceeded. + Future effort should allow asking UIProcess for quota extension. + + * NetworkProcess/NetworkProcess.cpp: + (WebKit::NetworkProcess::cacheStoragePerOriginQuota const): + * NetworkProcess/NetworkProcess.h: + * NetworkProcess/NetworkProcessCreationParameters.cpp: + (WebKit::NetworkProcessCreationParameters::encode const): + (WebKit::NetworkProcessCreationParameters::decode): + * NetworkProcess/NetworkProcessCreationParameters.h: + * NetworkProcess/cache/CacheStorageEngine.cpp: + (WebKit::CacheStorage::Engine::readCachesFromDisk): + * NetworkProcess/cache/CacheStorageEngineCache.cpp: + (WebKit::CacheStorage::Cache::toRecordInformation): + (WebKit::CacheStorage::isolatedCopy): + (WebKit::CacheStorage::Cache::open): + (WebKit::CacheStorage::Cache::storeRecords): + (WebKit::CacheStorage::Cache::put): + (WebKit::CacheStorage::Cache::writeRecordToDisk): + (WebKit::CacheStorage::Cache::updateRecordToDisk): + (WebKit::CacheStorage::Cache::removeRecordFromDisk): + (WebKit::CacheStorage::Cache::encode): + (WebKit::CacheStorage::Cache::decodeRecordHeader): + (WebKit::CacheStorage::Cache::decode): + * NetworkProcess/cache/CacheStorageEngineCache.h: + * NetworkProcess/cache/CacheStorageEngineCaches.cpp: + (WebKit::CacheStorage::Caches::Caches): + (WebKit::CacheStorage::Caches::initialize): + (WebKit::CacheStorage::Caches::initializeSize): + (WebKit::CacheStorage::Caches::requestSpace): + (WebKit::CacheStorage::Caches::writeRecord): + (WebKit::CacheStorage::Caches::removeRecord): + (WebKit::CacheStorage::Caches::removeCacheEntry): + * NetworkProcess/cache/CacheStorageEngineCaches.h: + (WebKit::CacheStorage::Caches::create): + (WebKit::CacheStorage::Caches::hasEnoughSpace const): + * NetworkProcess/cache/NetworkCacheStorage.cpp: + (WebKit::NetworkCache::Storage::traverse): + * NetworkProcess/cocoa/NetworkProcessCocoa.mm: + (WebKit::NetworkProcess::platformInitializeNetworkProcessCocoa): + * Shared/WebCoreArgumentCoders.cpp: + (IPC::ArgumentCoder::encode): + (IPC::ArgumentCoder::decode): + * UIProcess/API/APIProcessPoolConfiguration.cpp: + (API::ProcessPoolConfiguration::copy): + * UIProcess/API/APIProcessPoolConfiguration.h: + * UIProcess/WebProcessPool.cpp: + (WebKit::WebProcessPool::ensureNetworkProcess): + 2017-10-09 Sam Weinig Make HashMap::keys() and HashMap::values() work with WTF::map/WTF::copyToVector diff --git a/Source/WebKit/NetworkProcess/NetworkProcess.cpp b/Source/WebKit/NetworkProcess/NetworkProcess.cpp index 5a21c130d5e0f..497542ba74f08 100644 --- a/Source/WebKit/NetworkProcess/NetworkProcess.cpp +++ b/Source/WebKit/NetworkProcess/NetworkProcess.cpp @@ -766,6 +766,11 @@ void NetworkProcess::preconnectTo(const WebCore::URL& url, WebCore::StoredCreden #endif } +uint64_t NetworkProcess::cacheStoragePerOriginQuota() const +{ + return m_cacheStoragePerOriginQuota; +} + #if !PLATFORM(COCOA) void NetworkProcess::initializeProcess(const ChildProcessInitializationParameters&) { diff --git a/Source/WebKit/NetworkProcess/NetworkProcess.h b/Source/WebKit/NetworkProcess/NetworkProcess.h index 24f4a0e86ff9f..e072dd2aefb88 100644 --- a/Source/WebKit/NetworkProcess/NetworkProcess.h +++ b/Source/WebKit/NetworkProcess/NetworkProcess.h @@ -145,6 +145,7 @@ class NetworkProcess : public ChildProcess, private DownloadManager::Client { Seconds loadThrottleLatency() const { return m_loadThrottleLatency; } String cacheStorageDirectory(PAL::SessionID) const; + uint64_t cacheStoragePerOriginQuota() const; void preconnectTo(const WebCore::URL&, WebCore::StoredCredentialsPolicy); @@ -237,6 +238,7 @@ class NetworkProcess : public ChildProcess, private DownloadManager::Client { Vector> m_webProcessConnections; String m_cacheStorageDirectory; + uint64_t m_cacheStoragePerOriginQuota { 0 }; String m_diskCacheDirectory; bool m_hasSetCacheModel; CacheModel m_cacheModel; diff --git a/Source/WebKit/NetworkProcess/NetworkProcessCreationParameters.cpp b/Source/WebKit/NetworkProcess/NetworkProcessCreationParameters.cpp index 678d6cf7b1699..d68ae1a160cc4 100644 --- a/Source/WebKit/NetworkProcess/NetworkProcessCreationParameters.cpp +++ b/Source/WebKit/NetworkProcess/NetworkProcessCreationParameters.cpp @@ -46,6 +46,7 @@ void NetworkProcessCreationParameters::encode(IPC::Encoder& encoder) const encoder << diskCacheSizeOverride; encoder << canHandleHTTPSServerTrustEvaluation; encoder << cacheStorageDirectory; + encoder << cacheStoragePerOriginQuota; encoder << cacheStorageDirectoryExtensionHandle; encoder << diskCacheDirectory; encoder << diskCacheDirectoryExtensionHandle; @@ -116,6 +117,8 @@ bool NetworkProcessCreationParameters::decode(IPC::Decoder& decoder, NetworkProc return false; if (!decoder.decode(result.cacheStorageDirectory)) return false; + if (!decoder.decode(result.cacheStoragePerOriginQuota)) + return false; if (!decoder.decode(result.cacheStorageDirectoryExtensionHandle)) return false; if (!decoder.decode(result.diskCacheDirectory)) diff --git a/Source/WebKit/NetworkProcess/NetworkProcessCreationParameters.h b/Source/WebKit/NetworkProcess/NetworkProcessCreationParameters.h index d0a32dbb2f2aa..c2fdc81bfef1c 100644 --- a/Source/WebKit/NetworkProcess/NetworkProcessCreationParameters.h +++ b/Source/WebKit/NetworkProcess/NetworkProcessCreationParameters.h @@ -56,6 +56,7 @@ struct NetworkProcessCreationParameters { bool canHandleHTTPSServerTrustEvaluation { true }; String cacheStorageDirectory; + uint64_t cacheStoragePerOriginQuota; SandboxExtension::Handle cacheStorageDirectoryExtensionHandle; String diskCacheDirectory; SandboxExtension::Handle diskCacheDirectoryExtensionHandle; diff --git a/Source/WebKit/NetworkProcess/cache/CacheStorageEngine.cpp b/Source/WebKit/NetworkProcess/cache/CacheStorageEngine.cpp index bb1b964abde19..727ce49673650 100644 --- a/Source/WebKit/NetworkProcess/cache/CacheStorageEngine.cpp +++ b/Source/WebKit/NetworkProcess/cache/CacheStorageEngine.cpp @@ -190,7 +190,7 @@ void Engine::readCachesFromDisk(const String& origin, CachesCallback&& callback) { initialize([this, origin, callback = WTFMove(callback)](std::optional&& error) mutable { auto& caches = m_caches.ensure(origin, [&origin, this] { - return Caches::create(*this, String { origin }); + return Caches::create(*this, String { origin }, NetworkProcess::singleton().cacheStoragePerOriginQuota()); }).iterator->value; if (caches->isInitialized()) { diff --git a/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCache.cpp b/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCache.cpp index e9b1e5a2ba868..a6b25df3c1865 100644 --- a/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCache.cpp +++ b/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCache.cpp @@ -52,8 +52,6 @@ namespace WebKit { namespace CacheStorage { -static std::optional> decodeRecordHeader(const Storage::Record&); - static inline String computeKeyURL(const URL& url) { URL keyURL { url }; @@ -100,7 +98,7 @@ static inline void updateVaryInformation(RecordInformation& recordInformation, c RecordInformation Cache::toRecordInformation(const Record& record) { Key key { ASCIILiteral("record"), m_uniqueName, { }, createCanonicalUUIDString(), m_caches.salt() }; - RecordInformation recordInformation { WTFMove(key), monotonicallyIncreasingTimeMS(), record.identifier, 0 , record.request.url(), false, { } }; + RecordInformation recordInformation { WTFMove(key), monotonicallyIncreasingTimeMS(), record.identifier, 0 , record.responseBodySize, record.request.url(), false, { } }; updateVaryInformation(recordInformation, record.request, record.response); @@ -123,6 +121,11 @@ void Cache::dispose() void Cache::clearMemoryRepresentation() { + for (auto& records : m_records.values()) { + for (auto& record : records) + removeRecordFromDisk(record); + } + m_records = { }; m_nextRecordIdentifier = 0; m_state = State::Uninitialized; @@ -130,7 +133,7 @@ void Cache::clearMemoryRepresentation() static RecordInformation isolatedCopy(const RecordInformation& information) { - auto result = RecordInformation { information.key, information.insertionTime, information.identifier, information.updateResponseCounter, information.url.isolatedCopy(), information.hasVaryStar, { } }; + auto result = RecordInformation { information.key, information.insertionTime, information.identifier, information.updateResponseCounter, information.size, information.url.isolatedCopy(), information.hasVaryStar, { } }; HashMap varyHeaders; for (const auto& keyValue : information.varyHeaders) varyHeaders.set(keyValue.key.isolatedCopy(), keyValue.value.isolatedCopy()); @@ -175,7 +178,7 @@ void Cache::open(CompletionCallback&& callback) if (!storageRecord) { RunLoop::main().dispatch([caches = WTFMove(caches), callback = WTFMove(callback), traversalResult = isolatedCopy(WTFMove(traversalResult)) ]() mutable { for (auto& key : traversalResult.failedRecords) - caches->removeRecord(key); + caches->removeCacheEntry(key); auto* cache = caches->find(traversalResult.cacheIdentifier); if (!cache) { @@ -194,10 +197,10 @@ void Cache::open(CompletionCallback&& callback) return; } - auto& record = decoded->first; - auto insertionTime = decoded->second; + auto& record = decoded->record; + auto insertionTime = decoded->insertionTime; - RecordInformation recordInformation { storageRecord->key, insertionTime, 0, 0, record.request.url(), false, { } }; + RecordInformation recordInformation { storageRecord->key, insertionTime, 0, 0, record.responseBodySize, record.request.url(), false, { } }; updateVaryInformation(recordInformation, record.request, record.response); auto& sameURLRecords = traversalResult.records.ensure(computeKeyURL(recordInformation.url), [] { return Vector { }; }).iterator->value; @@ -377,36 +380,67 @@ class AsynchronousPutTaskCounter : public RefCounted Vector m_recordIdentifiers; }; -void Cache::put(Vector&& records, RecordIdentifiersCallback&& callback) +void Cache::storeRecords(Vector&& records, RecordIdentifiersCallback&& callback) { - ASSERT(m_state == State::Open); - auto taskCounter = AsynchronousPutTaskCounter::create(WTFMove(callback)); WebCore::CacheQueryOptions options; for (auto& record : records) { auto* sameURLRecords = recordsFromURL(record.request.url()); - auto matchingRecords = queryCache(sameURLRecords, record.request, options); - if (matchingRecords.isEmpty()) { + + auto position = !matchingRecords.isEmpty() ? sameURLRecords->findMatching([&](const auto& item) { return item.identifier == matchingRecords[0]; }) : notFound; + + if (position == notFound) { record.identifier = ++m_nextRecordIdentifier; taskCounter->addRecordIdentifier(record.identifier); auto& recordToWrite = addRecord(sameURLRecords, record); - writeRecordToDisk(recordToWrite, WTFMove(record), taskCounter.copyRef()); + writeRecordToDisk(recordToWrite, WTFMove(record), taskCounter.copyRef(), 0); } else { - auto identifier = matchingRecords[0]; - auto position = sameURLRecords->findMatching([&](const auto& item) { return item.identifier == identifier; }); - ASSERT(position != notFound); - if (position != notFound) { - auto& existingRecord = sameURLRecords->at(position); - taskCounter->addRecordIdentifier(identifier); - updateRecordToDisk(existingRecord, WTFMove(record), taskCounter.copyRef()); - } + auto& existingRecord = sameURLRecords->at(position); + taskCounter->addRecordIdentifier(existingRecord.identifier); + updateRecordToDisk(existingRecord, WTFMove(record), taskCounter.copyRef()); } } } +void Cache::put(Vector&& records, RecordIdentifiersCallback&& callback) +{ + ASSERT(m_state == State::Open); + + WebCore::CacheQueryOptions options; + uint64_t spaceRequired = 0; + + for (auto& record : records) { + auto* sameURLRecords = recordsFromURL(record.request.url()); + auto matchingRecords = queryCache(sameURLRecords, record.request, options); + + auto position = (sameURLRecords && !matchingRecords.isEmpty()) ? sameURLRecords->findMatching([&](const auto& item) { return item.identifier == matchingRecords[0]; }) : notFound; + + spaceRequired += record.responseBodySize; + if (position != notFound) + spaceRequired -= sameURLRecords->at(position).size; + } + + if (m_caches.hasEnoughSpace(spaceRequired)) { + storeRecords(WTFMove(records), WTFMove(callback)); + return; + } + + m_caches.requestSpace(spaceRequired, [caches = makeRef(m_caches), identifier = m_identifier, records = WTFMove(records), callback = WTFMove(callback)](std::optional&& error) mutable { + if (error) { + callback(makeUnexpected(error.value())); + return; + } + auto* cache = caches->find(identifier); + if (!cache) + return; + + cache->storeRecords(WTFMove(records), WTFMove(callback)); + }); +} + void Cache::remove(WebCore::ResourceRequest&& request, WebCore::CacheQueryOptions&& options, RecordIdentifiersCallback&& callback) { ASSERT(m_state == State::Open); @@ -446,9 +480,9 @@ void Cache::removeFromRecordList(const Vector& recordIdentifiers) } } -void Cache::writeRecordToDisk(const RecordInformation& recordInformation, Record&& record, Ref&& taskCounter) +void Cache::writeRecordToDisk(const RecordInformation& recordInformation, Record&& record, Ref&& taskCounter, uint64_t previousRecordSize) { - m_caches.writeRecord(*this, recordInformation, WTFMove(record), [taskCounter = WTFMove(taskCounter)](std::optional&& error) { + m_caches.writeRecord(*this, recordInformation, WTFMove(record), previousRecordSize, [taskCounter = WTFMove(taskCounter)](std::optional&& error) { if (error) taskCounter->setError(error.value()); }); @@ -473,6 +507,8 @@ void Cache::updateRecordToDisk(RecordInformation& existingRecord, Record&& recor if (position == notFound) return; auto& recordInfo = sameURLRecords->at(position); + auto previousSize = recordInfo.size; + recordInfo.size = record.responseBodySize; auto& recordFromDisk = result.value(); record.requestHeadersGuard = recordFromDisk.requestHeadersGuard; @@ -482,7 +518,7 @@ void Cache::updateRecordToDisk(RecordInformation& existingRecord, Record&& recor updateVaryInformation(recordInfo, record.request, record.response); - cache->writeRecordToDisk(recordInfo, WTFMove(record), WTFMove(taskCounter)); + cache->writeRecordToDisk(recordInfo, WTFMove(record), WTFMove(taskCounter), previousSize); }); } @@ -493,13 +529,14 @@ void Cache::readRecordFromDisk(const RecordInformation& record, WTF::Function> decodeRecordHeader(const Storage::Record& storage) +std::optional Cache::decodeRecordHeader(const Storage::Record& storage) { WTF::Persistence::Decoder decoder(storage.header.data(), storage.header.size()); @@ -532,6 +570,10 @@ static inline std::optional> decodeRecordHeader(const if (!decoder.decode(insertionTime)) return std::nullopt; + uint64_t size; + if (!decoder.decode(size)) + return std::nullopt; + if (!decoder.decode(record.requestHeadersGuard)) return std::nullopt; @@ -550,10 +592,13 @@ static inline std::optional> decodeRecordHeader(const if (!decoder.decode(record.response)) return std::nullopt; + if (!decoder.decode(record.responseBodySize)) + return std::nullopt; + if (!decoder.verifyChecksum()) return std::nullopt; - return std::make_pair(WTFMove(record), insertionTime); + return DecodedRecord { insertionTime, size, WTFMove(record) }; } std::optional Cache::decode(const Storage::Record& storage) @@ -563,7 +608,7 @@ std::optional Cache::decode(const Storage::Record& storage) if (!result) return std::nullopt; - auto record = WTFMove(result->first); + auto record = WTFMove(result->record); record.responseBody = WebCore::SharedBuffer::create(storage.body.data(), storage.body.size()); return WTFMove(record); diff --git a/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCache.h b/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCache.h index cdc481749aa8f..7fb6e0818ec24 100644 --- a/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCache.h +++ b/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCache.h @@ -43,6 +43,8 @@ struct RecordInformation { uint64_t identifier { 0 }; uint64_t updateResponseCounter { 0 }; + uint64_t size { 0 }; + WebCore::URL url; bool hasVaryStar { false }; HashMap varyHeaders; @@ -78,18 +80,33 @@ class Cache { static std::optional decode(const NetworkCache::Storage::Record&); static NetworkCache::Storage::Record encode(const RecordInformation&, const WebCore::DOMCacheEngine::Record&); + struct DecodedRecord { + DecodedRecord(double insertionTime, uint64_t size, WebCore::DOMCacheEngine::Record&& record) + : insertionTime(insertionTime) + , size(size) + , record(WTFMove(record)) + { } + + double insertionTime { 0 }; + uint64_t size { 0 }; + WebCore::DOMCacheEngine::Record record; + }; + static std::optional decodeRecordHeader(const NetworkCache::Storage::Record&); + private: Vector* recordsFromURL(const WebCore::URL&); const Vector* recordsFromURL(const WebCore::URL&) const; RecordInformation& addRecord(Vector*, const WebCore::DOMCacheEngine::Record&); + void storeRecords(Vector&&, WebCore::DOMCacheEngine::RecordIdentifiersCallback&&); + RecordInformation toRecordInformation(const WebCore::DOMCacheEngine::Record&); void finishOpening(WebCore::DOMCacheEngine::CompletionCallback&&, std::optional&&); void retrieveRecord(const RecordInformation&, Ref&&); void readRecordsList(WebCore::DOMCacheEngine::CompletionCallback&&); - void writeRecordToDisk(const RecordInformation&, WebCore::DOMCacheEngine::Record&&, Ref&&); + void writeRecordToDisk(const RecordInformation&, WebCore::DOMCacheEngine::Record&&, Ref&&, uint64_t previousRecordSize); void updateRecordToDisk(RecordInformation&, WebCore::DOMCacheEngine::Record&&, Ref&&); void removeRecordFromDisk(const RecordInformation&); void readRecordFromDisk(const RecordInformation&, WTF::Function&&)>&&); diff --git a/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCaches.cpp b/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCaches.cpp index 207b7d6fd6aa8..c7b4fbb94ecce 100644 --- a/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCaches.cpp +++ b/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCaches.cpp @@ -51,10 +51,11 @@ static inline String cachesListFilename(const String& cachesRootPath) return WebCore::pathByAppendingComponent(cachesRootPath, ASCIILiteral("cacheslist")); } -Caches::Caches(Engine& engine, String&& origin) +Caches::Caches(Engine& engine, String&& origin, uint64_t quota) : m_engine(&engine) , m_origin(WTFMove(origin)) , m_rootPath(cachesRootPath(engine, m_origin)) + , m_quota(quota) { } @@ -84,7 +85,7 @@ void Caches::initialize(WebCore::DOMCacheEngine::CompletionCallback&& callback) } m_storage = storage.releaseNonNull(); m_storage->writeWithoutWaiting(); - readCachesFromDisk([this, callback = WTFMove(callback)](Expected, Error>&& result) { + readCachesFromDisk([this, callback = WTFMove(callback)](Expected, Error>&& result) mutable { makeDirty(); if (!result.hasValue()) { @@ -96,12 +97,28 @@ void Caches::initialize(WebCore::DOMCacheEngine::CompletionCallback&& callback) return; } m_caches = WTFMove(result.value()); - m_isInitialized = true; - callback(std::nullopt); + initializeSize(WTFMove(callback)); + }); +} - auto pendingCallbacks = WTFMove(m_pendingInitializationCallbacks); - for (auto& callback : pendingCallbacks) +void Caches::initializeSize(WebCore::DOMCacheEngine::CompletionCallback&& callback) +{ + uint64_t size = 0; + m_storage->traverse({ }, 0, [protectedThis = makeRef(*this), this, protectedStorage = makeRef(*m_storage), callback = WTFMove(callback), size](const auto* storage, const auto& information) mutable { + if (!storage) { + m_size = size; + m_isInitialized = true; callback(std::nullopt); + + auto pendingCallbacks = WTFMove(m_pendingInitializationCallbacks); + for (auto& callback : pendingCallbacks) + callback(std::nullopt); + + return; + } + auto decoded = Cache::decodeRecordHeader(*storage); + if (decoded) + size += decoded->size; }); } @@ -299,11 +316,23 @@ void Caches::readRecordsList(Cache& cache, NetworkCache::Storage::TraverseHandle }); } -void Caches::writeRecord(const Cache& cache, const RecordInformation& recordInformation, Record&& record, CompletionCallback&& callback) +void Caches::requestSpace(uint64_t spaceRequired, WebCore::DOMCacheEngine::CompletionCallback&& callback) +{ + // FIXME: Implement quota increase request. + ASSERT(m_quota < m_size + spaceRequired); + callback(Error::QuotaExceeded); +} + +void Caches::writeRecord(const Cache& cache, const RecordInformation& recordInformation, Record&& record, uint64_t previousRecordSize, CompletionCallback&& callback) { ASSERT(m_isInitialized); - // FIXME: Check for storage quota. + ASSERT(m_size >= previousRecordSize); + m_size += recordInformation.size; + m_size -= previousRecordSize; + + ASSERT(m_size <= m_quota); + if (!shouldPersist()) { m_volatileStorage.set(recordInformation.key, WTFMove(record)); return; @@ -345,7 +374,16 @@ void Caches::readRecord(const NetworkCache::Key& key, WTF::Function= record.size); + m_size -= record.size; + removeCacheEntry(record.key); +} + +void Caches::removeCacheEntry(const NetworkCache::Key& key) { ASSERT(m_isInitialized); diff --git a/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCaches.h b/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCaches.h index 4a2a7792efea6..51296acc90d8b 100644 --- a/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCaches.h +++ b/Source/WebKit/NetworkProcess/cache/CacheStorageEngineCaches.h @@ -36,7 +36,7 @@ class Engine; class Caches : public RefCounted { public: - static Ref create(Engine& engine, String&& origin) { return adoptRef(*new Caches { engine, WTFMove(origin) }); } + static Ref create(Engine& engine, String&& origin, uint64_t quota) { return adoptRef(*new Caches { engine, WTFMove(origin), quota }); } void initialize(WebCore::DOMCacheEngine::CompletionCallback&&); void open(const String& name, WebCore::DOMCacheEngine::CacheIdentifierCallback&&); @@ -54,16 +54,22 @@ class Caches : public RefCounted { void readRecordsList(Cache&, NetworkCache::Storage::TraverseHandler&&); void readRecord(const NetworkCache::Key&, WTF::Function&&)>&&); - void writeRecord(const Cache&, const RecordInformation&, WebCore::DOMCacheEngine::Record&&, WebCore::DOMCacheEngine::CompletionCallback&&); - void removeRecord(const NetworkCache::Key&); + + bool hasEnoughSpace(uint64_t spaceRequired) const { return m_quota >= m_size + spaceRequired; } + void requestSpace(uint64_t spaceRequired, WebCore::DOMCacheEngine::CompletionCallback&&); + void writeRecord(const Cache&, const RecordInformation&, WebCore::DOMCacheEngine::Record&&, uint64_t previousRecordSize, WebCore::DOMCacheEngine::CompletionCallback&&); + + void removeCacheEntry(const NetworkCache::Key&); + void removeRecord(const RecordInformation&); const NetworkCache::Salt& salt() const; bool shouldPersist() const { return !m_rootPath.isNull(); } private: - Caches(Engine&, String&& origin); + Caches(Engine&, String&& origin, uint64_t quota); + void initializeSize(WebCore::DOMCacheEngine::CompletionCallback&&); void readCachesFromDisk(WTF::Function, WebCore::DOMCacheEngine::Error>&&)>&&); void writeCachesToDisk(WebCore::DOMCacheEngine::CompletionCallback&&); @@ -77,6 +83,8 @@ class Caches : public RefCounted { uint64_t m_updateCounter { 0 }; String m_origin; String m_rootPath; + uint64_t m_quota { 0 }; + uint64_t m_size { 0 }; Vector m_caches; Vector m_removedCaches; RefPtr m_storage; diff --git a/Source/WebKit/NetworkProcess/cache/NetworkCacheStorage.cpp b/Source/WebKit/NetworkProcess/cache/NetworkCacheStorage.cpp index 0b6528dd728fb..c1b1ee5368e6c 100644 --- a/Source/WebKit/NetworkProcess/cache/NetworkCacheStorage.cpp +++ b/Source/WebKit/NetworkProcess/cache/NetworkCacheStorage.cpp @@ -861,7 +861,7 @@ void Storage::traverse(const String& type, TraverseFlags flags, TraverseHandler& ioQueue().dispatch([this, &traverseOperation] { traverseRecordsFiles(recordsPath(), traverseOperation.type, [this, &traverseOperation](const String& fileName, const String& hashString, const String& type, bool isBlob, const String& recordDirectoryPath) { - ASSERT(type == traverseOperation.type); + ASSERT(type == traverseOperation.type || traverseOperation.type.isEmpty()); if (isBlob) return; diff --git a/Source/WebKit/NetworkProcess/cocoa/NetworkProcessCocoa.mm b/Source/WebKit/NetworkProcess/cocoa/NetworkProcessCocoa.mm index e70bd27771296..a2a1ba8f23e23 100644 --- a/Source/WebKit/NetworkProcess/cocoa/NetworkProcessCocoa.mm +++ b/Source/WebKit/NetworkProcess/cocoa/NetworkProcessCocoa.mm @@ -107,6 +107,7 @@ static void initializeNetworkSettings() if (!parameters.cacheStorageDirectory.isNull()) { m_cacheStorageDirectory = parameters.cacheStorageDirectory; + m_cacheStoragePerOriginQuota = parameters.cacheStoragePerOriginQuota; SandboxExtension::consumePermanently(parameters.cacheStorageDirectoryExtensionHandle); } diff --git a/Source/WebKit/Shared/WebCoreArgumentCoders.cpp b/Source/WebKit/Shared/WebCoreArgumentCoders.cpp index cc89e9a1406b7..d3ac5adbaaebf 100644 --- a/Source/WebKit/Shared/WebCoreArgumentCoders.cpp +++ b/Source/WebKit/Shared/WebCoreArgumentCoders.cpp @@ -261,6 +261,7 @@ void ArgumentCoder::encode(Encoder& encoder, const DOMCa encoder << record.responseHeadersGuard; encoder << record.response; encoder << record.updateResponseCounter; + encoder << record.responseBodySize; WTF::switchOn(record.responseBody, [&](const Ref& buffer) { encoder << true; @@ -309,6 +310,10 @@ std::optional ArgumentCoder::dec if (!decoder.decode(updateResponseCounter)) return std::nullopt; + uint64_t responseBodySize; + if (!decoder.decode(responseBodySize)) + return std::nullopt; + WebCore::DOMCacheEngine::ResponseBody responseBody; bool hasSharedBufferBody; if (!decoder.decode(hasSharedBufferBody)) @@ -332,7 +337,7 @@ std::optional ArgumentCoder::dec } } - return {{ WTFMove(identifier), WTFMove(updateResponseCounter), WTFMove(requestHeadersGuard), WTFMove(request), WTFMove(options), WTFMove(referrer), WTFMove(responseHeadersGuard), WTFMove(response), WTFMove(responseBody) }}; + return {{ WTFMove(identifier), WTFMove(updateResponseCounter), WTFMove(requestHeadersGuard), WTFMove(request), WTFMove(options), WTFMove(referrer), WTFMove(responseHeadersGuard), WTFMove(response), WTFMove(responseBody), responseBodySize }}; } void ArgumentCoder::encode(Encoder& encoder, const EventTrackingRegions& eventTrackingRegions) diff --git a/Source/WebKit/UIProcess/API/APIProcessPoolConfiguration.cpp b/Source/WebKit/UIProcess/API/APIProcessPoolConfiguration.cpp index 1df127547ab9a..2735dd463f8ff 100644 --- a/Source/WebKit/UIProcess/API/APIProcessPoolConfiguration.cpp +++ b/Source/WebKit/UIProcess/API/APIProcessPoolConfiguration.cpp @@ -111,7 +111,8 @@ Ref ProcessPoolConfiguration::copy() copy->m_applicationCacheDirectory = this->m_applicationCacheDirectory; copy->m_applicationCacheFlatFileSubdirectoryName = this->m_applicationCacheFlatFileSubdirectoryName; copy->m_cacheStorageDirectory = this->m_cacheStorageDirectory; - copy->m_diskCacheDirectory = this->m_diskCacheDirectory; + copy->m_cacheStorageDirectory = this->m_cacheStorageDirectory; + copy->m_cacheStoragePerOriginQuota = this->m_cacheStoragePerOriginQuota; copy->m_mediaCacheDirectory = this->m_mediaCacheDirectory; copy->m_indexedDBDatabaseDirectory = this->m_indexedDBDatabaseDirectory; copy->m_injectedBundlePath = this->m_injectedBundlePath; diff --git a/Source/WebKit/UIProcess/API/APIProcessPoolConfiguration.h b/Source/WebKit/UIProcess/API/APIProcessPoolConfiguration.h index d8323ff8afd6e..1f29a314501ea 100644 --- a/Source/WebKit/UIProcess/API/APIProcessPoolConfiguration.h +++ b/Source/WebKit/UIProcess/API/APIProcessPoolConfiguration.h @@ -69,6 +69,7 @@ class ProcessPoolConfiguration final : public ObjectImplcacheStorageDirectory(); + parameters.cacheStoragePerOriginQuota = m_configuration->cacheStoragePerOriginQuota(); if (!parameters.cacheStorageDirectory.isEmpty()) SandboxExtension::createHandleForReadWriteDirectory(parameters.cacheStorageDirectory, parameters.cacheStorageDirectoryExtensionHandle);