From 97546cd0f2308a2df52c9fe00d26bc6f21daf507 Mon Sep 17 00:00:00 2001 From: dcsommer Date: Tue, 4 Nov 2014 15:28:47 -0800 Subject: [PATCH] Initial code import --- .gitignore | 30 + .travis.yml | 9 + CONTRIBUTING.md | 44 + CoreProxygenArchitecture.png | Bin 0 -> 46413 bytes Doxyfile | 1252 +++++++ LICENSE | 30 + PATENTS | 23 + README.md | 141 + proxygen/Makefile.am | 1 + proxygen/VERSION | 1 + proxygen/configure.ac | 264 ++ proxygen/deps.sh | 64 + proxygen/external/http_parser/CONTRIBUTIONS | 4 + proxygen/external/http_parser/LICENSE-MIT | 23 + proxygen/external/http_parser/README.md | 171 + proxygen/external/http_parser/http_parser.c | 1 + proxygen/external/http_parser/http_parser.h | 319 ++ .../external/http_parser/http_parser_cpp.cpp | 2381 +++++++++++++ proxygen/external/http_parser/test.c | 3142 +++++++++++++++++ proxygen/httpserver/Filters.h | 125 + proxygen/httpserver/HTTPServer.cpp | 205 ++ proxygen/httpserver/HTTPServer.h | 142 + proxygen/httpserver/HTTPServerAcceptor.cpp | 92 + proxygen/httpserver/HTTPServerAcceptor.h | 48 + proxygen/httpserver/HTTPServerOptions.h | 86 + proxygen/httpserver/Makefile.am | 27 + proxygen/httpserver/Mocks.h | 78 + proxygen/httpserver/RequestHandler.h | 99 + proxygen/httpserver/RequestHandlerAdaptor.cpp | 186 + proxygen/httpserver/RequestHandlerAdaptor.h | 74 + proxygen/httpserver/RequestHandlerFactory.h | 76 + proxygen/httpserver/ResponseBuilder.h | 198 ++ proxygen/httpserver/ResponseHandler.h | 84 + proxygen/httpserver/ScopedHTTPServer.h | 192 + proxygen/httpserver/SignalHandler.cpp | 36 + proxygen/httpserver/SignalHandler.h | 37 + .../filters/DirectResponseHandler.h | 59 + .../httpserver/filters/RejectConnectFilter.h | 106 + proxygen/httpserver/samples/Makefile.am | 1 + .../httpserver/samples/echo/EchoHandler.cpp | 56 + .../httpserver/samples/echo/EchoHandler.h | 47 + .../httpserver/samples/echo/EchoServer.cpp | 86 + proxygen/httpserver/samples/echo/EchoStats.h | 41 + proxygen/httpserver/samples/echo/Makefile.am | 10 + .../samples/echo/test/EchoHandlerTest.cpp | 97 + proxygen/httpserver/tests/HTTPServerTest.cpp | 154 + proxygen/httpserver/tests/Makefile.am | 11 + proxygen/httpserver/tests/certs/ca_cert.pem | 20 + proxygen/httpserver/tests/certs/ca_key.pem | 27 + .../httpserver/tests/certs/test_cert1.pem | 18 + .../httpserver/tests/certs/test_cert2.pem | 18 + .../httpserver/tests/certs/test_cert3.pem | 18 + .../httpserver/tests/certs/test_cert4.pem | 18 + .../httpserver/tests/certs/test_cert5.pem | 18 + .../tests/certs/test_ecdsa_cert1.pem | 14 + .../tests/certs/test_ecdsa_cert2.pem | 14 + .../tests/certs/test_ecdsa_cert3.pem | 14 + .../tests/certs/test_ecdsa_key1.pem | 5 + .../tests/certs/test_ecdsa_key2.pem | 5 + .../tests/certs/test_ecdsa_key3.pem | 5 + proxygen/httpserver/tests/certs/test_key1.pem | 27 + proxygen/httpserver/tests/certs/test_key2.pem | 27 + proxygen/httpserver/tests/certs/test_key3.pem | 27 + proxygen/httpserver/tests/certs/test_key4.pem | 27 + proxygen/httpserver/tests/certs/test_key5.pem | 27 + proxygen/lib/Makefile.am | 13 + .../lib/http/HTTPCommonHeaders.template.gperf | 61 + .../lib/http/HTTPCommonHeaders.template.h | 61 + proxygen/lib/http/HTTPCommonHeaders.txt | 82 + proxygen/lib/http/HTTPConnector.cpp | 167 + proxygen/lib/http/HTTPConnector.h | 152 + proxygen/lib/http/HTTPConstants.cpp | 32 + proxygen/lib/http/HTTPConstants.h | 67 + proxygen/lib/http/HTTPException.cpp | 31 + proxygen/lib/http/HTTPException.h | 169 + proxygen/lib/http/HTTPHeaderSize.h | 34 + proxygen/lib/http/HTTPHeaders.cpp | 308 ++ proxygen/lib/http/HTTPHeaders.h | 423 +++ proxygen/lib/http/HTTPMessage.cpp | 831 +++++ proxygen/lib/http/HTTPMessage.h | 724 ++++ proxygen/lib/http/HTTPMessageFilters.h | 98 + proxygen/lib/http/HTTPMethod.cpp | 45 + proxygen/lib/http/HTTPMethod.h | 59 + proxygen/lib/http/Makefile.am | 133 + proxygen/lib/http/ProxygenErrorEnum.cpp | 34 + proxygen/lib/http/ProxygenErrorEnum.h | 72 + proxygen/lib/http/RFC2616.cpp | 81 + proxygen/lib/http/RFC2616.h | 73 + proxygen/lib/http/Window.cpp | 96 + proxygen/lib/http/Window.h | 84 + proxygen/lib/http/codec/CodecDictionaries.h | 218 ++ proxygen/lib/http/codec/CodecProtocol.cpp | 92 + proxygen/lib/http/codec/CodecProtocol.h | 51 + proxygen/lib/http/codec/ErrorCode.cpp | 39 + proxygen/lib/http/codec/ErrorCode.h | 52 + proxygen/lib/http/codec/FlowControlFilter.cpp | 124 + proxygen/lib/http/codec/FlowControlFilter.h | 97 + proxygen/lib/http/codec/HTTP1xCodec.cpp | 1071 ++++++ proxygen/lib/http/codec/HTTP1xCodec.h | 196 + proxygen/lib/http/codec/HTTPChecks.cpp | 49 + proxygen/lib/http/codec/HTTPChecks.h | 38 + proxygen/lib/http/codec/HTTPCodec.h | 468 +++ proxygen/lib/http/codec/HTTPCodecFilter.cpp | 263 ++ proxygen/lib/http/codec/HTTPCodecFilter.h | 175 + proxygen/lib/http/codec/HTTPSettings.cpp | 74 + proxygen/lib/http/codec/HTTPSettings.h | 71 + proxygen/lib/http/codec/Makefile.am | 1 + proxygen/lib/http/codec/SPDYCodec.cpp | 1584 +++++++++ proxygen/lib/http/codec/SPDYCodec.h | 378 ++ proxygen/lib/http/codec/SPDYConstants.cpp | 163 + proxygen/lib/http/codec/SPDYConstants.h | 130 + proxygen/lib/http/codec/SPDYUtil.cpp | 81 + proxygen/lib/http/codec/SPDYUtil.h | 144 + proxygen/lib/http/codec/SPDYVersion.h | 16 + proxygen/lib/http/codec/SPDYVersionSettings.h | 47 + proxygen/lib/http/codec/SettingsId.cpp | 16 + proxygen/lib/http/codec/SettingsId.h | 44 + .../lib/http/codec/TransportDirection.cpp | 33 + proxygen/lib/http/codec/TransportDirection.h | 28 + .../http/codec/compress/GzipHeaderCodec.cpp | 400 +++ .../lib/http/codec/compress/GzipHeaderCodec.h | 96 + .../lib/http/codec/compress/HPACKCodec.cpp | 105 + proxygen/lib/http/codec/compress/HPACKCodec.h | 56 + .../lib/http/codec/compress/HPACKConstants.h | 50 + .../lib/http/codec/compress/HPACKContext.cpp | 59 + .../lib/http/codec/compress/HPACKContext.h | 58 + .../http/codec/compress/HPACKDecodeBuffer.cpp | 105 + .../http/codec/compress/HPACKDecodeBuffer.h | 84 + .../lib/http/codec/compress/HPACKDecoder.cpp | 167 + .../lib/http/codec/compress/HPACKDecoder.h | 78 + .../http/codec/compress/HPACKEncodeBuffer.cpp | 112 + .../http/codec/compress/HPACKEncodeBuffer.h | 94 + .../lib/http/codec/compress/HPACKEncoder.cpp | 155 + .../lib/http/codec/compress/HPACKEncoder.h | 75 + .../lib/http/codec/compress/HPACKHeader.cpp | 36 + .../lib/http/codec/compress/HPACKHeader.h | 81 + proxygen/lib/http/codec/compress/Header.h | 46 + .../lib/http/codec/compress/HeaderCodec.h | 128 + .../lib/http/codec/compress/HeaderPiece.h | 61 + .../lib/http/codec/compress/HeaderTable.cpp | 210 ++ .../lib/http/codec/compress/HeaderTable.h | 221 ++ proxygen/lib/http/codec/compress/Huffman.cpp | 345 ++ proxygen/lib/http/codec/compress/Huffman.h | 138 + proxygen/lib/http/codec/compress/Logging.cpp | 158 + proxygen/lib/http/codec/compress/Logging.h | 39 + .../http/codec/compress/StaticHeaderTable.cpp | 111 + .../http/codec/compress/StaticHeaderTable.h | 24 + .../codec/compress/test/HPACKBufferTests.cpp | 345 ++ .../codec/compress/test/HPACKCodecTests.cpp | 283 ++ .../codec/compress/test/HPACKContextTests.cpp | 202 ++ .../codec/compress/test/HPACKHeaderTests.cpp | 65 + .../http/codec/compress/test/HTTPArchive.cpp | 96 + .../http/codec/compress/test/HTTPArchive.h | 37 + .../codec/compress/test/HeaderPieceTests.cpp | 35 + .../codec/compress/test/HeaderTableTests.cpp | 134 + .../http/codec/compress/test/HuffmanTests.cpp | 312 ++ .../http/codec/compress/test/LoggingTests.cpp | 146 + .../codec/compress/test/RFCExamplesTests.cpp | 212 ++ .../lib/http/codec/compress/test/TestUtil.cpp | 61 + .../lib/http/codec/compress/test/TestUtil.h | 28 + proxygen/lib/http/codec/test/FilterTests.cpp | 222 ++ .../lib/http/codec/test/HTTP1xCodecTest.cpp | 130 + proxygen/lib/http/codec/test/Makefile.am | 21 + proxygen/lib/http/codec/test/MockHTTPCodec.h | 130 + .../lib/http/codec/test/SPDYCodecTest.cpp | 1228 +++++++ proxygen/lib/http/codec/test/TestUtils.cpp | 207 ++ proxygen/lib/http/codec/test/TestUtils.h | 237 ++ proxygen/lib/http/generate_http_gperfs.rb | 87 + proxygen/lib/http/session/AckLatencyEvent.h | 23 + .../lib/http/session/ByteEventTracker.cpp | 171 + proxygen/lib/http/session/ByteEventTracker.h | 85 + proxygen/lib/http/session/ByteEvents.cpp | 33 + proxygen/lib/http/session/ByteEvents.h | 105 + .../session/CodecErrorResponseHandler.cpp | 67 + .../http/session/CodecErrorResponseHandler.h | 42 + .../session/HTTPDirectResponseHandler.cpp | 122 + .../http/session/HTTPDirectResponseHandler.h | 52 + .../http/session/HTTPDownstreamSession.cpp | 89 + .../lib/http/session/HTTPDownstreamSession.h | 73 + proxygen/lib/http/session/HTTPErrorPage.cpp | 34 + proxygen/lib/http/session/HTTPErrorPage.h | 70 + proxygen/lib/http/session/HTTPEvent.cpp | 47 + proxygen/lib/http/session/HTTPEvent.h | 126 + proxygen/lib/http/session/HTTPSession.cpp | 2002 +++++++++++ proxygen/lib/http/session/HTTPSession.h | 832 +++++ .../lib/http/session/HTTPSessionAcceptor.cpp | 102 + .../lib/http/session/HTTPSessionAcceptor.h | 141 + .../lib/http/session/HTTPSessionController.h | 71 + proxygen/lib/http/session/HTTPSessionStats.h | 25 + proxygen/lib/http/session/HTTPTransaction.cpp | 937 +++++ proxygen/lib/http/session/HTTPTransaction.h | 1155 ++++++ .../http/session/HTTPTransactionEgressSM.cpp | 118 + .../http/session/HTTPTransactionEgressSM.h | 71 + .../http/session/HTTPTransactionIngressSM.cpp | 134 + .../http/session/HTTPTransactionIngressSM.h | 73 + .../lib/http/session/HTTPUpstreamSession.cpp | 116 + .../lib/http/session/HTTPUpstreamSession.h | 95 + proxygen/lib/http/session/Makefile.am | 1 + .../lib/http/session/SimpleController.cpp | 66 + proxygen/lib/http/session/SimpleController.h | 66 + proxygen/lib/http/session/TTLBAStats.h | 27 + proxygen/lib/http/session/TransportFilter.cpp | 162 + proxygen/lib/http/session/TransportFilter.h | 124 + .../test/DownstreamTransactionTest.cpp | 268 ++ .../test/HTTPDownstreamSessionTest.cpp | 1367 +++++++ .../session/test/HTTPSessionAcceptorTest.cpp | 154 + .../lib/http/session/test/HTTPSessionMocks.h | 226 ++ .../lib/http/session/test/HTTPSessionTest.h | 71 + .../http/session/test/HTTPTransactionMocks.h | 153 + .../session/test/HTTPTransactionSMTest.cpp | 169 + .../session/test/HTTPUpstreamSessionTest.cpp | 1411 ++++++++ proxygen/lib/http/session/test/Makefile.am | 21 + .../session/test/MockCodecDownstreamTest.cpp | 1368 +++++++ proxygen/lib/http/session/test/TestUtils.cpp | 41 + proxygen/lib/http/session/test/TestUtils.h | 34 + proxygen/lib/http/session/test/test_cert1.key | 27 + proxygen/lib/http/session/test/test_cert1.pem | 18 + proxygen/lib/http/test/HTTPMessageTest.cpp | 463 +++ proxygen/lib/http/test/Makefile.am | 14 + .../lib/http/test/MockHTTPMessageFilter.h | 49 + proxygen/lib/http/test/RFC2616Test.cpp | 128 + proxygen/lib/http/test/WindowTest.cpp | 124 + proxygen/lib/services/Acceptor.cpp | 440 +++ proxygen/lib/services/Acceptor.h | 342 ++ proxygen/lib/services/AcceptorConfiguration.h | 59 + proxygen/lib/services/ConnectionCounter.h | 56 + proxygen/lib/services/HTTPAcceptor.h | 54 + .../lib/services/LoadShedConfiguration.cpp | 45 + proxygen/lib/services/LoadShedConfiguration.h | 109 + proxygen/lib/services/Makefile.am | 29 + proxygen/lib/services/NetworkAddress.h | 60 + proxygen/lib/services/RequestWorker.cpp | 40 + proxygen/lib/services/RequestWorker.h | 89 + proxygen/lib/services/ServerSocketConfig.h | 126 + proxygen/lib/services/Service.cpp | 33 + proxygen/lib/services/Service.h | 137 + proxygen/lib/services/ServiceConfiguration.h | 58 + proxygen/lib/services/ServiceWorker.h | 102 + proxygen/lib/services/TransportInfo.cpp | 66 + proxygen/lib/services/TransportInfo.h | 279 ++ proxygen/lib/services/WorkerThread.cpp | 160 + proxygen/lib/services/WorkerThread.h | 129 + proxygen/lib/services/test/AcceptorTest.cpp | 83 + proxygen/lib/ssl/ClientHelloExtStats.h | 24 + proxygen/lib/ssl/DHParam.h | 53 + proxygen/lib/ssl/Makefile.am | 28 + proxygen/lib/ssl/PasswordInFile.cpp | 31 + proxygen/lib/ssl/PasswordInFile.h | 38 + proxygen/lib/ssl/SSLCacheOptions.h | 23 + proxygen/lib/ssl/SSLCacheProvider.h | 69 + proxygen/lib/ssl/SSLContextConfig.h | 95 + proxygen/lib/ssl/SSLContextManager.cpp | 654 ++++ proxygen/lib/ssl/SSLContextManager.h | 183 + proxygen/lib/ssl/SSLSessionCacheManager.cpp | 350 ++ proxygen/lib/ssl/SSLSessionCacheManager.h | 293 ++ proxygen/lib/ssl/SSLStats.h | 42 + proxygen/lib/ssl/SSLUtil.cpp | 76 + proxygen/lib/ssl/SSLUtil.h | 102 + proxygen/lib/ssl/TLSTicketKeyManager.cpp | 308 ++ proxygen/lib/ssl/TLSTicketKeyManager.h | 198 ++ proxygen/lib/ssl/TLSTicketKeySeeds.h | 20 + proxygen/lib/ssl/test/Makefile.am | 16 + proxygen/lib/ssl/test/SSLCacheTest.cpp | 277 ++ .../lib/ssl/test/SSLContextManagerTest.cpp | 88 + proxygen/lib/test/Makefile.am | 27 + proxygen/lib/test/TestAsyncTransport.cpp | 635 ++++ proxygen/lib/test/TestAsyncTransport.h | 158 + proxygen/lib/test/TestMain.cpp | 22 + proxygen/lib/utils/CobHelper.h | 60 + proxygen/lib/utils/CryptUtil.cpp | 76 + proxygen/lib/utils/CryptUtil.h | 22 + proxygen/lib/utils/DestructorCheck.h | 84 + proxygen/lib/utils/DomainNameMisc.h | 71 + proxygen/lib/utils/Exception.cpp | 34 + proxygen/lib/utils/Exception.h | 46 + proxygen/lib/utils/FilterChain.h | 358 ++ proxygen/lib/utils/HTTPTime.cpp | 43 + proxygen/lib/utils/HTTPTime.h | 20 + proxygen/lib/utils/Makefile.am | 62 + proxygen/lib/utils/NullTraceEventObserver.cpp | 16 + proxygen/lib/utils/NullTraceEventObserver.h | 23 + proxygen/lib/utils/ParseURL.cpp | 159 + proxygen/lib/utils/ParseURL.h | 112 + proxygen/lib/utils/Result.h | 242 ++ proxygen/lib/utils/SocketOptions.cpp | 38 + proxygen/lib/utils/SocketOptions.h | 24 + proxygen/lib/utils/StateMachine.h | 46 + proxygen/lib/utils/TestUtils.h | 37 + proxygen/lib/utils/Time.h | 128 + proxygen/lib/utils/TraceEvent.cpp | 130 + proxygen/lib/utils/TraceEvent.h | 139 + proxygen/lib/utils/TraceEventContext.h | 32 + proxygen/lib/utils/TraceEventObserver.h | 24 + proxygen/lib/utils/TraceEventType.txt | 18 + proxygen/lib/utils/TraceFieldType.txt | 98 + proxygen/lib/utils/UtilInl.h | 24 + .../lib/utils/gen_trace_event_constants.py | 163 + proxygen/lib/utils/test/CryptUtilTest.cpp | 49 + proxygen/lib/utils/test/GenericFilterTest.cpp | 538 +++ proxygen/lib/utils/test/HTTPTimeTest.cpp | 54 + proxygen/lib/utils/test/Makefile.am | 13 + proxygen/lib/utils/test/MockTime.h | 42 + proxygen/lib/utils/test/ParseURLTest.cpp | 137 + proxygen/lib/utils/test/ResultBenchmark.cpp | 73 + proxygen/lib/utils/test/ResultTest.cpp | 112 + proxygen/lib/utils/test/UtilTest.cpp | 22 + proxygen/lib/workers/WorkerThread.h | 16 + proxygen/m4/ac_cxx_compile_stdcxx_0x.m4 | 109 + proxygen/m4/ax_boost_base.m4 | 258 ++ proxygen/m4/ax_boost_regex.m4 | 111 + 310 files changed, 51597 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 CONTRIBUTING.md create mode 100644 CoreProxygenArchitecture.png create mode 100644 Doxyfile create mode 100644 LICENSE create mode 100644 PATENTS create mode 100644 README.md create mode 100644 proxygen/Makefile.am create mode 100644 proxygen/VERSION create mode 100644 proxygen/configure.ac create mode 100755 proxygen/deps.sh create mode 100644 proxygen/external/http_parser/CONTRIBUTIONS create mode 100644 proxygen/external/http_parser/LICENSE-MIT create mode 100644 proxygen/external/http_parser/README.md create mode 120000 proxygen/external/http_parser/http_parser.c create mode 100644 proxygen/external/http_parser/http_parser.h create mode 100644 proxygen/external/http_parser/http_parser_cpp.cpp create mode 100644 proxygen/external/http_parser/test.c create mode 100644 proxygen/httpserver/Filters.h create mode 100644 proxygen/httpserver/HTTPServer.cpp create mode 100644 proxygen/httpserver/HTTPServer.h create mode 100644 proxygen/httpserver/HTTPServerAcceptor.cpp create mode 100644 proxygen/httpserver/HTTPServerAcceptor.h create mode 100644 proxygen/httpserver/HTTPServerOptions.h create mode 100644 proxygen/httpserver/Makefile.am create mode 100644 proxygen/httpserver/Mocks.h create mode 100644 proxygen/httpserver/RequestHandler.h create mode 100644 proxygen/httpserver/RequestHandlerAdaptor.cpp create mode 100644 proxygen/httpserver/RequestHandlerAdaptor.h create mode 100644 proxygen/httpserver/RequestHandlerFactory.h create mode 100644 proxygen/httpserver/ResponseBuilder.h create mode 100644 proxygen/httpserver/ResponseHandler.h create mode 100644 proxygen/httpserver/ScopedHTTPServer.h create mode 100644 proxygen/httpserver/SignalHandler.cpp create mode 100644 proxygen/httpserver/SignalHandler.h create mode 100644 proxygen/httpserver/filters/DirectResponseHandler.h create mode 100644 proxygen/httpserver/filters/RejectConnectFilter.h create mode 100644 proxygen/httpserver/samples/Makefile.am create mode 100644 proxygen/httpserver/samples/echo/EchoHandler.cpp create mode 100644 proxygen/httpserver/samples/echo/EchoHandler.h create mode 100644 proxygen/httpserver/samples/echo/EchoServer.cpp create mode 100644 proxygen/httpserver/samples/echo/EchoStats.h create mode 100644 proxygen/httpserver/samples/echo/Makefile.am create mode 100644 proxygen/httpserver/samples/echo/test/EchoHandlerTest.cpp create mode 100644 proxygen/httpserver/tests/HTTPServerTest.cpp create mode 100644 proxygen/httpserver/tests/Makefile.am create mode 100644 proxygen/httpserver/tests/certs/ca_cert.pem create mode 100644 proxygen/httpserver/tests/certs/ca_key.pem create mode 100644 proxygen/httpserver/tests/certs/test_cert1.pem create mode 100644 proxygen/httpserver/tests/certs/test_cert2.pem create mode 100644 proxygen/httpserver/tests/certs/test_cert3.pem create mode 100644 proxygen/httpserver/tests/certs/test_cert4.pem create mode 100644 proxygen/httpserver/tests/certs/test_cert5.pem create mode 100644 proxygen/httpserver/tests/certs/test_ecdsa_cert1.pem create mode 100644 proxygen/httpserver/tests/certs/test_ecdsa_cert2.pem create mode 100644 proxygen/httpserver/tests/certs/test_ecdsa_cert3.pem create mode 100644 proxygen/httpserver/tests/certs/test_ecdsa_key1.pem create mode 100644 proxygen/httpserver/tests/certs/test_ecdsa_key2.pem create mode 100644 proxygen/httpserver/tests/certs/test_ecdsa_key3.pem create mode 100644 proxygen/httpserver/tests/certs/test_key1.pem create mode 100644 proxygen/httpserver/tests/certs/test_key2.pem create mode 100644 proxygen/httpserver/tests/certs/test_key3.pem create mode 100644 proxygen/httpserver/tests/certs/test_key4.pem create mode 100644 proxygen/httpserver/tests/certs/test_key5.pem create mode 100644 proxygen/lib/Makefile.am create mode 100644 proxygen/lib/http/HTTPCommonHeaders.template.gperf create mode 100644 proxygen/lib/http/HTTPCommonHeaders.template.h create mode 100644 proxygen/lib/http/HTTPCommonHeaders.txt create mode 100644 proxygen/lib/http/HTTPConnector.cpp create mode 100644 proxygen/lib/http/HTTPConnector.h create mode 100644 proxygen/lib/http/HTTPConstants.cpp create mode 100644 proxygen/lib/http/HTTPConstants.h create mode 100644 proxygen/lib/http/HTTPException.cpp create mode 100644 proxygen/lib/http/HTTPException.h create mode 100644 proxygen/lib/http/HTTPHeaderSize.h create mode 100644 proxygen/lib/http/HTTPHeaders.cpp create mode 100644 proxygen/lib/http/HTTPHeaders.h create mode 100644 proxygen/lib/http/HTTPMessage.cpp create mode 100644 proxygen/lib/http/HTTPMessage.h create mode 100644 proxygen/lib/http/HTTPMessageFilters.h create mode 100644 proxygen/lib/http/HTTPMethod.cpp create mode 100644 proxygen/lib/http/HTTPMethod.h create mode 100644 proxygen/lib/http/Makefile.am create mode 100644 proxygen/lib/http/ProxygenErrorEnum.cpp create mode 100644 proxygen/lib/http/ProxygenErrorEnum.h create mode 100644 proxygen/lib/http/RFC2616.cpp create mode 100644 proxygen/lib/http/RFC2616.h create mode 100644 proxygen/lib/http/Window.cpp create mode 100644 proxygen/lib/http/Window.h create mode 100644 proxygen/lib/http/codec/CodecDictionaries.h create mode 100644 proxygen/lib/http/codec/CodecProtocol.cpp create mode 100644 proxygen/lib/http/codec/CodecProtocol.h create mode 100644 proxygen/lib/http/codec/ErrorCode.cpp create mode 100644 proxygen/lib/http/codec/ErrorCode.h create mode 100644 proxygen/lib/http/codec/FlowControlFilter.cpp create mode 100644 proxygen/lib/http/codec/FlowControlFilter.h create mode 100644 proxygen/lib/http/codec/HTTP1xCodec.cpp create mode 100644 proxygen/lib/http/codec/HTTP1xCodec.h create mode 100644 proxygen/lib/http/codec/HTTPChecks.cpp create mode 100644 proxygen/lib/http/codec/HTTPChecks.h create mode 100644 proxygen/lib/http/codec/HTTPCodec.h create mode 100644 proxygen/lib/http/codec/HTTPCodecFilter.cpp create mode 100644 proxygen/lib/http/codec/HTTPCodecFilter.h create mode 100644 proxygen/lib/http/codec/HTTPSettings.cpp create mode 100644 proxygen/lib/http/codec/HTTPSettings.h create mode 100644 proxygen/lib/http/codec/Makefile.am create mode 100644 proxygen/lib/http/codec/SPDYCodec.cpp create mode 100644 proxygen/lib/http/codec/SPDYCodec.h create mode 100644 proxygen/lib/http/codec/SPDYConstants.cpp create mode 100644 proxygen/lib/http/codec/SPDYConstants.h create mode 100644 proxygen/lib/http/codec/SPDYUtil.cpp create mode 100644 proxygen/lib/http/codec/SPDYUtil.h create mode 100644 proxygen/lib/http/codec/SPDYVersion.h create mode 100644 proxygen/lib/http/codec/SPDYVersionSettings.h create mode 100644 proxygen/lib/http/codec/SettingsId.cpp create mode 100644 proxygen/lib/http/codec/SettingsId.h create mode 100644 proxygen/lib/http/codec/TransportDirection.cpp create mode 100644 proxygen/lib/http/codec/TransportDirection.h create mode 100644 proxygen/lib/http/codec/compress/GzipHeaderCodec.cpp create mode 100644 proxygen/lib/http/codec/compress/GzipHeaderCodec.h create mode 100644 proxygen/lib/http/codec/compress/HPACKCodec.cpp create mode 100644 proxygen/lib/http/codec/compress/HPACKCodec.h create mode 100644 proxygen/lib/http/codec/compress/HPACKConstants.h create mode 100644 proxygen/lib/http/codec/compress/HPACKContext.cpp create mode 100644 proxygen/lib/http/codec/compress/HPACKContext.h create mode 100644 proxygen/lib/http/codec/compress/HPACKDecodeBuffer.cpp create mode 100644 proxygen/lib/http/codec/compress/HPACKDecodeBuffer.h create mode 100644 proxygen/lib/http/codec/compress/HPACKDecoder.cpp create mode 100644 proxygen/lib/http/codec/compress/HPACKDecoder.h create mode 100644 proxygen/lib/http/codec/compress/HPACKEncodeBuffer.cpp create mode 100644 proxygen/lib/http/codec/compress/HPACKEncodeBuffer.h create mode 100644 proxygen/lib/http/codec/compress/HPACKEncoder.cpp create mode 100644 proxygen/lib/http/codec/compress/HPACKEncoder.h create mode 100644 proxygen/lib/http/codec/compress/HPACKHeader.cpp create mode 100644 proxygen/lib/http/codec/compress/HPACKHeader.h create mode 100644 proxygen/lib/http/codec/compress/Header.h create mode 100644 proxygen/lib/http/codec/compress/HeaderCodec.h create mode 100644 proxygen/lib/http/codec/compress/HeaderPiece.h create mode 100644 proxygen/lib/http/codec/compress/HeaderTable.cpp create mode 100644 proxygen/lib/http/codec/compress/HeaderTable.h create mode 100644 proxygen/lib/http/codec/compress/Huffman.cpp create mode 100644 proxygen/lib/http/codec/compress/Huffman.h create mode 100644 proxygen/lib/http/codec/compress/Logging.cpp create mode 100644 proxygen/lib/http/codec/compress/Logging.h create mode 100644 proxygen/lib/http/codec/compress/StaticHeaderTable.cpp create mode 100644 proxygen/lib/http/codec/compress/StaticHeaderTable.h create mode 100644 proxygen/lib/http/codec/compress/test/HPACKBufferTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/HPACKCodecTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/HPACKContextTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/HPACKHeaderTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/HTTPArchive.cpp create mode 100644 proxygen/lib/http/codec/compress/test/HTTPArchive.h create mode 100644 proxygen/lib/http/codec/compress/test/HeaderPieceTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/HeaderTableTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/HuffmanTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/LoggingTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/RFCExamplesTests.cpp create mode 100644 proxygen/lib/http/codec/compress/test/TestUtil.cpp create mode 100644 proxygen/lib/http/codec/compress/test/TestUtil.h create mode 100644 proxygen/lib/http/codec/test/FilterTests.cpp create mode 100644 proxygen/lib/http/codec/test/HTTP1xCodecTest.cpp create mode 100644 proxygen/lib/http/codec/test/Makefile.am create mode 100644 proxygen/lib/http/codec/test/MockHTTPCodec.h create mode 100644 proxygen/lib/http/codec/test/SPDYCodecTest.cpp create mode 100644 proxygen/lib/http/codec/test/TestUtils.cpp create mode 100644 proxygen/lib/http/codec/test/TestUtils.h create mode 100755 proxygen/lib/http/generate_http_gperfs.rb create mode 100644 proxygen/lib/http/session/AckLatencyEvent.h create mode 100644 proxygen/lib/http/session/ByteEventTracker.cpp create mode 100644 proxygen/lib/http/session/ByteEventTracker.h create mode 100644 proxygen/lib/http/session/ByteEvents.cpp create mode 100644 proxygen/lib/http/session/ByteEvents.h create mode 100644 proxygen/lib/http/session/CodecErrorResponseHandler.cpp create mode 100644 proxygen/lib/http/session/CodecErrorResponseHandler.h create mode 100644 proxygen/lib/http/session/HTTPDirectResponseHandler.cpp create mode 100644 proxygen/lib/http/session/HTTPDirectResponseHandler.h create mode 100644 proxygen/lib/http/session/HTTPDownstreamSession.cpp create mode 100644 proxygen/lib/http/session/HTTPDownstreamSession.h create mode 100644 proxygen/lib/http/session/HTTPErrorPage.cpp create mode 100644 proxygen/lib/http/session/HTTPErrorPage.h create mode 100644 proxygen/lib/http/session/HTTPEvent.cpp create mode 100644 proxygen/lib/http/session/HTTPEvent.h create mode 100644 proxygen/lib/http/session/HTTPSession.cpp create mode 100644 proxygen/lib/http/session/HTTPSession.h create mode 100644 proxygen/lib/http/session/HTTPSessionAcceptor.cpp create mode 100644 proxygen/lib/http/session/HTTPSessionAcceptor.h create mode 100644 proxygen/lib/http/session/HTTPSessionController.h create mode 100644 proxygen/lib/http/session/HTTPSessionStats.h create mode 100644 proxygen/lib/http/session/HTTPTransaction.cpp create mode 100644 proxygen/lib/http/session/HTTPTransaction.h create mode 100644 proxygen/lib/http/session/HTTPTransactionEgressSM.cpp create mode 100644 proxygen/lib/http/session/HTTPTransactionEgressSM.h create mode 100644 proxygen/lib/http/session/HTTPTransactionIngressSM.cpp create mode 100644 proxygen/lib/http/session/HTTPTransactionIngressSM.h create mode 100644 proxygen/lib/http/session/HTTPUpstreamSession.cpp create mode 100644 proxygen/lib/http/session/HTTPUpstreamSession.h create mode 100644 proxygen/lib/http/session/Makefile.am create mode 100644 proxygen/lib/http/session/SimpleController.cpp create mode 100644 proxygen/lib/http/session/SimpleController.h create mode 100644 proxygen/lib/http/session/TTLBAStats.h create mode 100644 proxygen/lib/http/session/TransportFilter.cpp create mode 100644 proxygen/lib/http/session/TransportFilter.h create mode 100644 proxygen/lib/http/session/test/DownstreamTransactionTest.cpp create mode 100644 proxygen/lib/http/session/test/HTTPDownstreamSessionTest.cpp create mode 100644 proxygen/lib/http/session/test/HTTPSessionAcceptorTest.cpp create mode 100644 proxygen/lib/http/session/test/HTTPSessionMocks.h create mode 100644 proxygen/lib/http/session/test/HTTPSessionTest.h create mode 100644 proxygen/lib/http/session/test/HTTPTransactionMocks.h create mode 100644 proxygen/lib/http/session/test/HTTPTransactionSMTest.cpp create mode 100644 proxygen/lib/http/session/test/HTTPUpstreamSessionTest.cpp create mode 100644 proxygen/lib/http/session/test/Makefile.am create mode 100644 proxygen/lib/http/session/test/MockCodecDownstreamTest.cpp create mode 100644 proxygen/lib/http/session/test/TestUtils.cpp create mode 100644 proxygen/lib/http/session/test/TestUtils.h create mode 100644 proxygen/lib/http/session/test/test_cert1.key create mode 100644 proxygen/lib/http/session/test/test_cert1.pem create mode 100644 proxygen/lib/http/test/HTTPMessageTest.cpp create mode 100644 proxygen/lib/http/test/Makefile.am create mode 100644 proxygen/lib/http/test/MockHTTPMessageFilter.h create mode 100644 proxygen/lib/http/test/RFC2616Test.cpp create mode 100644 proxygen/lib/http/test/WindowTest.cpp create mode 100644 proxygen/lib/services/Acceptor.cpp create mode 100644 proxygen/lib/services/Acceptor.h create mode 100644 proxygen/lib/services/AcceptorConfiguration.h create mode 100644 proxygen/lib/services/ConnectionCounter.h create mode 100644 proxygen/lib/services/HTTPAcceptor.h create mode 100644 proxygen/lib/services/LoadShedConfiguration.cpp create mode 100644 proxygen/lib/services/LoadShedConfiguration.h create mode 100644 proxygen/lib/services/Makefile.am create mode 100644 proxygen/lib/services/NetworkAddress.h create mode 100644 proxygen/lib/services/RequestWorker.cpp create mode 100644 proxygen/lib/services/RequestWorker.h create mode 100644 proxygen/lib/services/ServerSocketConfig.h create mode 100644 proxygen/lib/services/Service.cpp create mode 100644 proxygen/lib/services/Service.h create mode 100644 proxygen/lib/services/ServiceConfiguration.h create mode 100644 proxygen/lib/services/ServiceWorker.h create mode 100644 proxygen/lib/services/TransportInfo.cpp create mode 100644 proxygen/lib/services/TransportInfo.h create mode 100644 proxygen/lib/services/WorkerThread.cpp create mode 100644 proxygen/lib/services/WorkerThread.h create mode 100644 proxygen/lib/services/test/AcceptorTest.cpp create mode 100644 proxygen/lib/ssl/ClientHelloExtStats.h create mode 100644 proxygen/lib/ssl/DHParam.h create mode 100644 proxygen/lib/ssl/Makefile.am create mode 100644 proxygen/lib/ssl/PasswordInFile.cpp create mode 100644 proxygen/lib/ssl/PasswordInFile.h create mode 100644 proxygen/lib/ssl/SSLCacheOptions.h create mode 100644 proxygen/lib/ssl/SSLCacheProvider.h create mode 100644 proxygen/lib/ssl/SSLContextConfig.h create mode 100644 proxygen/lib/ssl/SSLContextManager.cpp create mode 100644 proxygen/lib/ssl/SSLContextManager.h create mode 100644 proxygen/lib/ssl/SSLSessionCacheManager.cpp create mode 100644 proxygen/lib/ssl/SSLSessionCacheManager.h create mode 100644 proxygen/lib/ssl/SSLStats.h create mode 100644 proxygen/lib/ssl/SSLUtil.cpp create mode 100644 proxygen/lib/ssl/SSLUtil.h create mode 100644 proxygen/lib/ssl/TLSTicketKeyManager.cpp create mode 100644 proxygen/lib/ssl/TLSTicketKeyManager.h create mode 100644 proxygen/lib/ssl/TLSTicketKeySeeds.h create mode 100644 proxygen/lib/ssl/test/Makefile.am create mode 100644 proxygen/lib/ssl/test/SSLCacheTest.cpp create mode 100644 proxygen/lib/ssl/test/SSLContextManagerTest.cpp create mode 100644 proxygen/lib/test/Makefile.am create mode 100644 proxygen/lib/test/TestAsyncTransport.cpp create mode 100644 proxygen/lib/test/TestAsyncTransport.h create mode 100644 proxygen/lib/test/TestMain.cpp create mode 100644 proxygen/lib/utils/CobHelper.h create mode 100644 proxygen/lib/utils/CryptUtil.cpp create mode 100644 proxygen/lib/utils/CryptUtil.h create mode 100644 proxygen/lib/utils/DestructorCheck.h create mode 100644 proxygen/lib/utils/DomainNameMisc.h create mode 100644 proxygen/lib/utils/Exception.cpp create mode 100644 proxygen/lib/utils/Exception.h create mode 100644 proxygen/lib/utils/FilterChain.h create mode 100644 proxygen/lib/utils/HTTPTime.cpp create mode 100644 proxygen/lib/utils/HTTPTime.h create mode 100644 proxygen/lib/utils/Makefile.am create mode 100644 proxygen/lib/utils/NullTraceEventObserver.cpp create mode 100644 proxygen/lib/utils/NullTraceEventObserver.h create mode 100644 proxygen/lib/utils/ParseURL.cpp create mode 100644 proxygen/lib/utils/ParseURL.h create mode 100644 proxygen/lib/utils/Result.h create mode 100644 proxygen/lib/utils/SocketOptions.cpp create mode 100644 proxygen/lib/utils/SocketOptions.h create mode 100644 proxygen/lib/utils/StateMachine.h create mode 100644 proxygen/lib/utils/TestUtils.h create mode 100644 proxygen/lib/utils/Time.h create mode 100644 proxygen/lib/utils/TraceEvent.cpp create mode 100644 proxygen/lib/utils/TraceEvent.h create mode 100644 proxygen/lib/utils/TraceEventContext.h create mode 100644 proxygen/lib/utils/TraceEventObserver.h create mode 100644 proxygen/lib/utils/TraceEventType.txt create mode 100644 proxygen/lib/utils/TraceFieldType.txt create mode 100644 proxygen/lib/utils/UtilInl.h create mode 100755 proxygen/lib/utils/gen_trace_event_constants.py create mode 100644 proxygen/lib/utils/test/CryptUtilTest.cpp create mode 100644 proxygen/lib/utils/test/GenericFilterTest.cpp create mode 100644 proxygen/lib/utils/test/HTTPTimeTest.cpp create mode 100644 proxygen/lib/utils/test/Makefile.am create mode 100644 proxygen/lib/utils/test/MockTime.h create mode 100644 proxygen/lib/utils/test/ParseURLTest.cpp create mode 100644 proxygen/lib/utils/test/ResultBenchmark.cpp create mode 100644 proxygen/lib/utils/test/ResultTest.cpp create mode 100644 proxygen/lib/utils/test/UtilTest.cpp create mode 100644 proxygen/lib/workers/WorkerThread.h create mode 100644 proxygen/m4/ac_cxx_compile_stdcxx_0x.m4 create mode 100644 proxygen/m4/ax_boost_base.m4 create mode 100644 proxygen/m4/ax_boost_regex.m4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..511cfb0d64 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +*~ +\#*\# + +# downloaded dependencies +/proxygen/fbthrift/ +/proxygen/lib/test/gmock* + +# autoreconf artifacts +Makefile.in +/proxygen/aclocal.m4 +/proxygen/autom4te.cache/ +/proxygen/build-aux/ +/proxygen/config.guess +/proxygen/config.hin +/proxygen/configure + +# Configure artifacts +.deps +.dirstamp +_configs.sed +Makefile + +# Build artifacts +.libs +*.o +*.lo +*.a +*.la +gen-cpp +gen-cpp2 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..f8df50a84c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +language: cpp +compiler: gcc +install: + - sudo apt-get update -qq +before_script: + - cd proxygen + - ./deps.sh +script: + - make check \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..cffd1a53ad --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,44 @@ +# Contributing to Proxygen +Here's a quick rundown of how to contribute to this project. + +## Our Development Process +We develop on a private branch internally at Facebook. We regularly update +this github project with the changes from the internal repo. External pull +requests are cherry-picked into our repo and then pushed back out. + +## Pull Requests +We actively welcome your pull requests. + +1. Fork the repo and create your branch from `master`. +1. If you've added code that should be tested, add tests +1. If you've changed APIs, update the documentation. +1. Ensure the test suite passes. +1. Make sure your code lints. +1. If you haven't already, complete the Contributor License Agreement ("CLA"). + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You +only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: + +## Issues +We use GitHub issues to track public bugs. Please ensure your description +is clear and has sufficient instructions to be able to reproduce the issue. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for +the safe disclosure of security bugs. In those cases, please go through +the process outlined on that page and do not file a public issue. + +## Coding Style +* 2 spaces for indentation rather than tabs +* 80 character line length +* Use `Type* foo` not `Type *foo`. +* Align parameters passed to functions. +* Prefer `folly::make_unique` to `new Foo`. In general, we discourage +use of raw `new` or `delete`. + +## License +By contributing to Proxygen, you agree that your contributions will be +licensed under its BSD license. \ No newline at end of file diff --git a/CoreProxygenArchitecture.png b/CoreProxygenArchitecture.png new file mode 100644 index 0000000000000000000000000000000000000000..ee772bdea387cb546814f508a4eeb06d70735f2c GIT binary patch literal 46413 zcmZs?1yEekwk?W>;1FDcJAvRHf_rfH;L^A|Awc8q?(XjH*0=|EcYn<}_y2dR{->%} zb<@>*tv#2_F~%G_W{G6FsV1Ox=Kq=bkf1OzlQ1O((icsOv2T+tK?{^GN(goXnI z1WMn>3o@P_g#ZGg?p;zuNZEDyB>j^%rZ~aC2JZ-w=#O7O1bj9_VWIAd>EL#kYEfln zU|_9fShbp*a$YcSdU8FSG69$U?(aZff|+6dA}y#Hu>UUt8U;3R`^@!CFk&tJ^t|<4 zfVZ}mwFM*O&untSwsSr;RsKb&0|}33$Btq!uU`6umRfo`Xc-`1V)^azKzJ6uxYL?M%o0L=*%_*F-eFTaydlpaTq zDtn#(O1DNlcS3rbXohnq&(YPu!V4oXv(TwEQ&L`vuPu;Gr;qzoQzm5~J#m2dFLhNE zN-9%G6pHv=F_TL$5{|SER{_oR%-rxlI65~?(5|USHw)~}&nbxXYRk&}!{g&4%zf_% zKHaC7>|5)9Nk6OH<3a~EY^3uCcBTo5L;zMYoBIQ5iqW@ZzUs;%zBl%ha zid=0IYmGhJm4Iv}ag%m`G6}|2L3fwUVa|v)T92%8qo%}nv%%j@N|3eOZ+iV<1VMPs zf)e3eV3Uo0bEdAhT=h!g)9PDP%lkgTBkIvYQt64qxE4zQN z_L#iGS^$@vek3oOs3X*e$$r8?%AqO$hIXF0?2|@Q)K)K=ZLK{7xocV0g zv~h%(tqkeodFOkH?aSbRu^g=wW_pdyT1zzH@jqNjLIKYv)EJh^^ITjX{kX`0!_rcW`r&8iUMh+$iDFsqYgxCZo9UkM=Cc9Fg zMJiCP+2vdpIl=?~95u*hK9_`)6bAPGdi3me#0K)+r0k1yTP(eFG(>`-NHd7>Dfgfk?gb{=A0RN zRkV4G%34oJqlCcGooW2`H&!JR8gzJnC{lnhHwl=uqw-31kDGGfS79m@#$LU&*^BU# zdB)mG6So~4kP?K&tj{OXK;_9u7U+o;Ifl+Z{H}N|N1l<*ZtcV7d&HF#s|5QxmMi}p zSkEy^sAscM9y*!F$YYdT48vg5ALG7TK@YX> zh0HPzUSBT$wWDySu9NRy@!SB&B#D%igXe>2)1ptVHjymW(w5b=X0qE|hKAc!O?WnZ zK_L5eB6?g*gFYJU{8aj~Vs*kvkSM7%D|LrX?D>;o(@GLe8-%>`*g|uTs3zOY>{jma zEw)~4NJpMHE628Uusrw|pdivGQ%*;(cG znD0@_KbMq}FR9|;}Raf5)gK-1R@R*H+@3^FGV(=LC+#$DmDcMC&3BO$PT~S5H zI=&n&3$?|#?&}RpU_3DR?U$lQDCmTgDt$iXBk< zectKLUPS~w(4$)~Fe4t}^_G&>X-|w!_El7ze)jn)9^tYDBmYa7+4Jp+01Lt~(CGN;LP*bjUN}W!Ye4Ty?mgzW1_J`D+A(F+c%t8b*2tx#5Oy z&-1p?B$o6Q(F5t}V%!CFb;pr~-?<77;4f^cjAgrymOT`gbH`GSQ>H}PRJp~ToWBQc zYG^U}k2wIISLGnVPrhNduSW0vVo(V*ml{eYvI^p<3mngi6ZHK`_6$ z#nAcWTL4TtjWwUSZIqJ3PcASC3xi$$ zHJ9C(gEoia6XR@V>T5RHxC+oP_jJNep1u1`+@>3RK7-oat{n)+jU%G43_CpT<(9Qp zAFuaQ?b3pbl)J#+nb?gPZ3E>Lv-a;+}&GHg4!6Q_b`oR z(zvDtJKcbC2K;6;&hrq1m{Ac#yu`Z)LjzdpSG4K@bFdmrjYflOl^#sHE(dqlC?<$>uc>hKCqR-l_a`kfI*&|PD^`QGFHSVTY~=>9`^hyAw zP|U-fa&I})WwT()yXTO8m5(`3n<))TaE%etLEgA=vO}2bHg0JCCGd1GYbrv)P3s;( z_SdkWTuO)Bx==BL*U65vh$p|h$1%_Ac_IzdT!Pk}{yYY4-EwYp9SVe&;N}%B#xi@( z)a%{OZ24#Q@}kNeKIsCI0J6xWEt}&G2zfk5jvZ!xA9+v*EY0IRL*{D1EY7~Ya_Z*s zQ7i?!UyG}!FeeEWn##*d2Q09ZB;Y5n*sk>&V!Y!*DHAYP4!Pi5!d->XT)Qce zZ#YZHAHXdz#{koq?L5W&5`^fP!GkG4=`FENNDVyfK~GvF>9-x^x~?Gu;yl zjeR(6MZ=k()D_V(hYO}?hYcNL8)l$h92Lo(bHSkJ7bz)AiJU0oetQE*lrkGBtqza;Ddwem(pO7$I9DL7#$+7N`ILE`)Dd2 zqHcF0uk(JlJ#muM>9zAxCjD?*p` zNo;Piw!Xg8hNz2C>GkORAyQwVnbarb?Fvk!HM&j6`i*g?&U)Wt#1|@eLO;QsP;Y$Z z-vPjr``DAGp3C(_z$bx(o{nQd`UWD@ziqqi0}%f{_37%?`Z9CPhW2@c3>deg{qDwS z*o6z%VhdGuZ6uYCDxCmK&6xd-7jVINe9 zz$5IfqWY(9e_sV7<)!3yI+krpYhM?1=6?SYso1)FOH>#n8t3&kHzOtj`!+vOH0FQg z>gwuk3CP*s;c+iMqfx-)LZC5cx#}agoGn)83J#*k0)#Okp@Br$6b$-k4Dh(+S;zO{ z6mz92L$Tg>J$gIb?!Tf>0-Pu`qHI(Vfi-VvL5(%m%B{eDELTiOC95ia49847es zf|2o)2~~N62{Rl4oUeGRRdk%561mSzc_s2DptwblLF znNjo4E)|tG=9^HHWcq9gQ#!0PS%)%@H|rIAN4K?qhzo5=2lt`&&=cpKxdMq%|K&uc zPPyddf6APRv%x}aue?gY-Zij;3BxCgi#*(Qdl?zR7lrDeSuHRpWoA`KC_LI6y-@T{Ef{NS9ogj%NkS8 zHt*F?klZ11(Mm5en&&<#7@spCzv|baVKX4PRps&Lux&{TlEreDonFCPq+*G*Iiph` zaLa0m^Ncp4pEsn7-8w~0xMYAb+SIcs^aAYAnU6vYNo2lYo*i*N2V(rbUx~|S8fpKX zS(%H8W%RG*v)}%_IVfJ-&BE-ss1FZ zVDpWF;&)hLq8c;7ZAw+k z@e@7PJG`D}E7VmI4%He95|-;NUD96T#*OklWt`SlSLMj`XVJ2MCH0Xi%doPudrcao zcxdd2$l6mNV#r!1GH`J%g74#lj>%^V)T^A_EH5wf0DJKvmBRk~{Tnf%^HSofK#Pbi zn{TyL1E}BCCL<@ezEb+7K`K?L+kTe-^hFO8uUg5rh|r#*kr(aRazCy(+1-g?McvnX z$NVrSeC~BD`LwMe5mVg!mPYE2uhP`sY{}#B<=JHS}TK&$%~i9NkzCB}Dg@eDWm}o)X8$oPSA`to5j2zJPib)Yv#Q zY4^WW8gS#gc`UJ#m^Cs-We9q_GogbfY|j1yhjZF#8rC}he#*-+kq$Z4$k1k4MWr`c{&8V6o0!$nl2lRv(W> z14N@|TW4w}j!{Cq2BX7`F`B<|4KC>;xG+gK8(H_HZ;KwX5O5)pR+^I?x{6LPg^LoH zMXKGw62(RtcYY2%IPOcIRdL@eP;@LIrY!uj^tl=FsS@T$bp^hIaoAU(P03{4FAp!0 z&;@geC^%c4NS8F1OfLgZd5IOUXP7)Uf4V&J75(0bEyXPvf7ihZ#2aN?tjat+CFWgCHon)Kz1<8+ zMeZOYU~;HS#RKZ|fH{}>>=?j-O306_LKPel z?|iZFQrw94n~k8kWycDZNpf<8XAR+KxUmwu@(z{$kQ$(WLqmi7>B&h3=JUkgUBFb= zbVYuFaDGR5$)~$rx#p((>z&~*ymyAR;C;Fu`d)_cjvOT>5W@~we<37D_*?N64u!Bu zg<}V|=yZRS&ZDFj!2VrdN-2|;K|)G6%M1vJKK%4!1zJq?yoilPc6%WFn@)#kGy34J zbRrY`{N1&R%ZA18E-SD=kR!sR!1gxnM<|JwspCQDZd z*!Y`)CKczW7R4nkDMXTivoW*TEcfb-&r6}4DvP&=*NNu5rVVrS8XF1mZ%tm-3}^hv zmKwz&xn9lEl-SJ6AhAR;QfiWo8xmCx`Cm;D%Vg6HBZ&=IqsLFy!BJ(nXJhop9c!+4 zOQJr{_LOQ`&3R}Zb)=b}+3l63r!_GqG4Sf*=8;Fs&J^n|9{C6m=T~p==hWecN}fwX9I9sgFgzcoK+sH>>B`8UKtSA{KbA0}UdTs9kseYO*Ma6Y zE#IhCvX-bE@B}VRM>9N=#82XQIOcAi{rqm(7)xB1E;cu)H^jCu9zx-GRd|E5nWxm9 zt8T&D+lXJloFJ@P^ZiRsA_IZKU!J7_I2t!8iO^`2Kk*jTWH|N6*v4td+%ncs zdo$lUMW(0TwEeTJ=hG77qJPKJGG7ia(f{_!%`r+=A&~57Gsz0+P^t7)P&{_IXiV0t z1J2V+*sCI|DqLKZ$y`<<;O#(zvhVqP>O1x07M>{a=}HaKN}X%!nvC{t$JqJ@S=Q*d z6V#;Z`WjvB9+N$-EiZ!!7A;rYl8*{Vf?QNEnMK!_8+mG-RD(+L4}a3ib_Di7`1(RZ zKte-6!9l8~Wi?BYoj z$uDZ3KsorMD*S4CQGdd%f%lB?NjX8Vp$!I#185PVUH`qZ628Gk)zHIYBLlDr2((V zL;5wJB2#Kids6+nG@H2$2b_1#Jiv;d=RQ-;p9Wqt<;)|HZh(&vY`?)hEs?N?2Z z3F;y`q>|J%v#}4skK>8YMcXy>%C%`W^PbRW(P0EvV~0;7rZtEmSt=UGk9Nmg$~z6$ zz1SGzL@)tl`gN(Rc(3ctJTJriou(2Z88?5g$%{y3LjEVd1vi&BkMuR=>L`D~7}vkb zp@0DJ??vyezWBF}D8WQw8|nE_o%bzIZ!H)p7#-geb@*eUG5-iJ(rP>-v3eLJ@tF-| z2cyO>s~uh|=&C>d6JOEMo5L9#+6V`kAQd#`uO#ZeUnS6(A-?KEfWHR=sT8Q=(u~Rd zT6sMhu=@DzxR>~=zApwQGz=K0eoQ+sAvhS`R2>o$8X-6~c19NbE*Gopd3SxeRY5Xi zWRIJRr+AlTqcffJjhN3f`trI>j^}~~s$enrIm#kgugVO6u2l6ODo$qj+E)wn52OeT zR4Db1tkAc9(>RWBE|$t(AOEJ_*Yg%I9+w^IDh0=3+aZ85R$M#EvrR4JzLMTJu23eSC29VzQ(mv1@Un!@e|C`Aih`;q;(x>cv6Il4 zk4730D~H>Q107$4pr%Rn0(s)*_Ap+r13c}={=y^RWt1Lw)Jb4?ILF9EYLW;Iy!fBq z+w|#_9BO|-`O#&@!2ti?q(4}z*fP?(F^Wz@$Xn4)y`Cb+y_Y{om=`ZtL59IW7VRn& z!Swuld%Ni}SMYQnssJugRnX(@=GRuIru44IS&mTp$tDs^*b_S5p4e8^*d>!6I;^HR z3w-eNX~ue5h6v@9tBvRG>=Ni^D04Nq{k(G34f6JEhSpNn!DhYk>NG7=QILKSgEpdl zE2Km23y1c8!^WSY8BwlJkh$KAt{G#O5~{zjVn`*uWxcS|N6{rfr`|4#JioiIHg(ZH zTe_OUO7mEA$7`_qQdj=?I!w6086b?tyeuomS4d|>B|Z2FMwfcD>9ww>6iYO7%~^V6S-IcpAq%GmfKNcE+H@0=J0Uf!4mM6@Ob$9}zk zZUg$#a=6zOVYk@dSM1)VcXG@>g#HN1WGmGxAGx4|Do}s4Hr^n6KcI7QkP;FSS|=0& znuDPqovDGrB|b5DBQwB23E3+u#zeuQSxy0VN8-1sWs=ak_FloFrcMwtK?Az#&#pM0 zx2NOE_>?)HF8d7o2X-q{-d`c(vo%d9{|z7gi=?D9a}VqWRT=cs z2ceRjX?J)&CRTli63aDkLqRduxCUa;zaZ2^{GySe8xyF|ZU+70=x{oo_q#t^D`jyx zTP4x=f(cu`o)mmnHj?VOBCyiLW(X8YUdD{`gZ^X_A71-+jFFMiNZkMuOb!$X3S?6o znj~YXa9Ul?y|XmyE#8Ox^YaY&A!y*FH#)pr#*YriTOmaD0o>SxSK3i?xJ6@k@drISWFi8|6V9@LEjInwFD`d{Z#KfDOZ%>v4Fi2Sk2WF*jug~XP zUQdqn9Iq!!wNh159L~GBb`C1agL0n zB(r}92d=5@K|j09z5>?USk7mQXQDr_*o2p=}IH&k1otF&4P6;9`_Zh4Z4Fs`+60x^>zse(%dtK-Re;!38Y^0{WqX@Pco`i~b9cCnnCLCXbSV2bLj+W6;d*hjdm#at53Rmm-LOY;+#NSg8i zLy!l?kCu4qsdw+tobK}S!@W>U9>H4l$KUBBmSlHt9-p~!`3u$@8*zA%HiUR+$3ys8%pjmIK|lB(S8+z$+omjBSb`~3n~ z>soAAT6P-N6u(|H!(Hye!ltPWJzgcFX*j8IO(3^^?N8~4LJsYMitu381AX8g_R-d5 zINR$SuT4%8G!-pr!VQwFM4bS`xs0FI{Ln_7*QRYwW7${nOr%BYHO;4t1kD2Le-kc= zH)%1Yhe{78q#BcY3q#$r@p$x+4%U3eDjfntGjzCJjre%Xop1fuVk3Tgxq-cmJibzA|F_i{ic z&j{Co;_IaVf}1ffF*BuGqGbiUTuWC{n$3y$-4kj&b)y!!`C8VwKhtd{5^feoC-}s@ zMXGnbHQp!_%Z3sY(--~$tiKm7R4S;CQ7SRkWbE**S=tSZx_MV{lwBiQg0|l{E4p4Q z(Nfrbq~|XpmSynu0g19K$*7g^q^`+0?=eU9L^?^t_lxOKEv`U#a|;*ZM!AEhJ3%(@iQN0x_@=_9~LFrC^%*GOSQY5ze0#9^eWQt=@XpEcZC^8Goi~ z_CfXdin%N_7<9!&(dxPwsEj|Mw&oQo)7i35LP!Lt&8G4i0Dr=t2B*|stApKLtg%g4 z91GO3!tW%t-Zp5=qFvGC;yl#`5)wnW)~L9mDhAulf}`^sN%B};VZO9Ep%OEi_>bCB zqi%GqLLbPP7)*u2mIVvq5(eWR9$dG-8;|r=Up6&0eIFPYxLkgNFFA`e#m8Z0x0r1j zTo$ugs93YJ?o6eH3vT#?81NW(+urMm6i7>^9^iE%=XkPMedqdPiYZAx88FwZAhu&L zOE^!=Qs(RXNcMpp%QHf1nJX(U z!JQsyFesUWm=L*s`zn7jC$=$P{2I^w{Z$kxGg7K&`B+OJzQb3NPx|*% z%`Dq1-JO}Rz!!@hvuaUG1>udOl7;@PvSAfQ%1kP?dd{i(-bRaZ=HmM+fPiCDYt-2C zY2GKxLWSE)obQ$0sb3@ycH|Gg^+*=_i&xieXtm*}yvg*Q}{9M0FtH>|qsI z()R)bdL)HA3HwNY&rVWO%T&4fosz#<#T_cgzK`nH+2cGa2`GRNYQ^^Ch5T)6u|`sZ z_FWP4cXY{b#N#7XK4s>MIH^scg~L~A^m)m#E{l)|DeMZ{#MW$cjTTtUBdOZ2uGiV$ z1NMe6k5q@N+cb0XhUX&Lk?9e1Gn~#k=@w~f?EvR)M;ptr4G-A;BOm1K*R^kTA9Z3#WL0ZT! zz7ZJJRckm{=gSd!AgV+D&-67o(BaLHttPh-QAp3El42hW0)iy9!hywM+bC=LVL>kyzUnuRXDxBA z*z7@rqvV9SRwt;;G;KzT|4+GLUP;n)KWW%ux!d2ujaJ=&jjj*F+Nv55|hNdtX1N zs2-I2Zhh{A?bMktir1E*NE$}ujeJOO zznZ=oAmY(+%?eTl)gTr)WL3LS8kn&3Z=C5%%^`l$`VnqUrn-eKpz5czrXhO?!$3le zNKc2;N#i@W$A>741Ad~9(L?;V0Y{5&u()Lo+0aX1Pq)Rj6v)wK4Htl z(DqQd8UfOuC+1-M>x=Wj$h&U`>v#zeXt-G{j^>6W6W+^?av$0#w^cnuXGka(c)`#} zR_L}LJ$>MGt06lqY)A+w=pWHjAKa%VTZC-0o9ts*Js5tJAE@uy!~yDXAA`x16%&K= zIbUyY$DCPYlinZl4Frz_;zENNT__sN)p5aN{=dz~HK(t7Q3c1+IRWcstB2tFNDAkF zm=Y=YrXW1AWkljWBlW?4Tv~4o%!9_b)@kK>>HgcdjI8X&LZ#k2oVXXq@0luuk1GTJ zfY}b`ze`sPjz8=lE1lhN`AK@8Y0Z0{ZBzJ)T2QmDj!v9qx^ni6P@C^vydt+jf^GVZ zYM>|XY`nCbWT7;JvqB8qIsRmg4(C0ed;IF?)Q=x|{dsmnlt`1q(r_!?UAv?mx5^MtJqPKdvA?YS#+VIrbS(1cr`Cv-kWgN2i#=rkP*navf7na-?-$eexHDdk-{Lq}dA6DZ^c+a|ICZr}`h; zi&r+0N)*JL#xv%M&4UJbAi+zIn1{y`S>Q?iY5QJSgioi&!Fn)coI59uVD+2aAqdfg zq=0e(B^reQhcU<6$1^J!`QH4H9bm2*>*PRlTlqO}#gssW?aQzJ?v)~{Z!wDR+CTQT zNq&Vo>p2L{buOdB-%Mw+gC{pjQ>Mr!l!-g`=2+<<+1k-iNNc(MoB~hPN=&ICosvLU zfUz?{!V*p$MDpHN7$NhxvNuzPewL?HO*y{d z^7rNed5GQ~!L6BrycC=DU=RyY-yJ-mKlsu2T;F?=To_F!zPenhuh+G$4E+_iHqH3w z?RI}$%TJ|tm&L=_5k;>Cy7540yYq^m7;TWIO?rm>_%&=?Epu+w6eF_;so7E;M;OFc zW(ZH9CJI*bJC*_b$0vyYHsBrO3lai5_%yGTurOd{f>=KOv$gZj%v9>(jG~#dekhvw z-=<7D6nRmLg0tpqf^*&Nq&I~oo8nS=Y9Dd9xw<+`GQF4(NEE}vEmqGJW?0ZNt}oiv zRodd%C!KKo(Q#xlnV1W1CTTN#w{%VWK&ns|`D?VRVRS}i7k8Y<#0Bq1aOVtiQg6M? z$nOD%CRI46IC6O?ub*On1r3=GsMwDvlWU$TWNI>YibY`afZ}A+9#k^Ck~^wBq_@zC zYWnHP|2BMrnSq$LYT+MDwyEcn3eeJ$UTE%g#-()DkY>*v9C2X&oH+Q)%nT}o!zzI( zmKaZt-n5Kd;lWL+oi|+d8^4$?0ojo6itF+=gy!uwt*8S1Yg}{rq9RcKKY6B^ zC`G!K*E)ZT4B6^%eV8QmGQZ1A0q4$HA zji{QTb=hpHXIP2~wv?&fe-ptk_gPiV<{R3OzWrJ-1Gg?k58v)P_8kgLVQLY}X-E!F z=0`A-F<9Z5n!;c5a3+}Z_lF+DU>|-#p*fYLx!(vYyOE%?_|G8-)e zWrwl*jg;G-(*BQZT;b@n63<52EP*{^ zlYFD4bA2TIvaJL)cwJsSn`O@HA`2!eh2X8jX4`8f?}qeSC%1#-FW}>4&u;&0dq_|ZhvIFoVeXg^r|c7Mzc!g&RwsG=dVTwwyFHVO1+Md z3JW9rO|Ywi$9OIXTyBA8!&m5p^FH$QJZE}+5SjUw3L*TqCTKJ$x&1yo>}A5<@#JG zXlSqGuAw~-SJyv)#VUg)tEbK0AQeH+2mOPiBXxTDp zc|F*FU>l0f=#2^wpOn>v^pUgbe8H7|R6IfNmyr)+S*n276PZR!M~C-FiAw1}btXgj z=kUTp)1;bHvM1qDWJhD!;A_%WJ>9zB<$) z_S&;@+N>5$+6lAhP%&j1jwP{>o+Rp39Ci6YuLT4Iw1Y{cBg{JNN2*P}xp7Q4pDAEm z2RpJo%*+lBCenFH62=|CJ_Dbpn?u$9dVurHWcqxTP{46bnzTYTgIYd~McGjJ;I?E^ zu=nkp+WS_?M+6YM!M}Wde0V6kyt_j&6qPvnI|6cYbOa?bYH=jLDtpNQfA#>ExRlp- z?B+^TNQec!+tO>?K2+GTzCyK1U5`Yt^*Zn~Rs{w4q|J|ce|xSRDN@Lt|30mM&TnOH zojJ01d%OUkel9+)?(}(IS6+(<+1<|xTjR1E)Ktnh=xQ-+E zq2d`B7-XDy{RAVL`=LFRV#VFgmyChA^u%v^2@84K#{C{6z=y5v?eN!^7u7UgmkRD{ zd9YR+!~y+3SM6ZZV5#C$~)CF-^F zX!+S0LioSKV%Vsfq@ZAziXsIonfD^@t zeNCOQDVf!j$+!M1a~4FEKCRg)-MkqYJ;-9E;q-I0Is*nmpk$4kQh|)dhF~(@3Fa#^KCYLnA$Ts_ z$_B_SVg1|h4YtVXiA^?)t3R>d8O@ZDD$e(zm3{V`R#$K4x3pGU;`=B{*xA_`FE)X- zn#-;nflMzSxx_(z{hFPnVc--w%MG{R`*MuD6fl!Pqg$V8kv$i%278; z(g$oimz6lyq{73Wf4ueTFvx`6Y{c>SJ(|4OJR-vvaheo}v9Z4_*E(&d9%KD}&a2^;qMuisRq?;1Yv-xsQHPQ!L#7Sk0G|c(XQ!5**iDj9OS&x~78Zz7j#RjjW zWR&jDL_m<4hZ<^*!UHAbC-~KiB$4J%KU>h%?ido#gT!?h6%6R0gu#BglU-TwP4v7) zWBn^e63s7BA~_eoFg%*}TA#Y!=FE;GeBvg{8hk}qd8fOlCvGBRY?7Zo=@tUT@JQC) zl917V4=D-OwsS8A5VNn#G9|RcUEa~Hy-y}t@GSfRY-&AVcLa7}?m3FAil#X|+T*Ag z;bUV9?ID=b9z}77ErTK&-j*UoLdEu$fZ_X+_5h3M^e9_kUk7sD%G=_dl8yY=%LaUr z=xh-&9ZAdw@<2DXEH3BYo;h_$IRdM&0Mow0IKfvMTWRF)!3f#EZA{=eXG+eA@n?rK zDD3vl`kSgnppBb*XX8Su7FjBf;uXa+Y zm+o)fNN=)xn1y)j0`cTBe&#@s9yQU&eS>e`8aoLq4gFfNbEtvjqwuP|aoh4~?P15A zPlvSeN+U2dc}U5C@AiyMlQBgk=NHSRBaF!O!mu=l;3GoFAQIV;W3=vDO_DS-^}VS0 zP=VEiW`-E@p6yB|c_zw^)g_lXoyz>HntOD_enaB5p{D$C1Qt)+h?gU??X`||NxgS1A1m3`btLxyJs*>dH-ssU=u@Q%%gJ>l zzF)v2Tj0qXPI|K?@cL-Pmn*x){npH5CNZp1p}^>8rvRps`SoB;((g3OQ+vH=X9@K>3zGL?8iLm3}}s#4bH@nE?Y}t zDYt5})cAP<(mkE_4}-ak+q>ZB#zi~5w96IhrZzu0sRl9BAg#Rd76XDRhqKA1mx?PK z=bbSgZ^qW;XDB&oK&=8HkrzXKBAJFpvBh9_$M3O#^M4-;mC+iC+$x5gfZpe!sPV7# zWNe9}2Ht!?M686Qd*f8FzrfYbQitw%KLvt9QpM3A<94DEUcVD&BBdW9=Xs9fY}N#u zjVIV=kAt7FpXRPay<`@@x$53~T{dUy4Z|KMk4zTraChhaiB?c=L;Lv<6BBZy=OgT) zgdb|jj(%KDZf}q&Tk!1cEL5XLeraza8ajdn8w|fdJ3|$~$w|E!&0kG;NJ3to?#!zw zzbg&#r1CxeJsHz-w5ADx%~u>Xv-7Jx<5bQ50@4&g&5I+xXDMD@Uq`F*l<}m49k1*`vsTTiXD3+Lc?6Znf9oG?kA?{g*E z9sHn?Ledc0h%K`dAMe&8CFRC~je$Go1A8#e)GBV`wvso=(0Ii0m(Xgk9 z)!Rn45}{>9h4p76hlV5w=1*A&1l_Lk2#z76p=HG^A32Mye>1i2q zAFH^GFGmJ6UTV1Zg8vwQWwwAlu28Ylq`{;j9W5u?IkQic?QL9!44qlEL)!SMKMMpG zqbrpY&OYj`y(HKmZSRN8pytPXhZhJ$b0!9X>rm5(%PKS&F-Coy+!O!4(VwUOA@F}h zcnwf}Tp|8+fhX6uD^6}8f~fUrD4P7rB&bBS{3!}qJPfzm`{QW}1Jd!pM*tYeygC%# zgOT7qxb|Pr`!facdZWeZ_{x!IrrKyQ*U!(d<3GKqgKtQ^AMt8BSNyIf^-Edq5(5}z z{QXYq*JLIzQz);O#~?<^goazc4NlmyTudVcnfxBApH-((zy)`u(CvsVwz3?eDz07E z0^GqW1Q@YXv(=_cN+Y9fR}`ZULjo4PswYaR*(Y|(`8G?io440s_^%Z{`1k5*HLk|_ zC=A{jguQg&4k(zIDv(M=?=CibORD}lqZ1KX_j`{11;6dyihoS#|5lbA2vvLd&`#pR z^R(Z$_~*BgET^+q+2^RB!EQ_2{yEO?^8)vo(Fq^bI|bxQ+c;M?2=TXDPAm+nDX!^3l(Kf95x8 z-~y%VyVyutl|V&)bxv6JC2Gi#P)GOrAQO;0C+m7!h^z)@c3?D0W}+(mhKX%Qyf>4;h?Nol3`rApQc3Xe1p0dzb`u zxm)b|{TVjAN8VhMn;>or%92@pc7V0tIAzUs%JM&1&@!KIV@!wqfy)Ny>uXFIZwz>& z^%&yErK7{sV>&Kzo@0dRvy5lA87~?7oPu`-cx`=UrlUM03u$i~{>Dcp{^<%_u{M+a zfdPIB5NE9eV=`vZ1Eq0XkF}9RH$(*TK^3(F_JR?n05ev|fEbx30xw2BE@k$v>b18^T$>Elu)8*z zZ$5F;@(PLJR29pg5xv+OSYo@fbBxd&dc$ykg9%C9{ojtr9*yhqa0+;4Na{JJlFq+> z+q*#pG`}O9bKj0*&hLy4f>`a(gkutl95?uE;%B+~-F`j>(w6rz{Rm~fVDtr$GHZtX z|Kubl_y0F1ou}^ZA3R;`QLJZjTMN2&u&rdIiXoDNxg%>xq-|nn2q|TscUH@`Tuf>$ zY#8bW)B`qDZZ&)=&G-PqQTZl_b?~<2$!^=5dyl??96>>QIxsbHZk=+H=DcCWabn&3 zhqL|t+(<3RogHU?RlpYj>4^xIn;{_lXh|I(JYB^CrWUd}VeiUOGV2fjRJOCj_@SI* zd6uUC{{1^IPRbALR?;)th#I{#DR^L3|KfUc!R_+Jo*@YU{giO2>A%Di_@EP^yE5)CdA>G{kLa5a3kafR*uu~@7*Wy(hKVcIXFe%ibjHXru}T}t-SQP7axKfH_I;g=eKol ztVG2^i#cGQ_m6TkZMPSL#}qqWoHU&|d3c8p=?FJM+lzx=GDoS`h=Uc&uh{qYXM5>YWw*NZF9HPGuJM=fkN_>?dP@E`0FVW-~NV8kqh6X65!4 z&Uug}lwDfRV9$eZBI^zw)}7fO&TiKxj-c#+$*=zrVn2f5^tG!u`6);&DqO${ta3rb zX`WS{iv>;ck4Kx#+jAq|o8@kiOHx(`@rLxc^wNSm^Bsu2Dpb~P;}nUCN{mmqi|fl+ zqj9%|(I~Fz(YmsJfCkH#Q^bC5-LEr~*f{1Qi}uaI!6%|`TH<~V^HCMml==OGXouX< zX4b9+*!&w}r5$>7dh<5_Y>s9cEFyH;7MEfhGAm?qvLFs`NCi#ZFCvD9)FUyrl!|K~ z@VVn;^JN8to0`uL3(uctvL1QSl0L>HW^^J3l`^uc?*1!rxU|UKsZ|^C_b+B@rt(Bx z%KfY4?3&@;WbY0vX`SQvY<)Rxo=@7;pOgd&Io|D84n@taYx`+W&AF@vpMX?<&ST}M z*U}_Vw{*B55o6^FJK+oCbC`C}yyM{H@kJq(?m#YSP2hb5E*oyfq9$rJzh4vb)D^Yz z)%WKa?P#vVCc~#T1d@276Q2kIWlWcls7?o0 z!H;PA+t>(-Vz;9X;Y=6*n|Rd|9bWbE^hN+rh8Zb}j;XXy*u(RR+3EkG>#f70jJiEg zr4*z?8l+p0Zj_Yn?ijke5s)sC?k?%>?(Qz>?(X}JdcJe+bH6)(@ChyTg&xX>Ys7?9VLI$RzWYMEc(vk1&=w;>w z@F~AmeG{ZH6q_7KKoR0B%>rj2(MAy1fGLJ<{V38Cy@e*6{5xUJA-52mW*G6W`QJ1j zIiV(A-I0)o?W-I54+IUKDY&U|eLNtJ9tjDQAXCPbN@vXc%AH9PHcU-kXaM@ms?{3V zA_94TwZd*Hp9MdCut+*osI_b=(N&hxo!rTrOe-iisMHXXwR%Vy%%u8rVTtXy3=@bK8Zl?DR?bu&wK6Ho zVx~1c?Z~!u#ZLJs`jyls|H9#lieF5N{12Bk{HR*<>BcL_ekBmd6&wig0*^JU$Ku)H zUd67x0praNO80LQF0P9Hb{GCTqG{=;6=_y0e(Jd)?hOz7f$ok9WJYAK{}bE**ah(Z zzdwMB>n9KcM~>+AC1UcJ1v{v-Zb2qU zlh1qv4eBcsOBy=H#;_szNHDIKo6Ci4UP!x}`yeybZ7UThz|GxLzR~-U1xQ2s85IR> zPKo^X=b9?G@95Z`o}M&w41jwMcLvP%K1ZV9;T=K)lH(r>{B~cd*A2(lZKG;~4St{v z^OMW@CMb9VV3gnnVE6X+I0t%OgIOqVQfB~?lTvJt4b}{}fi@#1+s!_QfeEnJSO*{c zygRF`tn3W5DAOYWHYZ@gB!3qUD~R9US)tNiDZV58$vI3_6qxV7}L-MzgW+ui;i zU6j8=&?{GAc08P$uCP&o0zV<-Q(9`3E}=8WP7Cp$WxX;4SHaMM{__e4;FL>)q5*RS zQ{k@8SsqXL)~da&q&mg|lRZMsnWd#rzz}+qg#qE=!6PQx-)fswf|&M8f%aiF@vImp zAr7UdO%IrYDxdfIPGZRiN;#&})N<7jwDibu1p~7zI(7WW?;HHoSc{(ux_>>^J(cc0 z+z_^M)HKCG7r}rRAME=P?Sz@uearGiS~_5Vs+`S=%Vx9@?YVjQ4{!oUv=FvYUNgC{ z9oh7&X78`uV>`sv71qW|lA=G<6DTdgQo5y+cYbgz`VC*m{w}FO8SQJ{B*Ab_f*a?= zX{{wGFTq3q6#`cpQEP>-N&8ExA^GW`Ti0p6uq9_n_S&H zlW$_K*}{%nqxf%z6le$&;%f7WBg=X+M15%9gNg71-1CzHww+xy2SI?}36(w3c!2z> zO#bngb2Ag_v^cIK(O0s0N20coYBTX7)n0Kjce~Tfj$^uT9B3j%DbI7_XeRbMHt4P@ zOlI`&5d3=D-&hXc9Lrjfm#6>>fzz4x(luo=F{VK?lB4N??}|g8F-g)pzNt-0dH~gv zUCu}Gae@{O9OZp3l#)Lj?c9~#L1Qk^76OV5@bhL+>@E4|3N}TeJ2_Gguc(Fhz!ytD zZ}iBXR8q|IJr>0OvRl)J%*&@o%%LCS(W@aG?sp1aa6-L%sIFf=ktv%rJKzI@`r zXGXi$uMW6%GRt&HCYQgNs`#rCwza>K&ggAl60J_o_c3!`JiShIvuI@=u5R78xU$-O zNUT!>0@L3hGF6&l?htcN7o8s{eF!M_B3n!(;hJM_R1sr#~RF)EQ68Q;B=ylHp z>C8DkRal8l51#JI-hQ1ZoeWK!L*}{vkWq>dB>0iD>QM~_>J6YfP+FW?2+RC#31+TM z`9ZvzE!Qc#NM&!Nz1MUX71}rGC(<+BDE~X-sY`1L-bwXymS(QpEQ^=5YQfRe)n`UO zgzB7cT%!f^*{dd$aWQ)i4BkpNmFZhAC@zE&-|N7;c|UmhJB2}>^V;IeXK`{PXg=q)M$$4$J!9e+l5R#ld#<6Ezg7vO+*7v%k}y_Y!~W}T};{K zjVo<7V5ap4d-Ugz3@IIp;({{?CHwnras~)Mojj!iGH%~ZFanJ1AN%}2kNidRM{%L4;UPy$Nsi};JokCX zOQZWZL0O@Q%$oUMt-YUhc2#;yD#YGYd0r>=WPe<&%x}R-*o&n>@S)$M<9`UoEIG-y zIx5`mKJm=PF_b0_ZLEN9ICA`!68k%^nZf>Xw^x@Wn1uY+g&+V6NQ@^yiDe7 zHks!a#VY%1qk;?CLm5dabx$87N7Q?V8y~^&>8?RsfGMT775%nMbt%v`Sd4(TBe78# zVZKR!I#2dSqJX;bN)-rr?+`a?^=S{Qex@cnX;C^-=mhfIJ_YWW*6T}=9jH(ZgUWF7 zL$Ez3KPw1&lfhJfMlL~<;$+$5yX(zf*HOn$m_WlF?r&9q28ysBJBm1Om6R|x@}v=- zqeZx9znHPww=<80-0T@A2Ic)4m>xJOPx`wMxWbHwd?x`ry=Ccr>{udvtrZYn!$|Bi z*4a(%_(J!QsMQ{eEB@f}+zotYy1=}L3>=_c2Fbq}YzfGZnCD+0H>H(iv&Q1rniT8` zUJ}%89ZOYh{_5_WYp*p+7Pn_64sG+*Erm)2u2;vn8-HV&;@rf$L&C8SRf8JX7~fpm zF{2}c`8~VHvgNR|4r}jv5Ra#(T&Xm9uDabT_}^__6|n}kd9k*~rb`<(^H zmXvQx^&0Vbuq!<$z22gG#glXb<$s(Oi z4M(zjbdJ5SK%0S6zb15^{^`9(|MXsVXs{IoGvK%&>Ojw&{!Qb=`+K!}e{tduTy-hM zu#}wY)=&v}Dxz!1X?!QS28p~i4@zhF+SC@x{0;f^a?9n>jU#0TPfk_ z=pUaAinN7!cXQ+$Y9`lQK)@2#w>AplF2a!3+>F02rbHe+R~$lL~B39OjPN;p_C z?EfB~b<>5t>hAhXKhYd^fvy4pMr)mgMIE)|a1rxPS(kMVLb7bFG{0ympJ+dj|It}D z-kzF_+uwOVu!V(vXYfyyUF=~naPNIjqXP?;;Qg~P11tNBeYr2;??%~@;<47{?N~B$ zV;J&Xp|eOFmB^8F5WvrcB#CdONaE8F>~UPQWw}^`SlRm+S zSD7=2rc&qatSbWfQ?XfU)HHQHasGTs zeeekM7)+|)il{Ypg8Y5)Q{swj2l2)8WzkREO$v2)k92(>3YdX2aELB~?!=;?q<3eT zI-S2exG}Xg#HGiG4>ZW_ep_Y=K*r-kTAya_f$;FhJByIf9it(_UE`9Ho&DKC6iiZp z-=|YkE99RNx4O*C%*aMYMzZ$z_xpebV6J2K_(D+1vHp#5O3az4M*4l@DA@+R#v_g%~3Nm+h0x<$j6nX zRe)4&FbAjW>g!X6cuoMHatdD!As1I8MUk^)>|np-A1DAJu2Jtg7~yZrN9!ct?CPL^ z0RwCxW^egCDGx5!%Uzjk)Rom$uXS)%5~jT-XuYHFY5k9zox^Zb3&^yyDdeq2A{S_kNx#3WC^j)>jR(F2kxxF=;e7kqwA4`1UHd@ zz5>d-j8t};Os@b${P!A7&f^CjEnesHCVEr(vQP4bI*ei^8-05$*qbL7i#5|f4s!vd zsrptU7B;p}xqT1@{J01@DykmcEHhA?X_G3iWz(kto^ZXgqqB3Ecjo42k@&)5-$8E# z;UZuwdFsl@{HU#JrP6A;{S?bx?n89{>iT$DT~^9N@t1;0M6%pZt}5Q1o%@FeX$gx* z068OK3jZjRW~Q%y!G7lyeZa-{e7o`XPNJroNQpu4hL1uqv{)u7~nn{3R#GrDu3Nu2q7wC7MP%!$~(wRVI_JUqM~Pek$$A&0hdI z$-zrWu(bNGV3%TzOWHSYWvR)o9B1)UWLqnNk4_IyABrnB~hkdRm zL!rSU4ugDy7#$<8og*H{%M9#~697%KAgnw)4In3!W@kh#OJjqD9&Ub;i7$xdJOX%K zi68Q&OC5n2@E%H=RpF1fqP{{L_W*3k{&}cug`Avx)ZPO>=XC-6xtb;N4|$W^+*}mf ztZyM9gOZjWU2y&t==EsgC4_w@_#9VcdF_lFex{lnLa|z4as@!XCA6g*8yi=fI1@h5 z?>L8&IV#nGVhYN3gQQ9f(Ny>xw%OUq_vcqvPa?EmBon z4y5ut;P)F>d5H^?q_~P9!A^XksI>=PBlP1Q1}FjmH6=%eTumgM?wl_pARuJ%1)NaL zni~9=ov2O1V^+wQ(M?<0io>?F{{W0G=&L{{0OE?k<+R^Lu7|$v3PKa&>t%Nj_wWV4 zN1(W+F!Iuzl%Go(?~##Z1dqjNZ1 zb75(7DB)rsNPBNw1gg>Ld{=27Gm=;k?2F8^$E0sY5hqj*6zh=;6Mli*PJ&V%{gv_tItCLvc6>syk-;hpye4U< zRL%V2ek2hpKSso3#6S>PVB+`&FvHsP;VcqVW+} zrk&+J6Bd`b#AN>%dDB0?Bv^${AeJTK0@GY2udA@q!f)f~*g}THAL?M-!4FbOmS|DKo;Mo_zd;>TvH(Me_oc8{=8q5RrvuSS>=zhI7RP#0Je8ERr; zVu`IgyuAdrj~}J9OEMmxA5J7Ur{XT%X&|$1ZB$iMxJ919ro?mkJC4h!r!>3DNA(<~D?I6oy1BDVMK3B9+r<-(K<5cD?l)zhwfs$PnY| zcUUyfAKv2+RgaI4&l9InvlDM{NzV78X_Swxjs*5MRRvNw9TF*G!jr`c6$&O&r(z?M z-ZnU$@Oli!(p8(z6xGMb8bC+o^i4tm0&$DBbju{XpC%rNgfhp4X-j$aYqBf^2u2>? zZpCSpxGXB)gMoJ~9nKbI;N&b&84 zzpuZ}>xCl^srP(pPzUmLOUMXsPS*su_Bu`GD?r8B?O98y{p!pPNii{slv(Kyn`aOU zBJe=vm0#bI#iDR^H&xjdT(Z@1ITXpA-c)F)wi~F~VoIoaxP}3_zv)ZC{gfaA+e?}V zDjibB00?NfWKQefpH=H)O>}gSlWZFdvOE0|aA}PYu;`Q+5`s1Q;@;TiuxJwOGdrLD z)(utICUQFKr@|H{wsV>EPCoRFqL4mL;O@+zRcc>y$C5og7+-|wAjymd%EV?RWx5xy zEHys)Ct)>YTvTKE>!Vaue+CN~*=KrrYglX6UV}b0%{VTGr7u z#X8f>>S*4Nxk$dLIYGfJWqsV=DUK^yg>?v?-m4h))=OD&zQ~b08yTz^S_W;C`;WF1 zXqRb6iPiI|>FcBv>!?L6<7bRbGEqhIQQWxVUs4j_i+NqB2MAQE8PusqXSAlyxtv{3 zQkQU42*6Afle(?uoYqHqzIdFcQb|fMHJsK>zdQqQl7Ooe&B`w@6IA2@zG;#v&&$8c z-SSvjp3Q0EO>jpn7?_yG88*o?ItQYMLME1#x(E~JHRziMA3e6U9vY?Bf3b6_eEP;& zWQ;-|#p&@<*JvW)Yd?$nI*yR7&Ymr^Mcxq?MkWB`V%S1q*vp|WJSeP~ZxGJ#YJ|zN z8_vyEybnqY{Ys92+oRDj;ZbVOwWhbt+F->n zp1ZrP+Oj+H@tU5M<6UK;j(gK`9kc=Jy{ju$#p&;fBQ)H_O7lvs`EYF;y@9C{npr9B z`36SggnaWj3$6!&RVA_p4@Zp9^73Hy&*Z2$1Qxv&!-b`PD1`ZbwZ=nu%Wv0;|X{j2a|WVqKOI9MtYA-?s;*~LjR6Wi`XY05EN5!*& z>QRL3WOy@V=S9Q}t`xl`74ub+0`ZucY};U=_I_UcTnmI0&v7us%iS|4<^2?mmup^e_JoH5`7sQFfh~K97h~k+%c+jck^Yw{e z^=_XMF5h%FU2)B}?bk1e1X7+oWupBcKU;iJhRP#&-IFUE7#ip!9x`pL)+xY5_WEK0 z!;d>xS{t^D>{VD^+P5SVIVWke28{MA?~?aPO1EySPE{7^HZ^*zMCj?JufCfKo^3df|dMP_tareTh^FL8%Kd#dUYXN zhU^AsBHhC4;imx7aHc*c{z{@2caZ}hzi*m5XxeES)tED$X4(=o9E`8XKz8od)K?f` zjiJuNUeHI{mJo^qYC6*m7WvZ(;jcm{EVF}F`P8w8TWP?A2h1|7nKHvXSB_`-smDI6 zhm~Ax@l-%YxoFP_MN-pJdPFE~5M-R5&QUBOxj#&zeugqO`2n$94eZqg!h+Yz`y%Wq zWHdHJ2~J3!?YB*M%tt4GT2mBYh~z;L1cwLxLpRHL{? zMc4=a*xKle>YIPVjl~EZ5esxBYb$TBM2)atf7v(_I&+HCADF{EO^dge9VZ=CPW0OO zLQwl28zH{gxyK);Fqd!|FFRWFvGB8R#jRV*7RjF{N8{>Q_Pmt@s7I^WD;u7lZuw0li zU4(D)F*mN5qeviFPp=V!fssW>K@h5lT3VboAC6~1XCaU?zIGT|`ea$~!KAXhOhlO$ zhY9WIUdlIWfUW$C5~{vC+Zg5mI|n096njgj&*n}n#x83neeN|~xWLRvAYwh)O-N>Q zEv(*X`K=0g&5w+f+TA%QK1`qSU^esesuxRndr=Rz<=gLX$VVqEEst=m2h`{;%HLj>n%b*e1GK%(m21eNL6#cT-HvYxHI$E*-mUtYJKK#5$W`D zJh#S}9C8W-j~ZY{3GK)zC`?$y#DHe^hrI>{skx4n!@ds?jy=JTmJeCj*Cp}k< zNXiHW>K~dfM7UjCAsu5&LX(e~0z;OjZw@9_bDf_1@}zxPKnEjzAfHM^o5%+yxf<19 zJ^;N3d1Aa&?-N7Y*@%znj?0dtV4QP(?CWDR3%IMFKE5$>ZsF#T!2YrfvM742jd#2D z!&WuCl4(t@zAtjyj~ZPLMVZo!6&|>N);x^!j1HDi_AN8h6A((UQtsNiww|f1{K`U(a)uyxGtshnARO(gL6R;UF8x(46%4KzRDL7d_PViVECZVM{K$9Sj z7^i|f-GM-57z=Ub$hO*JN~{*!`5>Mu8S2^DSx}zc)#c^-9aC<-V7;5k7OfU{ z7xA#IP_Y>XAc4c`XX{LScfJ8aMJf4e0E;jaH6W};{+Q(2gGk6U@kLoc`bgBzRzv70 z2U2?R`2f}SGM{l#SXfwkg77DxAmBS;(u2V8a8`*U7LaHE&GhC2L%Gj5cA;{qWwPxO z@SEbscp!VmTA+f9vb05VkE1Y!U6coNwTG%cOI9bGu#x@V1W0PY2Dq>73}nSThDvSE z0HubY3&Nrn)v!~BB(KM(u0cu+K*t7va-$Y<;aqmxp-3cAcguj$z{0aZc_C^+C~2Wh zuEdarZe{n#J)pF}US6bH1LzK5;s)p)Fye`c|Ed$@Q5UDDU$5$CfgF`<9s3HUdD9m| zU|$4b42gi_)u|O2WjQ$@EO7e`4uc#n2TwH0C7X=Q%%O$`jIBVeoq2yhKUH#i^&B}M z!3mfQD*c|&y+yzh6fLA4xSc^q=MpAZi@p}6VJS!qt&1TF&10D&Hy3kqj z&o`|vVbZGY%oYpw7{JpG`%@-b18~AUpBNL;4zs9eu;$~iv4Q24#Ig7UCzznG4T2O0 z{w)P8tBY#su0Q54Fz)~el#tvw&48*DJF-aNTJZ?Y7({3&sM5Ym%0XXRtOEm7XG%ut z=iF>+s_|LlxGxgAR1_y;1f$~Yr>B0!F%v77?bBIg4nvUE^^QQ+&MXdPj?zZ)jB^FF{Cj`Y}F zIK8;#`Bqkv;H9{{$}n>nx^D|G>wR>5eBLfcVANo5J#J#Q?2P>VQqRuO#@h&)RiN)M zG{Z#HDyiY1w-9TCa0F=M`>myjs^t&s(i|H2G6U_Y<5T6FhE8Kq-SVE=OfC;y!ZT?W z(01Fdz;PsT0-WqdJCK|9G5i^yTiiMD6Oopk9lL)t+)XqtbuUT}9&y-d%h%6+3gjH> zrHIq4Ok+*ezLQ_(-qYOP?lkgaC1{LMssC|ItuofRKltpWrOo$iJ@x}qiw&ve{cx^7#snO=CtT5RjY4X<#KfEL|%wY_N-3sAX1A3xc^`iDXc#xlmuMaTeZc3>dj!$LtR_sM`) znap?#l%?cnktUI_RE8SfhWgMLY@<7m3MlImu~jYND1ueX-tR%?+Sv5Caw&VS`z!{q z%FTm7;(wcwFhsS{d%G>6jpR3gz90X#t(@aggn|dv;MaCwf#(j63Xjv`MSaYz=Q!-H zi5Abo%xiJahd1OiHO}u*q1l?JR{5TvT_yIopcrQG!i0DsiVoVm*84C|wDVFiXI@-T zh7I4eCEVPX0lzp1M}DAEH2;+*j0aKMZGJ+Jp&U5q2CR*G%+q?eLa^Y@WPS>|7@0L! z5wYw}53ST%7gqDS=j#LgdNmcnQ0~E^(tLevIy7aTJ>5$v@_I3w&ZS_R=Nbm-i;$Dz z`hykcs=~!{m?|=@y54++Prv!`P@#X&#YF1!QRX)TuYwP=Hys;4>N0}&EvQ@K6PH!& zpwMv&In>~!9=QVcE<`+N%p*;KnvlvF3Ans9^hi;^lq0H9AEI>2o1%YjijS>|`sQPI+X5Y*1Uv$BFMlbcL@YlIYJ`OWB)sS&my4rC=2yB= z&-mNb!($9>Y>VU#(Cil_a@hJ^LQv{)7>N&?|Y7S<< zuQ$==<8C+Mhgdp5Ep0cj)Efk<#iTM`8xRr8?J4|NhuJD*DcN(k{9U>^V^MxI_ z^F-0*r4eeF8sdwp*W&Hu6OtwpwYN~bxxk0H@6TTa9sTEuf+0D6f%-FnlkDz zr7Pypfkuc~iGfYENUPW$NSD<*Rk~ny6@_;^KU_?gFtqzKevVzx-yi1MT%*qHIoVN! zvEgxhT>ICryHZ#Opc2m}6~;0u4G8zWV}*J%#45&_?OfdWR&98(l}h6%ArfBY{QUd^ zgY0xcMOcVPbgp^h?cF8q{!!gd7T))L+t17iq0nq0MJByn!b+k1GgU6;kZ!`>_ANd{ ze?_g}|KT6Nr9bNCe=hxz$p3rkKSKKdDy9GA_J6tb|5`sVe2ta&In_(nMY=p_Ff~6IDwls8XoajE+WVi%!8t+rIY=%^3?PMWJ5Br9H=*2mPS}cBx3d)!--cHIkHtTCi6H z!6fYsylD<}QFCE>`YIb`^5~Z+r5u4Jgh#FdD#zq7h~$Ngy9l1&29LH+Qz&2FE;PLL z7C~G)exmsL4T3tU$(m*wB4>Zb+c$_6Kjic26txO2^2OuH;M{C2`?ZNyf`V3XxCLBU z2kF0w2@PUfs(kHq?7#6iVbZsF;?-x8;5wl5dj*k&E0>Xo`c)%Lm55Ft`I?Qz>sHT= zwLRk^QIx5`#@2TadDlIrFK6F8aI5gf-_2LKNDOg2@??{RuS_fOYo+|2eF0d_7^zHT zdtF@(8*6xBQ1dDOeB_|aME{2{dgO@Htfw;j8AL%MieMy_;BzfQdWv4HuC)wFo6}gs z<>aIc%TdYG0n1;iwNPD4D?ZJWFzd6DzUh|)f!7{5o%|+nHz(ogJHA^gGe&p>ZNAC? z#ek_L^7!%IE&X!TtJ7}O+WZy-{k&UGJTGXcvhy(r51P8IQiZI}fJu6*rktXVvC&NK ze4Pg(kd zEhgjcfHxb*cb#u}J+NjS5d?lGwvXn;cNA~Y=WFvA$#tW6y#9h2Wm#)WsR&PAz;dd* z&{HqaPrY^7Os zPf{UHiLs>h-!_<$FQcY2MtMA#(^>SYwyrI@7Xt}07*AY8Avg@0S< zWtEJhjG|04DzrG$t-T^_xa-G#Ct-7M40U|O_w5ueOeb|<8pDC#hdgZ^gh@Eom%_dZ1d$gurf|{(EtfQAL$iJ!oedYFz@-=;XqyQ!|1%~hn z;dpw@!$l48cHg|i>tG~PKYh$M`p;v7`8S$un^lpq_oiJd^FGT_Z+>5Td|o?_$fc`p zc1|D`$>TUhu=Mr%h|s?D{5pTSnBu%nl^+E=RQ%wL&Sb1zU;vjG%SQxxNl%oN0c9B# zrB2Mr+V#gOn7OJ(Cc|WN|6G#$4zf>ZOn&OULX;kbREV0`-Y(ApGC94k&@anG%?G1S zFPWw255n+8j!l22z~ytfqBa%(M8(d@VENgAP!}9RVU=!OjVII9W!i&#PL24F9U6ps z$hw%nqVDDCNYO8}**}%Si|2i#yS8BYJw`?qkU`2O3b(r4?(Q18KhEE3#UO3m;*`6jn! zWbL@Y4>fWR(MUj6-7K5f&|YAzKZQGuN-!U)?I9haX3R_ruWlsS?YD*S>57eMF8Uc( zl2buaA{P)qfb2u@m{4nbKQHbiinkIwk;wp8otTNucs~x3Y05IZjktqb9L)P(g5Q?^qvI)k+ZVryEG-QtJ;2)94G=$~){P9ga#{4$3;UuER| zKwO8%hW+#8W$K(tB6nDD@FzA#xldSvCkY8H_i1u>6?`Dle=?adUouFEkB*}uH$9@! zo4-BJ8M&EW7-m4^B^lNcYq9!~EfnTY>#ouR7}hIg<~ouQxdP7(Kj^IxNzn9Hj`%N2 z@FZduZ}6ILKgo1S5S+D&ycfSOIesUIj(k>OwPcB53cPS8Wffg+)Ex49R)>03W`;bln$#uB68+f|gfg9QT9u%{5NU=vThhAe60Z`rCVQHy zwx~d?q`-WYcfz}q@9J%#X6nTlkvTiY07TWuz-@S391CS5f*SVUv8*2ELNMWG2hdyRRT_ps%0bwyo6xpWdBpGSP2a8$Y_dv z0V-(hWy(-ag#wHCN5e>A0!Q+|hkgB=J#T&FalbwN^1nB2@6JIHU|d1Gd-CHG5Y45( zK0U1)dJ8NOSm;k+1GuE*w3*xGMki&%@BfHn?>m`zY}In5k#x>fZohLvN%{9kK#>7+ zhD%W)At7vslNFEqr0vK(pr7g{1_tQef&Uu3HvnKbpXj)`mkuQ_`s@>c?z*I4efp;b zrF8+NXvBxp_*(T{w`1-AEJ0biLV;WWHpjmQ2vZsx-O4U`V;7kBTxhqL(a@rdLUH~_ zfDo(Km^-a9^C*{SC}WZRcg^ZQ#P_em{~yhZ!uou(pC70NQYLM74*-RJk-aRAj2!E( zb@)O4Z$C6OG?G05*g|a{3(r;+;QhkXpZcGzgDMpAo@X3@UnSryKbvVKNN*ec=N@Ae zx#WH|a9oTMP@pB&WLO^nx{zOdQv0`*yi_q_7))UIA5iY5p1at^pkb)j_uNqS zG%cEV2SN`5a}`-~WP6&T^FHke`bOIgSw|Wg`ME{4wz)ZEVSkv3z{ix;)v>;}i%uz@ z^F9`19bVdAbq-X*3kVM-Rl;Ko+%^eh77Hub(-M(U^#} zAHdY{XFu$s$w3CM4N93Wx}M6hSkW8JaV?M|3Z~JB&V+?21o?(JtV0VtjX77{JI<^r zwYNJF!_-nu=T?*+s0j!E;R7B92S}O+JcjdY<o|w=8ZiSfF+W zUfuR~Z)s(Ez^z^-&EVgT6=87hA2045SuK?Pg@uNh8Op%K;$wwEf3Z^eva;0T*h1J! zdrzQ>r%auecGklSqyS7~m~9p*{_V z>j|E81hO~1?WH$M;><@uV}|y!Yb7t5zM^C53(fbo6B>1-4Al0SA}vZvc*$_pGF;vXGzJ~bhUH1 zekqno*!8oGVqQIZsc#m|L&wOG$~n04w3SOC#;+E_!X3^*Y7bg#;HAFXcnS4Bxpl)* z>n9TOVMMe*33Ky8=p9ejcaAQEI}ByE%THTd3D;F=wniDzYAO}=zSP~r}vkf z{&MX%sQ1hhLvt!u2_6Sn+>K5cNb+|pXEy{S&+?@!m6hyXnisA5HgmXUwmv^v%Z^9i zd~V+|Rvjk_E2QzazQpyW*V@GGEvxtfdnk9w|1y1ZT6|3`GelKnrG(65WG|e?x6>xj zjQj5C*YCh(WBA2p{P0n;=3V(!{XUH05WJtiA4B+(&^yStBCswOCG97+)jt`XD!-$* zPg^4NReFKPgwl}|&^DJx=>CD>bSA0iJpFc66AALEqTTHg+Fic?c)1jDZ~wKg_WQZgHSvb z3)Sar0#vsWsldpcoewBf2)6_JV z`Av1}D&Cg+z)DdpM-Cq?PN(lFT*DxsVs1#kV=XZ5@6Qaw#jFgu{KvgX^_HoMwKC5x+=oW9YtnP?(DkZ zz3ugeT?P9y_5oDC74eDR2O^jC_inhg1g6imlcAMYQtqX@AAYv=5+J&|)5XSj&{<>5gif6; zDi@tmhgk+ABBc?)sZLXs#)aJybmM)+KPoH2Q01ixU&n?VmZ<2*V!1w%=s8z*f(?hj z7i?;o2nEx8=5W!C)!0NT{5@Pa%iUg% zdXhX-llnqk=>;E6o|$4)zJ}#d3tJy1;zhGF`Gt&78Hr3S*SQf+{L}I6gtG7!-6Xu? zo2L5&C6*Z-xv8BRAmFoo*yjh&>;;@svw64=lIY3=d+|eLsX-Xh+nsyCbhh(m)t;ZsD^%D zQWt$TdX48FD-@@9H&CnUNZNpy4~f=YxJq=S)`hBc+`Xn+f)N}`2S3Sd7NhcAdoD2Q zp=RB5jDy=1%m23PObknWw+z?_d7brBeyl{?&)@_lq!(9R|7`!muo=%> z>>VPg^$FI}GF*m=!c2y6x3Ri`fhtG73r4tVQ^cA@3Jr%Ywho zE6T`;IfC~+RK3C^6SsHLwJp#?M;bZ_`}Fl^T(n6G4Jz$(OPt+`q`2`5b?rpjn$bMxZW&umCTOK=^v(3n_58U#YBHs?qR?TZ5X{eK1cZiV$emFfkDOnH z+=jdj=|?nEtgxpde)~loljyXMXsP)H3KpolJ-_LvUhtk$u@(_eLQC3dYII~{smG7q z<&hEquusyGs0QF4Hc_|b8H%8v_dhR8r-ssAXe;Zi3xuz4`jfHaTSFiTWJPsYZb8R& z0Ou4Bl3Q1LI(TC$uW3A_cmC*+N!9f6@Olg%R6Ze*iW^1k8;wfc#TI3mhq;=X%HxXkbw0c~HWRLf=xhY$CzDB7xC z0w)uq%I|-K){s#iy6M!)k;`X zpA0qi(N~|TrQKV5@%I#=r!k`^yGB^u^gsLe6*#yedQq`*BiAeRd;W>3uEL1?+L8~} zgj!!sm-rV&>p@GFq!D~y%^KU7%pZr_P>$U5Y^ciTyFhPN*J!W1r>L=z2%I;F_M+pr2C~l8{U-{hTkhEUsovFCu z<=1h+yHo9$FpP>BahC~^MArCKXo|LE4>j*wDDR$BmJY|!W{I!YhN{2Tx#HUG{IhAE z=UI|{7yOD>*`>|Ltcz3}snN)F;jlPjgjY(pvlO~B3)t3`<2zc;jjLwi)y~%1>)f6h ze546`sx>>2>kLmXsB*2%?%wSykHS9fzIzK7ED=2V*VS}@g=!6fqV3$qpMdZ8K-XPq zb7bQzFD|is$&F-lDO5r{{aZ7a)wV&cIgz$`n6V(zp(TFmz+0d|Pc(%nJG&~tOc(Qa zkE9?pzMye3{b5w^aM=g$6nJfIsZsPEMovZ;AaJ-;{?DYxf~g@WvwMor>LZ}g^cb7& zoVdKqjzw`bJhuKwE?^h!!TWJ-&9rEr3w1V*txL>z*jQEVZv6sp(e$|_n(ishO5+s> zhEO0O+evuH_p=5^3nXUvNN1XQ5BaeETEIO?0C8Nx`B9uGS%#Zv%Jn^&aUPZYos{`+ z{=Z_6x8-}l+r)VYTxss_Um}Bc=dx^n@gz61o7bNDr(jz@z#`)k_(2KKZ}uAnPl}LO zTfMygu51i-p%8vF4GSwnDKGUyYXUW$c}!g5t-^InpS-$1Z(=KnwS}Q%5yv%VO_BNO z`umd5k5SbB#YMKt@yuERL%LjInApl)q=JU_*Eq&cukv*Y_*Cc73<&}8_^Gg&sv^b= zY1KC7_u)De8VMOHF(Jui4_{q{7i>6Q!#DPV`19vLnCfLT+^R+gjqek+iR)uADDgX9 zZxvnPKYu|=$}UutmfpCmSedigEFrha|JIO2(XNg&EAmFqsMV1j`@}7 z7mu+;w+!6u+^}5XLZ4k_FQ|7?qMMq!8OIAeP3$%g8(ww4sqQEb@%`kr3z~W;!L?n; zM@7q?<5DkkJAakjf<(FWG+qUIttc8l=2QyD@S%M;Oz9w#_!8H&3hOvN*>}@IUS?F4}1LHCSO52^0>xUWf$Nk!?6iB z%hIhy(y977c6Q`*?m-Q#E_kEidi929$e-QzU)T$P-l<1SoZdOo1-~D@JFf(GUse8j z#$)`eXb;ptdEEDFZGmF9B!Ua)l+0E!gvU9;=?*CiWo&E3dVf1BM<9-~+uhSkfwjBm zvFR(6HF0Yz^|`@_jg}dbXYxu$P{6>f8sWKk#W07l&K#z-Gsz5B-GF|X%*GiPj*0M= zIVl}`qVYTVC)*P5i4+z{-XX@9s}JHiLb~XtEJ94#CzA;0;lejUS*xes-7-doulC)4 zLBa@i6l+MuO5YX!4cqXh542c%X*Li}v~e`j`F^=XT+br$J%}D~_!;@(uTy92cMX}k z`}oP|qEc)wYc9F_IpO%?X_Y+PipQa0khj2-8Y9kMs}oXg??A|kV79GgUK)3lkBRA~ zthUym6Ef3Y?v<$#6D_yvzVLGl-`$px;9pZgBwc|L)st{P&h4dNT|4yls_$nqM@o?k zi~sI$QzbV2o!!m+ud7z-Gm+9BKQ6%A1J+=6NochDTsmj?5iB4+u9Ti`JVLetkSDUa zIn_oitkXP(EWNC>FBxOLEdw%nGc>#(VPMZ}7#Ns!iJJZJ9EBTCmORlNpBFkI@Tp^x z#_N@QOR+KjY8FA@Fk`Ka1f-D*J^{8_{oUzwX?SX*`q4r&MLi*3kS%;jpkc_%%jxdK zYmH@|gOpVfX%PI5o+L6RMtEZ$D-jXve#+`(*8NSHBmD@!*c*8gUkbz-^4cZ5VfyHy zBoDVEISwxtrV+HBzQ2+ZA7p5bJdO(Jc#@W1hC&C zQ#llXzw7wIAUuCPd8{^@U1kOpRRr;dtmVQU9=F43t~-AeMZ z6^~M@l>Kf6&K3qvYHhj;20TVHmzS4Kq{>bPKSq@)(M@BC{|;w>fAd!_aMGEX_|FOe zX=HkLzQy&>jj3(^gUIXoF%Pu6+WJCy^tf1SRjLK#{#o>|c6M~E_XCF@G>Ubc<&Gu- z#gCmZ>9sD!c%^;iqXN{w6gY7qOFUI z^qrK~(^cFmP~|QphDv@reMmh|Dn;`6__&?c<)8Z~OHWTv;o& z^=|A7DeQql&B7YVKmeCK^`0D@Zt`+l31vI{jR~WrueO9R>yCbXsh#TrPmE7XGrU5^ z$$Ed)Y_XBwk7YBsh{Ock!-wydM*&L7JS!|6(Q9sFL&hi<{iDqL{NQplCjXxxo%|~J zR(v23Sy1hJeZih9-k6Xu*}>j&1!*FC1Jeu!V%upk%rx~?0{|2<_&d}$#}k2&cLnh` zL^?#|rJ`b*rG2`B0Fo0sJbJ8~3Oe%}igiva5L?5K;0pc|qN-V3Ts%76n>RAVj;B)% z{`{FM8r4irgGsxju>s&qVP@Sd^9&yC*HyA`q75a16-``&1yCVY>T`MWz)V))<;I<`)TG{x&jC6(R#%4crBc*_4)W z<~UaUlXq383v+*N-0K06bLH3+!aCA$SOP68go;V-Gf$5FMA5W7v1r$n>|Zf6?i)J_ z(Zq7EMZ0Qcx$<@lSXZi&pV`;Hg~PW2AU2D(zCuLqJ&kQjZP%Y2DNpF}F1se-TeeI@ zd}LZd6^>HuB$sT2Ug($~P!bP|d=U}lo(gb-{cHyHeNj(Vs#;rH+aMj%mkz2Mx=Qf5 z&hOtBM6+dnC;dga58k-djVoY3TtSH#`CA_k-}1Rp)!X4m^8iqWPfS$cMx$+9V$`zR zDf=e03MX1zP*_&fFzuMeHmqvLwu)S=yG9=<8XAnWll;>n7o*3Q1t4oBFCR@EARK1s zRmKoj2&`PA9-8!KdlbBj2a_oHR5DksqkHEwmtqo6f%buAW@e_=jIF_pXqTK zoqQ}m(_p8`-QTg>DC}C-9X)K@27Hf;df`TuOXIdG3UgV?pjWt&=rUY*lxYB<^j2@D z?0rDMY_+!S8_K8;(#Z&SPfx?w;P(Ezo`L`u*Mtejc}!Gfc5d#*zb2s+ATue>PvG_z z-L0}41a%)nZH+}mHXbXdjm8}4ihyESM=-~WGi&fyO`g0K$ap) zu2{6X6@|1IY6D!@_c35YQ&yMk4}gZ5fF$qU};fHy^8%Oj*I3<-ABYnv|4$yZ1tc`L*ho z%kfDAAmBEerV4L;yZ182vAU!9p;LFoJp||N< zR2D2NEV*FuNqoGY8f)^i)t`o#@nUESEl`xUj)X=sUDT(Oo|2M~Fs}uS;q%P=_sGbH z)^b*xV+ECRN;61P#_fsRxa%}A7S^&}_?8LOkE;+yB|Bgl4i5ofyA&6P+ED=#!dQpg zGv~LUT6`7az__&1SC0_<@%ONo*xw_Q5rsPCi-yF2`W4P_&NGcRS)@BE8%??+=}1>M zd=i%2v>K4k7wa~- z+O~siT+Gjy+nt34M{v3D=27p=sQVF~YC$MN=&P=_HNp@NrSc!{~gN#etDIir92G^1ItzO7$Q&X(v z&@>I-==^S5o*7rhI3~@Rm+Ph1ZKDn)=Z6K>>lJYH&_q>O#GT*OBsohXM)yqU>P z`>NQj&6bU+;iPiWwq3XEA#&{DTd(lM1 z*!vl5(P|r@zVOnFQr0@HKt0d5OIB4YQ?20jvQ;Q1qaQr?@XiivDybd{82_le!C^27aDG9-IfdfWa=@2h_idX!6we2{sJK`%iPySDEiiWUkG)*R zp%i`f?L{!2Of^HpTy=Q#Ni<}u+&O|B>e_qLJwgS#k=P0j8=I9n6vn~3u$9|5W^KO9 z(v=m0(w5tsh7<$=)!q~3&+)xdXmy{dF`_U-d4>y@7vh{&`!)`cUagYo+|(35msx}g zNQG@oWdSvYQ8RFk01p~uIkjFzlNR^^>8vDmLF=e`8A1E<@^X(Wqdk$gLm3MPxACm{ zR&w@!_diPP0IFlhYD*YR?Vz?VztZl({rs9vI1{PGJGC+*XmVl-A= zlOkHdV6<|~2`_{>iLWk7f2gV{M{btxQv3Py=TWguDDrBwm+6=Xm}DC-XsXczko)A~ z({fimvJfArwIB7dpHe9VoP$F9t0xIND}Aw;p|r!7hg+pBzI4zMS#*MtdppnOy}3hF zpLlvnOG~Rl9!m!jbDMi7dnX;4pCh$$kzRHFvnCmL_gY;fJ3}{>Ed@x!X@J=Y&a8R!8B#srYco9cQMge7Bllw7enac}S?frzzr=x=Je^gsp&zQFFe`);V`Xm+K zPDGr$F#)b{P{+Y0p}b-Q)uoh&fca>+ULduE2ZED$*8) z?alt6)$>MB^cjhcEc%4Bn!I_7;U%PSuFmV|Gc@cZ{%mZ1jCp9SkmS76uX~W9lF4EA zD{8{#O3%)FiF=T8j!z?Uc;3X@HY_=XTL`gYewXvjakw%Paij_mN3P!8gK2dOeT$Fb zgEhw|5OI|B6c)^^qg^Odzpu6F9eq-#xK-s-PV2$c@oi`(s(!*-{me+1Cp|}cR9=*0 z_)DOTGTGNh1faZJR4?|rN80Y@e)eM4+HMkbJzV%X0YjT)JZcuB0cebGV(zB-z~6E# zr$*QZ&&7SLDOR%oh{+1rRo*@^qSa;Hx?_%26LR7gM)o{t0Uu46OFn$ z&GKM6%&ND3w6KPJC`)tf#E8}1G7?ZqFMw*6hJ3Ww=Bp7-t`$h&NWI9lw( zD)tVWS|}=)vI+{yIZySoPGKT4d@&(z5>|cZhw~-|n~%w^MCTi3r^v+vJEd2V{krtE zc0ZrE?)4*YJg7EGd=&!z4jM&}u;%JM!~|K*%D^C`+1dl^m_NNVtUYNhRt#>$EhbO5pd;_~|CO)#Pd;bJi@@dYt(Lt& z1ewzOJIT)_vM|HQ%I!s!O=VRPlp#qae z4Tol&L|2wQw8)W3>`k~)v2*|)Uk}qgUcA@sxQ?gzY6C8`nYc` zAW9vUPMG{p7RHK^q@}&0pg81lXn(PK%6Nr;K^Q67!MIeQKbN`Q=No7%H06S#z410x{@Ld#F$KFKcAwSgdi4uj_ zXRp)t$jGx0GjyTS`@agNaX)%jNZwX|rdrxMd)-s!=U_FecI9W9MB_b1TaDo-19HaB;aDRSIxevvUv>W(x;=jz=2&sTRdn0`hCW)rmbNN|gjhcV=tn_kC?pSc`T=}Z{- z*Mv10*{p*cOE*UThoG&pk}k}ACM?O~xnx!C!|eE9%amlv`&iglGiZ4?za(S!+ZU{k zu2?4au=ZhJ_22CzAFQGqadtfHRCfW1f)x%5G+!Fhi_xS%^?o408`g77=SeQJF zcU~^6_OhYO&&s~&gc4^8Dh|VKihOxoQ%>_gi|vh|+p1=Jj)(4^SHTkXuk9P+SJT{& zyrAhHQW7|dIdI1hlNv8^zNv|ApfmH_`DWNKRFEO{`H^9+iuEEztRKk(8q$UYi`V#p zWlzsUlDyq;D;X%CPnC4k9(FvfB@s<88dC6_?|xW|Jrm4ShiQ~{T=7e)XRm9~JR+@@ zpd81dM5=8Zu5??V4)w0ls)t`>DOFhu@m{{!m;I6j0+sg~iV+#KPpRG2H{fgIP zZuy^k+xc370_S#JIli@D!6zkh?|C-Y|JwbRtjHy#Xv{^~5a^`RPo;%HygbHn%h%NmgCN3d9PXm zR}?$M3ymT151r?2o3^jJuH*j<em=bmj8XNED5D{tce(2db zD9p`mM-`57RNnXnEQ8*X?DHsI^QqcuFNC5Lv#Uas-X`u`hJ=|J-lLI zSrw-$A)EJqAhWhG$Z8t1TK-G&I!Y1%m(EYN@>L}EuP(nWiH%J}3TJFqS$OR;mny36 z%Ku+>7TH1I3U7#IJK}Y@&dGXjT=hf7m`sj0osZQKb6yWQ_l_lV3BTNZigkqNr}D2f zk!&NTNG1IQ+2sR$Ek4=CD{Bzg%RhLri|FKcp~gGKCKG?r!ESQWTdMmP^8z$ED0{PMY zZG#`?${l7eZDh8ck8;)?U)tPV+rV;r(2tWUh?nVqSne{%^&_t6zLQPc=ohM2Erfuo z1#^A@UHGBUWp#UxN`Hhx`CL9c3-^^xT8oi45d~S=2l6gzs-_?AafgB5g^IHoA)0&lTN zmlE?7dIq-xp<4t)kHC}hS5h&K@^}+1R7N&}^d5`sxT-HZRBcl@>*s5g&*hXlQ=`y! zzJ<^W$y9Y_QB+orxmyJzo@GL}#m-wQ&f;EpxQ}rZESS`aIxOedMBCaoQOT%D@!Sj_ zye%Cf75lXuhYnG&6SwPOt;_XCvGeaVc%L>)qkK~)QB!J2mo9oezvXp1q>UM)R~#hU zHBHk|CsC4fkMzy#XP1iG6>~_{iR=c&tnw?nm77qkoSi-*OGnp`AKuM+HH}>QY&q9A zvw8t}?r1!5_3dKufmqg`44p1v>z0^t6QekChw~@hnac6qa|Nfcug)BNSluubY zVf~J*^#0~`?u*mrTwckI8)$6A_asCi?uq=KUCY_X42olthG<4CQqwhmS?(H6LqsXTq1UOF5PyOf882ObYaIK zwu_tTpzS{6+JPE}pEgAHL-w}D_n-0e6EO-pBqO$~9feL)f2r0LZ|_FFKy|{gB;18o z>@=oPh9_0p!k~_8dpOufxHxVW_fccjv-QxM#r9GSX?hQuubN(k?|eDV zpFMafWAv(Kf8G6d)38%MjZGNiIpcIUNmy*4xmXj4Ciu5rPHmgD1t@9AmL}x+U@Su>{^9d~P$tKR#8w;kl z)pc%M&}+(XLaHz8Jm{~#`s8`~`TSbTkC97gE75^?vvPL@TVOo>)hSa7D_XZo$6Mc` znEd9ymq$)x)tEQVH_#9R8#=uC2_1XGm0WG)>vq8z2R}<^|B#n!(U51cT6Pcj>GQnt zWi97QWsC!t2wTp*7L~3+WlAdm=sQWMDGmspV(do~U}g zlqc`F=v`~BvC^v(x4IVWYlc;BR*|@kp9ulBfwcn6x5n?(_O>8r%MojQ<5D$0U9CZ zdebez+f2~d2na1;fcTSC{3Eacv8^BmhDKOD0wVD)ZpYHyuesy?vI&|gQ_+Yy0K{jm zDwF!&FLLj|Gr9D~q`!kS6R+x0`HX{XF8I&*OPk|`NUSXe+WAagif z-g{C;d-%;!cclg}N5J#RKp+0{_Vym2;CIMS{SHG?o@SK6Tm%g$G*Nfis?iy=TD5gilI$H|#YK6V}IEp5K|23W#&YE%S%?jl3K(ER*- zUMT}3js^f82;Q71hU+n*BSb(R(h{j!S}LCe2bc}a#%XbO6cQpt%vi4@h&^4moUbOY zK1+rIDiLN&!*>nJyflEMHeLV4oQ)wbW&Z=@Ak@RM37YKq?7<3Y$@D<3aLnpB27Z42 z=30+XY10XSItWr_ZRKMD=!7z$sSw@sW`uw@LaUIs;5?tem~@Q)o2YnyZ!g!J&8x3^ zU}&E+!|!neDF#LY(~u;K)xN|*2M*y@SAa9%l_DSa0xYbhguD^;hW#?tci_YL#H9}0tHJ+uJNSbvO^{8*?4(A1$3G_rAU z{K~0MV?wEs3d>`HqGOZ2)%YdI&R*(dCHz$$^hEx-&-k1D#f|`wA3xIlzBTqa9?Xo~ zN+k&QHgvOs!ajXaA+Yzru%y$3mqPssm>s$t-)%T3D2QOKUp3+bU>2x^q$G|yc`IK5 z<>krVykFuBvCNIxW@CNI=A9b3utF>EYNAv>S~Tt< zEt=M_oGcM+>!l`|LNviPs36Ra0$vY3ox;jVFK_wsNku}k!Byx&G4-CaUBtC_!T6VZ z4WdeeeaF8Fta^$o1E!+%At6~#$*DBjC2w>fUX1O%b5%9A-8MU36EZN0Xk3VfP zncuLOclVn=YE7EMBUr7GW#lk zv!63r3VwN9s#a>g>@JNUR1|*)U*1?Ks#c8aWIW3t$Q|zgDdFmpR(;4;E*zzpA zqjwxSTL5Y9k|>u@o<=x3C+9!0Cr+n~5WjjBw3) zOc4PmI8I`nIXmQm4&a1tE`hfwQ{>d_>Iy;IE%x}`1!xAvYU6P}q7~&!W7#K=2dQ7& zJFla(f|t#Z0Oz%J_{SlN&G&n~frUW|xmaGl+ZH>|BpewAK1S+i*zniYpT$bq;ZP`G zTT9sab1iIdsvWoJFUbtIB4`c{6thMx8Sko~VWI`hDl;Z*#fsbUE)zV>e)iC_$y9fl z@7Z7u9rXFMgQMfnHsVk{1k!S|06fnB@l1g+j_$tFnh+CF#H*0{hCN3ulI-#M>zC=u z#XuO`eg`+^gF;@N0&VxOG^KW@lCIFArGV{0;MJZ^X|RL*J9+j0HiO_#;i%2DEjXa+RL zqB#i(N($C6d&h{>)g$-d`9{Upr#{SUYNIR>vNUKoumc%kdfce^6}P8lSLMn*SGt!! zwx3C(H4kYFl#nsNTLkD6u~T!qLbW@(x(ojTK0NYykA_ymU|MS9ZI@4`K*;$)T48cB z;S)4ut|G8Ex~hGz`$G58@^#pwAH0w_Vcf@8T@N&~-5#duJ-)41ng~l_2%ISG4H&eI zm4$ttjCEBx86;(DnBQX#65F7i%~Qq~Yna!bZ7Gi7H8qUN5ylPbWQQ#+HGIlbrHMXG zrnU7HC+HJls{5+O;x`~S4ba>bYei;>-k!3{>+G~;-?GhItfIG}3`8L|4|;K)%M9F&%DubZ{*_k)GmtJs_6%H=f7 zdQ<81LHKOd)zXnpW?Ps^{^J$Gag?4_MtWw3cFtWT+5X}RrN>Q3;lu@?$WK`B$i|GapfF|1WvewBb7ypUMTRonc4o%TN~4FT2$TVxPPEn&+7t#4M* z#ZQOKNz1B42456?cn9%xbZSd5 z!==pmeqY$=W73KEHc?)_32zl9n3jNkGDVD&8MA2biG;$4jU<~)-LCL8arU3rqt>96nd(GDrh;kI)HzB^N7QK#RsHfj~Uf!PfVs=-Dha z|6&RAI#-+26y}W`VpO`E@NBYkHiLn^8CGq5*GFp-6MN<{azGt<@Y#Qc0RU{Q1&@s* zmt-ndwm03DHM*S^_p&OB{JECZwqH*-uA}zYj%cNyuzeGB0h(DTZ1j@V^@ zUa)EvYa4A;Ru$&#CNVVd^(WxSG_%nUhnQBcs+Gc$xvqu_E4*yjgX_M~=?c(IiF_7S zNm3G|#nq1J#Ytz6n}Q0#Cgw|KYuUT37H6sCDSVp}?z&GOdIx|*`5kzMT7Q<5TQeCD zkTLCVcb0i@dFq->uKy?>xD)Jc`8k)V(2!bc$d@VF zm)|7`@0495tV-764Mb5g{L3|^14P+{7_=>h)VW$W&qu!Cfm?ai#m)@zu|Paxvxn8# z=R|K5L&2_8@7gu&!bZUkYnh--v{-<>CmQawa_v~=`ZNt@{52ih~7AvZ-D z^Rxc!{KThF(^0@=2flatd`HgZN~Ad?4Z3uQc8toj-oxqu-*$$W``Kh&3k}wgs*J*q z^Q-M!v>PG|o{S4)hbA@I9=5R5@e;Wg{)J!*&{D(kNq?J0IlCKN(ank#ie*i_T4~<3 zy&L#Ui$dq1WA2Bjc(u2)T}e1T+EY4pEpv(Oa_%@ZxBIELo1u2xu!Cljd)nua%c9** z%fitZaNll9_HA5&;}Y&~=GT3_?K zxE{lSl2ICr&MO!VkA4JEg++8b6sSN3eW1j<>yusXihOj=zzP^DKm9pn3XiaRe(GF4G+N4YW_+uYF;UQY* z+)RlV6RCYWU%E)Jv|{M8vA105u;*|S@LibbSf7e;!v$zU5P$i>Z0{M=G=$8ufA`%* zPo6lOo66N)=L!&|K2&0BFALfdEB(5S5MBzL6~msQNFEd6JXcFCoGe+J@CQK&jyXWV zPQi8(kEkaP&EIOOgSBk2x^PIpy3P|(BKZRU`8H2^Ycz9!hgTIV!@G=IqjsXw_7@BC z4Z)+XJF9oExtRVN%0KJ(|8s9cD%$~X9l6T?9Z3WK{?Eey&&z+ea_7&>zZ?-{EB{;t p|GSlcn*86r1b+9=9rAhNrC7B?8V9Q!ksq=JIVq(VB@)K({||;Vr4axC literal 0 HcmV?d00001 diff --git a/Doxyfile b/Doxyfile new file mode 100644 index 0000000000..f952be9e1c --- /dev/null +++ b/Doxyfile @@ -0,0 +1,1252 @@ +# Doxyfile 1.4.7 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = proxygen + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +#### OPTION GENERATED IN generate_doxygen.sh + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, +# Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, +# Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, +# Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, +# Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# This tag can be used to specify the encoding used in the generated output. +# The encoding is not always determined by the language that is chosen, +# but also whether or not the output is meant for Windows or non-Windows users. +# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES +# forces the Windows encoding (this is the default for the Windows binary), +# whereas setting the tag to NO uses a Unix-style encoding (the default for +# all platforms other than Windows). + +USE_WINDOWS_ENCODING = NO + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like the Qt-style comments (thus requiring an +# explicit @brief command for a brief description. + +JAVADOC_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 2 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to +# include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = YES + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +#### OPTION GENERATED IN generate_doxygen.sh + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +#### OPTION GENERATED IN generate_doxygen.sh + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = YES + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = YES + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentstion. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = YES + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = YES + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +#### OPTION GENERATED IN generate_doxygen.sh + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = YES + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will +# generate a caller dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. + +CALLER_GRAPH = YES + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_WIDTH = 1024 + +# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height +# (in pixels) of the graphs generated by dot. If a graph becomes larger than +# this value, doxygen will try to truncate the graph, so that it fits within +# the specified constraint. Beware that most browsers cannot cope with very +# large images. + +MAX_DOT_GRAPH_HEIGHT = 1024 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that a graph may be further truncated if the graph's +# image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH +# and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), +# the graph is not depth-constrained. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..337758d78a --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Proxygen software + +Copyright (c) 2014, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/PATENTS b/PATENTS new file mode 100644 index 0000000000..c24e33be1b --- /dev/null +++ b/PATENTS @@ -0,0 +1,23 @@ +Additional Grant of Patent Rights + +"Software" means the Proxygen software distributed by Facebook, Inc. + +Facebook hereby grants you a perpetual, worldwide, royalty-free, non-exclusive, +irrevocable (subject to the termination provision below) license under any +rights in any patent claims owned by Facebook, to make, have made, use, sell, +offer to sell, import, and otherwise transfer the Software. For avoidance of +doubt, no license is granted under Facebook’s rights in any patent claims that +are infringed by (i) modifications to the Software made by you or a third party, +or (ii) the Software in combination with any software or other technology +provided by you or a third party. + +The license granted hereunder will terminate, automatically and without notice, +for anyone that makes any claim (including by filing any lawsuit, assertion or +other action) alleging (a) direct, indirect, or contributory infringement or +inducement to infringe any patent: (i) by Facebook or any of its subsidiaries or +affiliates, whether or not such claim is related to the Software, (ii) by any +party if such claim arises in whole or in part from any software, product or +service of Facebook or any of its subsidiaries or affiliates, whether or not +such claim is related to the Software, or (iii) by any party relating to the +Software; or (b) that any right in any patent claim of Facebook is invalid or +unenforceable. diff --git a/README.md b/README.md new file mode 100644 index 0000000000..ec088e226a --- /dev/null +++ b/README.md @@ -0,0 +1,141 @@ +## Proxygen: Facebook's C++ HTTP Libraries + +This project comprises the core C++ HTTP abstractions used at +Facebook. Internally, it is used as the basis for building many HTTP +servers, proxies, and clients. This release focuses on the common HTTP +abstractions and our simple HTTPServer framework. Future releases will +provide simple client APIs as well. The framework supports HTTP/1.1, +SPDY/3, and SPDY/3.1. HTTP/2 support is in progress. The goal is to +provide a simple, performant, and modern C++ HTTP library. + +We have a Google group for general discussions at https://groups.google.com/d/forum/facebook-proxygen. + +Build Status: [![Build Status](https://travis-ci.org/facebook/proxygen.png?branch=master)](https://travis-ci.org/facebook/proxygen) + +### Installing + +Note that currently this project has only been tested on Ubuntu 14.04, +although it likely works on many other platforms. Support for Mac OSX is +incomplete. + +You will need at least 2 GiB of memory to compile proxygen and its +dependencies. + +##### Easy Install + +Just run `./deps.sh` from the `proxygen/` directory to get all the +dependencies and build proxygen. This will also install folly, fbthrift, +and proxygen on your machine. + +A note on compatibility: this project relies on system installed fbthrift +and folly. If you rebase proxygen and `make` starts to fail, you likely +need to update to the latest version of fbthrift and folly. First try to +uninstall, rebase, and reinstall fbthrift and folly. We are still working +on a solution to make upgrading less painful. + +For now, you can rerun `./deps.sh` after fetching and rebasing +proxygen. This will update the installed fbthrift and folly libraries to +the correct (latest) version. + +##### Other Platforms + +If you are running on another platform, you may need to install several +packages first. Proxygen, fbthrift, and folly are all autotools based projects. + +### Introduction + +Directory structure and contents: + +* `proxygen/external/` Contains non-installed 3rd-party code proxygen depends on. +* `proxygen/lib/` Core networking abstractions. +* `proxygen/lib/http/` HTTP specific code. +* `proxygen/lib/services/` Connection management and server code. +* `proxygen/lib/ssl/` TLS abstractions and OpenSSL wrappers. +* `proxygen/lib/utils/` Miscellaneous helper code. +* `proxygen/httpserver/` Contains code wrapping `proxygen/lib/` for building simple C++ http servers. We recommend building on top of these APIs. + +### Architecture + +The central abstractions to understand in `proxygen/lib` are the session, codec, +transaction, and handler. These are the lowest level abstractions, and we +don't generally recommend building off of these directly. + +When bytes are read off the wire, the `HTTPCodec` stored inside +`HTTPSession` parses these into higher level objects and associates with +it a transaction identifier. The codec then calls into `HTTPSession` which +is responsible for maintaining the mapping between transaction identifier +and `HTTPTransaction` objects. Each HTTP request/response pair has a +separate `HTTPTransaction` object. Finally, `HTTPTransaction` forwards the +call to a handler object which implements `HTTPTransation::Handler`. The +handler is responsible for implementing business logic for the request or +response. + +The handler then calls back into the transaction to generate egress +(whether the egress is a request or response). The call flows from the +transaction back to the session, which uses the codec to convert the +higher level semantics of the particular call into the appropriate bytes +to send on the wire. + +The same handler and transaction interfaces are used to both create requests +and handle responses. The API is generic enough to allow +both. `HTTPSession` is specialized slightly differently depending on +whether you are using the connection to issue or respond to HTTP +requests. + +![Core Proxygen Architecture](CoreProxygenArchitecture.png) + +Moving into higher levels of abstraction, `proxygen/httpserver` has a +simpler set of APIs and is the recommended way to interface with proxygen +when acting as a server if you don't need the full control of the lower +level abstractions. + +The basic components here are `HTTPServer`, `RequestHandlerFactory`, and +`RequestHandler`. An `HTTPServer` takes some configuration and is given a +`RequestHandlerFactory`. Once the server is started, the installed +`RequestHandlerFactory` spawns a `RequestHandler` for each HTTP +request. `RequestHandler` is a simple interface users of the library +implement. Subclasses of `RequestHandler` should use the inherited +protected member `ResponseHandler* downstream_` to send the response. + +### Using it + +Proxygen is a library. After installing it, you can build your own C++ +server. Try `cd`ing to the directory containing the echo server at +`proxygen/httpserver/samples/echo/`. You can then build it with this one +liner: + + +g++ -std=c++11 -o my_echo EchoServer.cpp EchoHandler.cpp -lproxygenhttpserver -lfolly -lglog -lgflags -pthread + + +After running `./my_echo`, we can verify it works using curl in a different terminal: +```shell +$ curl -v http://localhost:11000/ +* Trying 127.0.0.1... +* Connected to localhost (127.0.0.1) port 11000 (#0) +> GET / HTTP/1.1 +> User-Agent: curl/7.35.0 +> Host: localhost:11000 +> Accept: */* +> +< HTTP/1.1 200 OK +< Request-Number: 1 +< Date: Thu, 30 Oct 2014 17:07:36 GMT +< Connection: keep-alive +< Content-Length: 0 +< +* Connection #0 to host localhost left intact +``` + +### Documentation + +We use Doxygen for Proxygen's internal documentation. You can generate a +copy of these docs by running `doxygen Doxyfile` from the project +root. You'll want to look at `html/namespaceproxygen.html` to start. This +will also generate folly and thrift documentation. + +### Whitehat + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for +the safe disclosure of security bugs. If you find a vulnerability, please +go through the process outlined on that page and do not file a public issue. diff --git a/proxygen/Makefile.am b/proxygen/Makefile.am new file mode 100644 index 0000000000..52c08368a2 --- /dev/null +++ b/proxygen/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = lib httpserver diff --git a/proxygen/VERSION b/proxygen/VERSION new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/proxygen/VERSION @@ -0,0 +1 @@ +1 diff --git a/proxygen/configure.ac b/proxygen/configure.ac new file mode 100644 index 0000000000..26970bea96 --- /dev/null +++ b/proxygen/configure.ac @@ -0,0 +1,264 @@ +# Process this file with autoconf to produce a configure script. + +AC_PREREQ(2.65) + +m4_define([proxygen_version_str], m4_esyscmd_s([cat VERSION])) + +AC_INIT([proxygen], m4_translit(proxygen_version_str, [:], [.])) + +LT_VERSION=proxygen_version_str:0 +AC_SUBST([LT_VERSION]) + +AC_CONFIG_SRCDIR([lib/http/Window.h]) +AC_CONFIG_HEADERS([config.h]) +AX_PREFIX_CONFIG_H([proxygen-config.h], [proxygen], [config.h]) +AC_CONFIG_AUX_DIR([build-aux]) + +AM_INIT_AUTOMAKE([1.9 foreign tar-ustar nostdinc subdir-objects]) +AC_CONFIG_MACRO_DIR([m4]) + +PKG_PROG_PKG_CONFIG + +AC_PROG_INSTALL +AC_PROG_LIBTOOL + +AC_LANG([C++]) + +# Checks for programs. +AC_PROG_CXX +AC_PROG_CC +AC_PROG_CPP +AC_CXX_COMPILE_STDCXX_0X +AC_PROG_RANLIB + +# Be sure to add any -std option to CXXFLAGS before we invoke any +# AC_COMPILE_IFELSE() or similar macros. Any such macros that are invoked +# before we update CXXFLAGS will not be run with the same options that we use +# during the real build. +STD="" +if test "x$ac_cv_cxx_compile_cxx0x_cxx" = xyes; then + STD="-std=c++0x" +fi +if test "x$ac_cv_cxx_compile_cxx0x_gxx" = xyes; then + STD="-std=gnu++0x" +fi + +CXXFLAGS="$STD $CXXFLAGS" +CXXFLAGS="-fno-strict-aliasing $CXXFLAGS" +CXXFLAGS="-O3 -g -W -Wall -Wextra -Wno-unused-parameter $CXXFLAGS" +CXXFLAGS=" -Wno-missing-field-initializers -Wno-deprecated $CXXFLAGS" +CXXFLAGS="-DLIBMC_FBTRACE_DISABLE $CXXFLAGS" + +CFLAGS="-DLIBMC_FBTRACE_DISABLE $CFLAGS" + +# Checks for libraries. +AC_CHECK_LIB([glog],[openlog],[],[AC_MSG_ERROR( + [Please install google-glog library])]) +AC_CHECK_LIB([gflags],[getenv],[],AC_MSG_ERROR( + [Please install google-gflags library])) + +# check for boost libs +AX_BOOST_BASE([1.51.0], [], [AC_MSG_ERROR( + [Please install boost >= 1.51.0])]) + +# Checks for header files. +AC_HEADER_STDC +AC_CHECK_HEADERS([arpa/inet.h fcntl.h limits.h netdb.h stdint.h string.h sys/file.h sys/time.h syslog.h unistd.h malloc.h]) + +AC_CHECK_LIB([event], [event_set], [], [AC_MSG_ERROR([Unable to find libevent])]) +AC_CHECK_LIB([cap], [cap_get_proc], [], [AC_MSG_ERROR([Unable to find libcap])]) +AC_CHECK_LIB([crypto], [MD5_Init], [], [AC_MSG_ERROR([Unable to find libcrypto])]) +AC_CHECK_LIB([ssl], [SSL_library_init], [], [AC_MSG_ERROR([Unable to find libssl])]) +AC_CHECK_LIB([z], [gzread], [], [AC_MSG_ERROR([Unable to find zlib])]) +AC_CHECK_LIB([folly],[getenv],[],[AC_MSG_ERROR( + [Please install the folly library from https://github.com/facebook/folly])]) +AC_CHECK_HEADER([folly/Likely.h], [], [AC_MSG_ERROR( +[Couldn't find folly, please download from https://github.com/facebook/folly] +)], []) + +AC_CHECK_LIB([thriftcpp2], [main], [], [AC_MSG_ERROR( +[Unable to find thriftcpp2, you need to install FBThrift: https://github.com/facebook/fbthrift])]) +AC_CHECK_LIB([thrift], [main], [], [AC_MSG_ERROR( +[Unable to find thrift, you need to install FBThrift: https://github.com/facebook/fbthrift])]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_HEADER_STDBOOL +AC_C_CONST +AC_C_INLINE +AC_TYPE_SIZE_T +AC_HEADER_TIME +AC_C_VOLATILE +AC_FUNC_ERROR_AT_LINE +AC_FUNC_FORK +AC_TYPE_INT32_T +AC_TYPE_INT64_T +AC_TYPE_MODE_T +AC_TYPE_PID_T +AC_TYPE_SSIZE_T +AC_TYPE_UINT16_T +AC_TYPE_UINT32_T +AC_TYPE_UINT64_T + +AC_CHECK_TYPE([__int128], + [AC_DEFINE([HAVE_INT128_T], [1], [Define if __int128 exists])], + [AC_DEFINE([HAVE_INT128_T], [0], [Define if __int128 does not exist])]) +AC_CHECK_TYPES([ptrdiff_t]) +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE[ + #pragma GCC diagnostic error "-Wattributes" + extern "C" void (*test_ifunc(void))() { return 0; } + void func() __attribute__((ifunc("test_ifunc")));] + ], + [AC_DEFINE([HAVE_IFUNC], [1], [Define to 1 if the compiler supports ifunc])], + [AC_DEFINE([HAVE_IFUNC], [0], [Define to 0 if the compiler doesn't support ifunc])] +) +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE[class C { virtual void f() final {} virtual void g() {} }; + class D : public C { virtual void g() override {} };]], + [AC_DEFINE([FINAL], [final], + [Define to "final" if the compiler supports C++11 "final"]) + AC_DEFINE([OVERRIDE], [override], + [Define to "override" if the compiler supports C++11 "override"])], + [AC_DEFINE([FINAL], [], + [Define to "final" if the compiler supports C++11 "final"]) + AC_DEFINE([OVERRIDE], [], + [Define to "override" if the compiler supports C++11 "override"])] +) + +AC_MSG_CHECKING([for static_assert]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([static_assert (1, "");], + [(void) 0])], + [AC_DEFINE([HAVE_STATIC_ASSERT], [1], + [Whether static_assert can be used or not]) + AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no])]) + +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE[ + #include + #include + void func() { std::this_thread::sleep_for(std::chrono::seconds(1)); }]], + [AC_DEFINE([HAVE_STD__THIS_THREAD__SLEEP_FOR], [1], + [Define to 1 if std::this_thread::sleep_for() is defined.])]) + +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE[ + #include + static constexpr int val = strlen("foo");]], + [AC_DEFINE([HAVE_CONSTEXPR_STRLEN], [1], + [Define to 1 if strlen(3) is constexpr.])]) + +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE[ + #include + #if !_LIBCPP_VERSION + #error No libc++ + #endif + void func() {}] + ], + [AC_DEFINE([USE_LIBCPP], [1], [Define to 1 if we're using libc++.])]) + +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE[ + #include + const bool val = std::is_trivially_copyable::value;] + ], + [AC_DEFINE([HAVE_STD__IS_TRIVIALLY_COPYABLE], [1], + [Define to 1 if we have a usable std::is_trivially_copyable + implementation.])]) + +# Figure out if we support weak symbols. If not, we will link in some null +# stubs for functions that would otherwise be weak. +AC_LINK_IFELSE( + [AC_LANG_SOURCE[ + extern "C" void configure_link_extern_weak_test() __attribute__((weak)); + int main(int argc, char** argv) { + return configure_link_extern_weak_test == nullptr; + }] + ], + [ + ac_have_weak_symbols="yes" + AC_DEFINE([HAVE_WEAK_SYMBOLS], [1], + [Define to 1 if the linker supports weak symbols.])]) + +AC_SEARCH_LIBS([cplus_demangle_v3_callback], [iberty]) +if test "$ac_cv_search_cplus_demangle_v3_callback" != "no" ; then + AC_DEFINE([HAVE_CPLUS_DEMANGLE_V3_CALLBACK], [1], + [Define to 1 if we have cplus_demangle_v3_callback.]) +fi + +# Check for clock_gettime(2). This is not in an AC_CHECK_FUNCS() because we +# want to link with librt if necessary. +AC_SEARCH_LIBS([clock_gettime], [rt], + AC_DEFINE( + [HAVE_CLOCK_GETTIME], + [1], + [Define to 1 if we support clock_gettime(2).]), + []) + +# Checks for library functions. +AC_CHECK_FUNCS([gettimeofday \ + localtime_r \ + memchr \ + memset \ + mkdir \ + socket \ + strcasecmp \ + strdup \ + strerror \ + strtol \ + dup2 \ + ftruncate]) + +# Check for python, ruby, and gperf (needed for auto generated code) +AC_CHECK_PROG(HAVE_PYTHON, python, yes) +if test x"$HAVE_PYTHON" != x"yes"; then + AC_MSG_ERROR([Please install python first.]) +fi + +AC_CHECK_PROG(HAVE_RUBY, ruby, yes) +if test x"$HAVE_RUBY" != x"yes"; then + AC_MSG_ERROR([Please install ruby first.]) +fi + +AC_CHECK_PROG(HAVE_GPERF, gperf, yes) +if test x"$HAVE_GPERF" != x"yes"; then + AC_MSG_ERROR([Please install gperf first.]) +fi + +LIBS="$LIBS $BOOST_LDFLAGS -lpthread -pthread -lfolly -lglog -lthrift" +LIBS="$LIBS -ldouble-conversion -lboost_system -lboost_thread" + +AM_CONDITIONAL([HAVE_STD_THREAD], [test "$ac_cv_header_features" = "yes"]) +AM_CONDITIONAL([HAVE_X86_64], [test "$build_cpu" = "x86_64"]) +AM_CONDITIONAL([HAVE_LINUX], [test "$build_os" == "linux-gnu"]) +AM_CONDITIONAL([HAVE_WEAK_SYMBOLS], [test "$ac_have_weak_symbols" = "yes"]) +AM_CONDITIONAL([HAVE_BITS_FUNCTEXCEPT], [test "$ac_cv_header_bits_functexcept" = "yes"]) + +# Include directory that contains "proxygen" so #include "proxygen/Foo.h" works +# Also add includes for gmock and gtest +AM_CPPFLAGS='-I$(top_srcdir)/.. -I$(top_srcdir)/lib/test/gmock-1.6.0/include -I$(top_srcdir)/lib/test/gmock-1.6.0/gtest/include' +AM_CPPFLAGS="$AM_CPPFLAGS $CXX_FLAGS $BOOST_CPPFLAGS" +AC_SUBST([AM_CPPFLAGS]) + +# Output +AC_CONFIG_FILES([Makefile + lib/Makefile + lib/test/Makefile + lib/utils/Makefile + lib/utils/test/Makefile + lib/ssl/Makefile + lib/ssl/test/Makefile + lib/services/Makefile + lib/http/Makefile + lib/http/codec/Makefile + lib/http/codec/test/Makefile + lib/http/session/Makefile + lib/http/session/test/Makefile + lib/http/test/Makefile + httpserver/Makefile + httpserver/samples/Makefile + httpserver/samples/echo/Makefile + httpserver/tests/Makefile]) + +AC_OUTPUT diff --git a/proxygen/deps.sh b/proxygen/deps.sh new file mode 100755 index 0000000000..96ed859199 --- /dev/null +++ b/proxygen/deps.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -e +start_dir=`pwd` +trap "cd $start_dir" EXIT + +# Must execute from the directory containing this script +cd "$(dirname "$0")" + +# Some extra dependencies for Ubuntu 13.10 and 14.04 +sudo apt-get install \ + flex \ + bison \ + libkrb5-dev \ + libsasl2-dev \ + libnuma-dev \ + pkg-config \ + libssl-dev \ + libcap-dev \ + ruby \ + gperf \ + autoconf-archive \ + libevent-dev \ + libgoogle-glog-dev + +git clone https://github.com/facebook/fbthrift || true +cd fbthrift/thrift + +# Rebase and uninstall in case we've already downloaded thrift and folly +git fetch && git rebase origin/master +sudo make uninstall || true +if [ -e folly/folly ]; then + # We have folly already downloaded + cd folly/folly + git fetch && git rebase origin/master + sudo make uninstall || true + cd ../.. +fi + +# Build folly and fbthrift +./deps.sh + +# Install folly +cd folly/folly +sudo make install + +# Install fbthrift +cd ../.. +sudo make install + +# Build proxygen +sudo /sbin/ldconfig +cd ../.. +autoreconf -ivf +./configure +make -j8 + +# Run tests +make check + +# Install +sudo make install + +sudo /sbin/ldconfig diff --git a/proxygen/external/http_parser/CONTRIBUTIONS b/proxygen/external/http_parser/CONTRIBUTIONS new file mode 100644 index 0000000000..11ba31e4b9 --- /dev/null +++ b/proxygen/external/http_parser/CONTRIBUTIONS @@ -0,0 +1,4 @@ +Contributors must agree to the Contributor License Agreement before patches +can be accepted. + +http://spreadsheets2.google.com/viewform?hl=en&formkey=dDJXOGUwbzlYaWM4cHN1MERwQS1CSnc6MQ diff --git a/proxygen/external/http_parser/LICENSE-MIT b/proxygen/external/http_parser/LICENSE-MIT new file mode 100644 index 0000000000..58010b3889 --- /dev/null +++ b/proxygen/external/http_parser/LICENSE-MIT @@ -0,0 +1,23 @@ +http_parser.c is based on src/http/ngx_http_parse.c from NGINX copyright +Igor Sysoev. + +Additional changes are licensed under the same terms as NGINX and +copyright Joyent, Inc. and other Node contributors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/proxygen/external/http_parser/README.md b/proxygen/external/http_parser/README.md new file mode 100644 index 0000000000..405dd5f342 --- /dev/null +++ b/proxygen/external/http_parser/README.md @@ -0,0 +1,171 @@ +HTTP Parser +=========== + +This is a parser for HTTP messages written in C. It parses both requests and +responses. The parser is designed to be used in performance HTTP +applications. It does not make any syscalls nor allocations, it does not +buffer data, it can be interrupted at anytime. Depending on your +architecture, it only requires about 40 bytes of data per message +stream (in a web server that is per connection). + +Features: + + * No dependencies + * Handles persistent streams (keep-alive). + * Decodes chunked encoding. + * Upgrade support + * Defends against buffer overflow attacks. + +The parser extracts the following information from HTTP messages: + + * Header fields and values + * Content-Length + * Request method + * Response status code + * Transfer-Encoding + * HTTP version + * Request URL + * Message body + + +Usage +----- + +One `http_parser` object is used per TCP connection. Initialize the struct +using `http_parser_init()` and set the callbacks. That might look something +like this for a request parser: + + http_parser_settings settings; + settings.on_path = my_path_callback; + settings.on_header_field = my_header_field_callback; + /* ... */ + + http_parser *parser = malloc(sizeof(http_parser)); + http_parser_init(parser, HTTP_REQUEST); + parser->data = my_socket; + +When data is received on the socket execute the parser and check for errors. + + size_t len = 80*1024, nparsed; + char buf[len]; + ssize_t recved; + + recved = recv(fd, buf, len, 0); + + if (recved < 0) { + /* Handle error. */ + } + + /* Start up / continue the parser. + * Note we pass recved==0 to signal that EOF has been recieved. + */ + nparsed = http_parser_execute(parser, &settings, buf, recved); + + if (parser->upgrade) { + /* handle new protocol */ + } else if (nparsed != recved) { + /* Handle error. Usually just close the connection. */ + } + +HTTP needs to know where the end of the stream is. For example, sometimes +servers send responses without Content-Length and expect the client to +consume input (for the body) until EOF. To tell http_parser about EOF, give +`0` as the forth parameter to `http_parser_execute()`. Callbacks and errors +can still be encountered during an EOF, so one must still be prepared +to receive them. + +Scalar valued message information such as `status_code`, `method`, and the +HTTP version are stored in the parser structure. This data is only +temporally stored in `http_parser` and gets reset on each new message. If +this information is needed later, copy it out of the structure during the +`headers_complete` callback. + +The parser decodes the transfer-encoding for both requests and responses +transparently. That is, a chunked encoding is decoded before being sent to +the on_body callback. + + +The Special Problem of Upgrade +------------------------------ + +HTTP supports upgrading the connection to a different protocol. An +increasingly common example of this is the Web Socket protocol which sends +a request like + + GET /demo HTTP/1.1 + Upgrade: WebSocket + Connection: Upgrade + Host: example.com + Origin: http://example.com + WebSocket-Protocol: sample + +followed by non-HTTP data. + +(See http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 for more +information the Web Socket protocol.) + +To support this, the parser will treat this as a normal HTTP message without a +body. Issuing both on_headers_complete and on_message_complete callbacks. However +http_parser_execute() will stop parsing at the end of the headers and return. + +The user is expected to check if `parser->upgrade` has been set to 1 after +`http_parser_execute()` returns. Non-HTTP data begins at the buffer supplied +offset by the return value of `http_parser_execute()`. + + +Callbacks +--------- + +During the `http_parser_execute()` call, the callbacks set in +`http_parser_settings` will be executed. The parser maintains state and +never looks behind, so buffering the data is not necessary. If you need to +save certain data for later usage, you can do that from the callbacks. + +There are two types of callbacks: + +* notification `typedef int (*http_cb) (http_parser*);` + Callbacks: on_message_begin, on_headers_complete, on_message_complete. +* data `typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);` + Callbacks: (requests only) on_uri, + (common) on_header_field, on_header_value, on_body; + +Callbacks must return 0 on success. Returning a non-zero value indicates +error to the parser, making it exit immediately. + +In case you parse HTTP message in chunks (i.e. `read()` request line +from socket, parse, read half headers, parse, etc) your data callbacks +may be called more than once. Http-parser guarantees that data pointer is only +valid for the lifetime of callback. You can also `read()` into a heap allocated +buffer to avoid copying memory around if this fits your application. + +Reading headers may be a tricky task if you read/parse headers partially. +Basically, you need to remember whether last header callback was field or value +and apply following logic: + + (on_header_field and on_header_value shortened to on_h_*) + ------------------------ ------------ -------------------------------------------- + | State (prev. callback) | Callback | Description/action | + ------------------------ ------------ -------------------------------------------- + | nothing (first call) | on_h_field | Allocate new buffer and copy callback data | + | | | into it | + ------------------------ ------------ -------------------------------------------- + | value | on_h_field | New header started. | + | | | Copy current name,value buffers to headers | + | | | list and allocate new buffer for new name | + ------------------------ ------------ -------------------------------------------- + | field | on_h_field | Previous name continues. Reallocate name | + | | | buffer and append callback data to it | + ------------------------ ------------ -------------------------------------------- + | field | on_h_value | Value for current header started. Allocate | + | | | new buffer and copy callback data to it | + ------------------------ ------------ -------------------------------------------- + | value | on_h_value | Value continues. Reallocate value buffer | + | | | and append callback data to it | + ------------------------ ------------ -------------------------------------------- + + +See examples of reading in headers: + +* [partial example](http://gist.github.com/155877) in C +* [from http-parser tests](http://github.com/ry/http-parser/blob/37a0ff8928fb0d83cec0d0d8909c5a4abcd221af/test.c#L403) in C +* [from Node library](http://github.com/ry/node/blob/842eaf446d2fdcb33b296c67c911c32a0dabc747/src/http.js#L284) in Javascript diff --git a/proxygen/external/http_parser/http_parser.c b/proxygen/external/http_parser/http_parser.c new file mode 120000 index 0000000000..91c5c18b0f --- /dev/null +++ b/proxygen/external/http_parser/http_parser.c @@ -0,0 +1 @@ +http_parser_cpp.cpp \ No newline at end of file diff --git a/proxygen/external/http_parser/http_parser.h b/proxygen/external/http_parser/http_parser.h new file mode 100644 index 0000000000..50c75b9de6 --- /dev/null +++ b/proxygen/external/http_parser/http_parser.h @@ -0,0 +1,319 @@ +/* Copyright Joyent, Inc. and other Node contributors. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#ifndef http_parser_h +#define http_parser_h + +#define HTTP_PARSER_VERSION_MAJOR 1 +#define HTTP_PARSER_VERSION_MINOR 0 + +#include +#if defined(_WIN32) && !defined(__MINGW32__) +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; + +typedef unsigned int size_t; +typedef int ssize_t; +#else +#include +#endif + +#if __cplusplus +namespace proxygen { +#endif /* __cplusplus */ + +/* Compile with -DHTTP_PARSER_STRICT=1 to parse URLs and hostnames + * strictly according to the RFCs + */ +#ifndef HTTP_PARSER_STRICT +# define HTTP_PARSER_STRICT 0 +#endif + +/* Compile with -DHTTP_PARSER_DEBUG=1 to add extra debugging information to + * the error reporting facility. + */ +#ifndef HTTP_PARSER_DEBUG +# define HTTP_PARSER_DEBUG 0 +#endif + + +/* Maximium header size allowed */ +#define HTTP_MAX_HEADER_SIZE (80*1024) + + +typedef struct http_parser http_parser; +typedef struct http_parser_settings http_parser_settings; +typedef struct http_parser_result http_parser_result; + + +/* Callbacks should return non-zero to indicate an error. The parser will + * then halt execution. + * + * The one exception is on_headers_complete. In a HTTP_RESPONSE parser + * returning '1' from on_headers_complete will tell the parser that it + * should not expect a body. This is used when receiving a response to a + * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding: + * chunked' headers that indicate the presence of a body. + * + * http_data_cb does not return data chunks. It will be call arbitrarally + * many times for each string. E.G. you might get 10 callbacks for "on_path" + * each providing just a few characters more data. + */ +typedef int (*http_data_cb) (http_parser*, const char *at, size_t length); +typedef int (*http_cb) (http_parser*); + + +/* Request Methods */ +enum http_method + { HTTP_DELETE = 0 + , HTTP_GET + , HTTP_HEAD + , HTTP_POST + , HTTP_PUT + /* pathological */ + , HTTP_CONNECT + , HTTP_OPTIONS + , HTTP_TRACE + /* webdav */ + , HTTP_COPY + , HTTP_LOCK + , HTTP_MKCOL + , HTTP_MOVE + , HTTP_PROPFIND + , HTTP_PROPPATCH + , HTTP_UNLOCK + /* subversion */ + , HTTP_REPORT + , HTTP_MKACTIVITY + , HTTP_CHECKOUT + , HTTP_MERGE + /* upnp */ + , HTTP_MSEARCH + , HTTP_NOTIFY + , HTTP_SUBSCRIBE + , HTTP_UNSUBSCRIBE + /* RFC-5789 */ + , HTTP_PATCH + }; + + +enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH }; + + +/* Flag values for http_parser.flags field */ +enum flags + { F_CHUNKED = 1 << 0 + , F_TRAILING = 1 << 3 + , F_UPGRADE = 1 << 4 + , F_SKIPBODY = 1 << 5 + }; + + +/* Map for errno-related constants + * + * The provided argument should be a macro that takes 2 arguments. + */ +#define HTTP_ERRNO_MAP(XX) \ + /* No error */ \ + XX(OK, "success") \ + \ + /* Callback-related errors */ \ + XX(CB_message_begin, "the on_message_begin callback failed") \ + XX(CB_path, "the on_path callback failed") \ + XX(CB_query_string, "the on_query_string callback failed") \ + XX(CB_url, "the on_url callback failed") \ + XX(CB_fragment, "the on_fragment callback failed") \ + XX(CB_header_field, "the on_header_field callback failed") \ + XX(CB_header_value, "the on_header_value callback failed") \ + XX(CB_headers_complete, "the on_headers_complete callback failed") \ + XX(CB_body, "the on_body callback failed") \ + XX(CB_message_complete, "the on_message_complete callback failed") \ + XX(CB_reason, "the on_reason callback failed") \ + XX(CB_chunk_header, "the on_chunk_header callback failed") \ + XX(CB_chunk_complete, "the on_chunk_complete callback failed") \ + \ + /* Parsing-related errors */ \ + XX(INVALID_EOF_STATE, "stream ended at an unexpected time") \ + XX(HEADER_OVERFLOW, \ + "too many header bytes seen; overflow detected") \ + XX(CLOSED_CONNECTION, \ + "data received after completed connection: close message") \ + XX(INVALID_VERSION, "invalid HTTP version") \ + XX(INVALID_STATUS, "invalid HTTP status code") \ + XX(INVALID_METHOD, "invalid HTTP method") \ + XX(INVALID_URL, "invalid URL") \ + XX(INVALID_HOST, "invalid host") \ + XX(INVALID_PORT, "invalid port") \ + XX(INVALID_PATH, "invalid path") \ + XX(INVALID_QUERY_STRING, "invalid query string") \ + XX(INVALID_FRAGMENT, "invalid fragment") \ + XX(LF_EXPECTED, "LF character expected") \ + XX(INVALID_HEADER_TOKEN, "invalid character in header") \ + XX(INVALID_CONTENT_LENGTH, \ + "invalid character in content-length header") \ + XX(HUGE_CONTENT_LENGTH, \ + "content-length header too large") \ + XX(INVALID_CHUNK_SIZE, \ + "invalid character in chunk size header") \ + XX(HUGE_CHUNK_SIZE, \ + "chunk header size too large") \ + XX(INVALID_CONSTANT, "invalid constant string") \ + XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\ + XX(STRICT, "strict mode assertion failed") \ + XX(PAUSED, "parser is paused") \ + XX(UNKNOWN, "an unknown error occurred") + + +/* Define HPE_* values for each errno value above */ +#define HTTP_ERRNO_GEN(n, s) HPE_##n, +enum http_errno { + HTTP_ERRNO_MAP(HTTP_ERRNO_GEN) +}; +#undef HTTP_ERRNO_GEN + + +/* Get an http_errno value from an http_parser */ +#define HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno) + +/* Get the line number that generated the current error */ +#if HTTP_PARSER_DEBUG +#define HTTP_PARSER_ERRNO_LINE(p) ((p)->error_lineno) +#else +#define HTTP_PARSER_ERRNO_LINE(p) 0 +#endif + + +struct http_parser { + /** PRIVATE **/ + unsigned char type : 2; /* enum http_parser_type */ + unsigned char flags : 6; /* F_* values from 'flags' enum; semi-public */ + unsigned char state; /* enum state from http_parser.c */ + unsigned char header_state; /* enum header_state from http_parser.c */ + unsigned char index; /* index into current matcher */ + + uint32_t nread; /* # bytes read in various scenarios */ + int64_t content_length; /* # bytes in body (0 if no Content-Length header) */ + + /** READ-ONLY **/ + unsigned short http_major; + unsigned short http_minor; + unsigned short status_code; /* responses only */ + unsigned char method; /* requests only */ + unsigned char http_errno : 7; + + /* 1 = Upgrade header was present and the parser has exited because of that. + * 0 = No upgrade header present. + * Should be checked when http_parser_execute() returns in addition to + * error checking. + */ + char upgrade : 1; + +#if HTTP_PARSER_DEBUG + uint32_t error_lineno; +#endif + + /** PUBLIC **/ + void *data; /* A pointer to get hook to the "connection" or "socket" object */ +}; + + +struct http_parser_settings { + http_cb on_message_begin; + http_data_cb on_url; + http_data_cb on_header_field; + http_data_cb on_header_value; + http_data_cb on_headers_complete; + http_data_cb on_body; + http_cb on_message_complete; + http_data_cb on_reason; + /* When on_chunk_header is called, the current chunk length is stored + * in parser->content_length. + */ + http_cb on_chunk_header; + http_cb on_chunk_complete; +}; + + +enum http_parser_url_fields + { UF_SCHEMA = 0 + , UF_HOST = 1 + , UF_PORT = 2 + , UF_PATH = 3 + , UF_QUERY = 4 + , UF_FRAGMENT = 5 + , UF_USERINFO = 6 + , UF_MAX = 7 +}; + + +/* Result structure for http_parser_parse_url(). + * + * Callers should index into field_data[] with UF_* values iff field_set + * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and + * because we probably have padding left over), we convert any port to + * a uint16_t. + */ +struct http_parser_url { + uint16_t field_set; /* Bitmask of (1 << UF_*) values */ + uint16_t port; /* Converted UF_PORT string */ + + struct { + uint16_t off; /* Offset into buffer in which field starts */ + uint16_t len; /* Length of run in buffer */ + } field_data[UF_MAX]; +}; + + +void http_parser_init(http_parser *parser, enum http_parser_type type); + + +size_t http_parser_execute(http_parser *parser, + const http_parser_settings *settings, + const char *data, + size_t len); + +/* Returns a string version of the HTTP method. */ +const char *http_method_str(enum http_method m); + +/* Return a string name of the given error */ +const char *http_errno_name(enum http_errno err); + +/* Return a string description of the given error */ +const char *http_errno_description(enum http_errno err); + +/* Parse a URL; return nonzero on failure */ +int http_parser_parse_url(const char *buf, size_t buflen, + int is_connect, + struct http_parser_url *u); + +/* Pause or un-pause the parser; a nonzero value pauses */ +void http_parser_pause(http_parser *parser, int paused); + +#if __cplusplus +} +#endif /* __cplusplus */ + +#endif diff --git a/proxygen/external/http_parser/http_parser_cpp.cpp b/proxygen/external/http_parser/http_parser_cpp.cpp new file mode 100644 index 0000000000..8af3f3ed6f --- /dev/null +++ b/proxygen/external/http_parser/http_parser_cpp.cpp @@ -0,0 +1,2381 @@ +/* Based on src/http/ngx_http_parse.c from NGINX copyright Igor Sysoev + * + * Additional changes are licensed under the same terms as NGINX and + * copyright Joyent, Inc. and other Node contributors. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include "proxygen/external/http_parser/http_parser.h" + +#include +#include +#include +#include + +#if __cplusplus +#include + +namespace proxygen { + +#ifndef INT64_MAX +# define INT64_MAX std::numeric_limits::max() +#endif + +#else +#define nullptr NULL + +#endif /* __cplusplus */ + +#ifndef MIN +# define MIN(a,b) ((a) < (b) ? (a) : (b)) +#endif + + +#if HTTP_PARSER_DEBUG +#define SET_ERRNO(e) \ +do { \ + parser->http_errno = (e); \ + parser->error_lineno = __LINE__; \ +} while (0) +#else +#define SET_ERRNO(e) \ +do { \ + parser->http_errno = (e); \ +} while(0) +#endif + +#define RETURN(r) \ +do { \ + parser->state = state; \ + return (r); \ +} while(0) + +/* Run the notify callback FOR, returning ER if it fails */ +#define _CALLBACK_NOTIFY(FOR, ER) \ +do { \ + parser->state = state; \ + assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \ + \ + if (0 != settings->on_##FOR(parser)) { \ + SET_ERRNO(HPE_CB_##FOR); \ + } \ + \ + /* We either errored above or got paused; get out */ \ + if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \ + return (ER); \ + } \ +} while (0) + +/* Run the notify callback FOR and consume the current byte */ +#define CALLBACK_NOTIFY(FOR) _CALLBACK_NOTIFY(FOR, p - data + 1) + +/* Run the notify callback FOR and don't consume the current byte */ +#define CALLBACK_NOTIFY_NOADVANCE(FOR) _CALLBACK_NOTIFY(FOR, p - data) + +/* Run data callback FOR with LEN bytes, returning ER if it fails */ +#define _CALLBACK_DATA(FOR, LEN, ER) \ +do { \ + parser->state = state; \ + assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \ + \ + if (FOR##_mark) { \ + if (0 != settings->on_##FOR(parser, FOR##_mark, (LEN))) { \ + SET_ERRNO(HPE_CB_##FOR); \ + } \ + \ + /* We either errored above or got paused; get out */ \ + if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \ + return (ER); \ + } \ + FOR##_mark = nullptr; \ + } \ +} while (0) + +/* Run the data callback FOR and consume the current byte */ +#define CALLBACK_DATA(FOR) \ + _CALLBACK_DATA(FOR, p - FOR##_mark, p - data + 1) + +/* Run the data callback FOR and don't consume the current byte */ +#define CALLBACK_DATA_NOADVANCE(FOR) \ + _CALLBACK_DATA(FOR, p - FOR##_mark, p - data) + +/* We just saw a synthetic space */ +#define CALLBACK_SPACE(FOR) \ +do { \ + parser->state = state; \ + if (0 != settings->on_##FOR(parser, SPACE, 1)) { \ + SET_ERRNO(HPE_CB_##FOR); \ + return (p - data); \ + } \ + \ + /* We either errored above or got paused; get out */ \ + if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \ + return (p - data); \ + } \ +} while (0) + +/* Set the mark FOR; non-destructive if mark is already set */ +#define MARK(FOR) \ +do { \ + if (!FOR##_mark) { \ + FOR##_mark = p; \ + } \ +} while (0) + + +#define CONTENT_LENGTH "content-length" +#define TRANSFER_ENCODING "transfer-encoding" +#define UPGRADE "upgrade" +#define CHUNKED "chunked" +#define SPACE " " + + +static const char *method_strings[] = + { "DELETE" + , "GET" + , "HEAD" + , "POST" + , "PUT" + , "CONNECT" + , "OPTIONS" + , "TRACE" + , "COPY" + , "LOCK" + , "MKCOL" + , "MOVE" + , "PROPFIND" + , "PROPPATCH" + , "UNLOCK" + , "REPORT" + , "MKACTIVITY" + , "CHECKOUT" + , "MERGE" + , "M-SEARCH" + , "NOTIFY" + , "SUBSCRIBE" + , "UNSUBSCRIBE" + , "PATCH" + }; + + +/* Tokens as defined by rfc 2616. Also lowercases them. + * token = 1* + * separators = "(" | ")" | "<" | ">" | "@" + * | "," | ";" | ":" | "\" | <"> + * | "/" | "[" | "]" | "?" | "=" + * | "{" | "}" | SP | HT + */ +static const char tokens[256] = { +/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ + ' ', '!', '"', '#', '$', '%', '&', '\'', +/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ + 0, 0, '*', '+', 0, '-', '.', '/', +/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ + '0', '1', '2', '3', '4', '5', '6', '7', +/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ + '8', '9', 0, 0, 0, 0, 0, 0, +/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ + 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g', +/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', +/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', +/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ + 'x', 'y', 'z', 0, 0, 0, '^', '_', +/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ + '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', +/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', +/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', +/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ + 'x', 'y', 'z', 0, '|', '}', '~', 0 }; + + +static const int8_t unhex[256] = + {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1 + ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 + ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + }; + +#if HTTP_PARSER_STRICT +# define T(v) 0 +#else +# define T(v) v +#endif + +static const uint8_t normal_url_char[256] = { +/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ + 0, T(1), 0, 0, T(1), 0, 0, 0, +/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ + 0, 1, 1, 0, 1, 1, 1, 1, +/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ + 1, 1, 1, 1, 1, 1, 1, 0, +/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ + 1, 1, 1, 1, 1, 1, 1, 1, +/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ + 1, 1, 1, 1, 1, 1, 1, 0, }; + +#undef T + +enum state + { s_dead = 1 /* important that this is > 0 */ + , s_pre_start_req_or_res + , s_start_req_or_res + , s_res_or_resp_H + + , s_pre_start_res + , s_start_res + , s_res_H + , s_res_HT + , s_res_HTT + , s_res_HTTP + , s_res_first_http_major + , s_res_http_major + , s_res_first_http_minor + , s_res_http_minor + , s_res_first_status_code + , s_res_status_code + , s_res_status_start + , s_res_status + , s_res_line_almost_done + + , s_pre_start_req + , s_start_req + , s_req_method + , s_req_spaces_before_url + , s_req_schema + , s_req_schema_slash + , s_req_schema_slash_slash + , s_req_server_start + , s_req_server + , s_req_server_with_at + , s_req_host_start + , s_req_host + , s_req_host_ipv6 + , s_req_host_done + , s_req_port + , s_req_path + , s_req_query_string_start + , s_req_query_string + , s_req_fragment_start + , s_req_fragment + , s_req_http_start + , s_req_http_H + , s_req_http_HT + , s_req_http_HTT + , s_req_http_HTTP + , s_req_first_http_major + , s_req_http_major + , s_req_first_http_minor + , s_req_http_minor + , s_req_line_almost_done + + , s_header_field_start + , s_header_field + , s_header_value_start + , s_header_value + , s_header_value_lws + + , s_header_almost_done + + , s_chunk_size_start + , s_chunk_size + , s_chunk_parameters + , s_chunk_size_almost_done + + , s_headers_almost_done + , s_headers_done + + /* Important: 's_headers_done' must be the last 'header' state. All + * states beyond this must be 'body' states. It is used for overflow + * checking. See the PARSING_HEADER() macro. + */ + + , s_chunk_data + , s_chunk_data_almost_done + , s_chunk_data_done + + , s_body_identity + , s_body_identity_eof + + , s_message_done + }; + + +#define PARSING_HEADER(state) (state <= s_headers_done) + + +enum header_states + { h_general = 0 + + , h_general_and_quote + , h_general_and_quote_and_escape + + , h_matching_content_length + , h_matching_transfer_encoding + , h_matching_upgrade + + , h_content_length + , h_transfer_encoding + , h_upgrade + + , h_matching_transfer_encoding_chunked + + , h_transfer_encoding_chunked + }; + +enum http_host_state + { + s_http_host_dead = 1 + , s_http_userinfo_start + , s_http_userinfo + , s_http_host_start + , s_http_host_v6_start + , s_http_host + , s_http_host_v6 + , s_http_host_v6_end + , s_http_host_port_start + , s_http_host_port +}; + + +/* Macros for character classes; depends on strict-mode */ +#define CR '\r' +#define LF '\n' +#define QT '"' +#define BS '\\' +#define LOWER(c) (unsigned char)(c | 0x20) +#define TOKEN(c) (tokens[(unsigned char)c]) +#define IS_ALPHA(c) (LOWER(c) >= 'a' && LOWER(c) <= 'z') +#define IS_NUM(c) ((c) >= '0' && (c) <= '9') +#define IS_ALPHANUM(c) (IS_ALPHA(c) || IS_NUM(c)) +#define IS_HEX(c) (IS_NUM(c) || (LOWER(c) >= 'a' && LOWER(c) <= 'f')) +#define IS_MARK(c) ((c) == '-' || (c) == '_' || (c) == '.' || \ + (c) == '!' || (c) == '~' || (c) == '*' || (c) == '\'' || (c) == '(' || \ + (c) == ')') +#define IS_USERINFO_CHAR(c) (IS_ALPHANUM(c) || IS_MARK(c) || (c) == '%' || \ + (c) == ';' || (c) == ':' || (c) == '&' || (c) == '=' || (c) == '+' || \ + (c) == '$' || (c) == ',') + +#if HTTP_PARSER_STRICT +#define IS_URL_CHAR(c) (normal_url_char[(unsigned char) (c)]) +#define IS_HOST_CHAR(c) (IS_ALPHANUM(c) || (c) == '.' || (c) == '-') +#else +#define IS_URL_CHAR(c) \ + (normal_url_char[(unsigned char) (c)] || ((c) & 0x80)) +#define IS_HOST_CHAR(c) \ + (IS_ALPHANUM(c) || (c) == '.' || (c) == '-' || (c) == '_') +#endif + + +#define start_state (parser->type == HTTP_REQUEST ? s_pre_start_req : s_pre_start_res) + +#define STRICT_CHECK(cond) +#define NEW_MESSAGE() start_state + +/* Map errno values to strings for human-readable output */ +#define HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s }, +static struct { + const char *name; + const char *description; +} http_strerror_tab[] = { + HTTP_ERRNO_MAP(HTTP_STRERROR_GEN) +}; +#undef HTTP_STRERROR_GEN + +/* Our URL parser. + * + * This is designed to be shared by http_parser_execute() for URL validation, + * hence it has a state transition + byte-for-byte interface. In addition, it + * is meant to be embedded in http_parser_parse_url(), which does the dirty + * work of turning state transitions URL components for its API. + * + * This function should only be invoked with non-space characters. It is + * assumed that the caller cares about (and can detect) the transition between + * URL and non-URL states by looking for these. + */ +static enum state +parse_url_char(enum state s, const char ch) +{ + if (ch == ' ' || ch == '\r' || ch == '\n') { + return s_dead; + } + +#if HTTP_PARSER_STRICT + if (ch == '\t' || ch == '\f') { + return s_dead; + } +#endif + + switch (s) { + case s_req_spaces_before_url: + /* Proxied requests are followed by scheme of an absolute URI (alpha). + * All methods except CONNECT are followed by '/' or '*'. + */ + + if (ch == '/' || ch == '*') { + return s_req_path; + } + + if (IS_ALPHA(ch)) { + return s_req_schema; + } + + break; + + case s_req_schema: + if (IS_ALPHA(ch)) { + return s; + } + + if (ch == ':') { + return s_req_schema_slash; + } + + break; + + case s_req_schema_slash: + if (ch == '/') { + return s_req_schema_slash_slash; + } + + break; + + case s_req_schema_slash_slash: + if (ch == '/') { + return s_req_server_start; + } + + break; + + case s_req_server_with_at: + if (ch == '@') { + return s_dead; + } + + /* FALLTHROUGH */ + case s_req_server_start: + case s_req_server: + if (ch == '/') { + return s_req_path; + } + + if (ch == '?') { + return s_req_query_string_start; + } + + if (ch == '@') { + return s_req_server_with_at; + } + + if (IS_USERINFO_CHAR(ch) || ch == '[' || ch == ']') { + return s_req_server; + } + + break; + + case s_req_path: + if (IS_URL_CHAR(ch)) { + return s; + } + + switch (ch) { + case '?': + return s_req_query_string_start; + + case '#': + return s_req_fragment_start; + } + + break; + + case s_req_query_string_start: + case s_req_query_string: + if (IS_URL_CHAR(ch)) { + return s_req_query_string; + } + + switch (ch) { + case '?': + /* allow extra '?' in query string */ + return s_req_query_string; + + case '#': + return s_req_fragment_start; + } + + break; + + case s_req_fragment_start: + if (IS_URL_CHAR(ch)) { + return s_req_fragment; + } + + switch (ch) { + case '?': + return s_req_fragment; + + case '#': + return s; + } + + break; + + case s_req_fragment: + if (IS_URL_CHAR(ch)) { + return s; + } + + switch (ch) { + case '?': + case '#': + return s; + } + + break; + + default: + break; + } + + /* We should never fall out of the switch above unless there's an error */ + return s_dead; +} + +size_t http_parser_execute (http_parser *parser, + const http_parser_settings *settings, + const char *data, + size_t len) +{ + char c, ch; + int8_t unhex_val; + const char *p = data; + + /* Optimization: within the parsing loop below, we refer to this + * local copy of the state rather than parser->state. The compiler + * can't be sure whether parser->state will change during a callback, + * so it generates a lot of memory loads and stores to keep a register + * copy of the state in sync with the memory copy. We know, however, + * that the callbacks aren't allowed to change the parser state, so + * the parsing loop works with this local variable and only copies + * the value back to parser->loop before returning or invoking a + * callback. + */ + unsigned char state = parser->state; + + /* We're in an error state. Don't bother doing anything. */ + if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { + RETURN(0); + } + + if (len == 0) { + switch (state) { + case s_body_identity_eof: + /* Use of CALLBACK_NOTIFY() here would erroneously return 1 byte read if + * we got paused. + */ + CALLBACK_NOTIFY_NOADVANCE(message_complete); + RETURN(0); + + case s_pre_start_req_or_res: + case s_pre_start_res: + case s_pre_start_req: + RETURN(0); + + default: + SET_ERRNO(HPE_INVALID_EOF_STATE); + RETURN(1); + } + } + + /* technically we could combine all of these (except for url_mark) into one + variable, saving stack space, but it seems more clear to have them + separated. */ + const char *header_field_mark = 0; + const char *header_value_mark = 0; + const char *url_mark = 0; + const char *reason_mark = 0; + const char *body_mark = 0; + + if (state == s_header_field) + header_field_mark = data; + if (state == s_header_value) + header_value_mark = data; + if (state == s_req_path || + state == s_req_schema || + state == s_req_schema_slash || + state == s_req_schema_slash_slash || + state == s_req_port || + state == s_req_query_string_start || + state == s_req_query_string || + state == s_req_host_start || + state == s_req_host || + state == s_req_host_ipv6 || + state == s_req_host_done || + state == s_req_fragment_start || + state == s_req_fragment) + url_mark = data; + if (state == s_res_status) + reason_mark = data; + + /* Used only for overflow checking. If the parser is in a parsing-headers + * state, then its value is equal to max(data, the beginning of the current + * message or chunk). If the parser is in a not-parsing-headers state, then + * its value is irrelevant. + */ + const char* data_or_header_data_start = data; + + for (p = data; p != data + len; p++) { + ch = *p; + + reexecute_byte: + switch (state) { + + case s_pre_start_req_or_res: + if (ch == CR || ch == LF) + break; + state = s_start_req_or_res; + CALLBACK_NOTIFY_NOADVANCE(message_begin); + goto reexecute_byte; + + case s_start_req_or_res: + { + parser->flags = 0; + parser->content_length = -1; + + if (ch == 'H') { + state = s_res_or_resp_H; + } else { + parser->type = HTTP_REQUEST; + state = s_start_req; + goto reexecute_byte; + } + + break; + } + + case s_res_or_resp_H: + if (ch == 'T') { + parser->type = HTTP_RESPONSE; + state = s_res_HT; + } else { + if (ch != 'E') { + SET_ERRNO(HPE_INVALID_CONSTANT); + goto error; + } + + parser->type = HTTP_REQUEST; + parser->method = HTTP_HEAD; + parser->index = 2; + state = s_req_method; + } + break; + + case s_pre_start_res: + if (ch == CR || ch == LF) + break; + state = s_start_res; + CALLBACK_NOTIFY_NOADVANCE(message_begin); + goto reexecute_byte; + + case s_start_res: + { + parser->flags = 0; + parser->content_length = -1; + + switch (ch) { + case 'H': + state = s_res_H; + break; + + default: + SET_ERRNO(HPE_INVALID_CONSTANT); + goto error; + } + + break; + } + + case s_res_H: + STRICT_CHECK(ch != 'T'); + state = s_res_HT; + break; + + case s_res_HT: + STRICT_CHECK(ch != 'T'); + state = s_res_HTT; + break; + + case s_res_HTT: + STRICT_CHECK(ch != 'P'); + state = s_res_HTTP; + break; + + case s_res_HTTP: + STRICT_CHECK(ch != '/'); + state = s_res_first_http_major; + break; + + case s_res_first_http_major: + if (ch < '0' || ch > '9') { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_major = ch - '0'; + state = s_res_http_major; + break; + + /* major HTTP version or dot */ + case s_res_http_major: + { + if (ch == '.') { + state = s_res_first_http_minor; + break; + } + + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_major *= 10; + parser->http_major += ch - '0'; + + if (parser->http_major > 999) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + break; + } + + /* first digit of minor HTTP version */ + case s_res_first_http_minor: + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_minor = ch - '0'; + state = s_res_http_minor; + break; + + /* minor HTTP version or end of request line */ + case s_res_http_minor: + { + if (ch == ' ') { + state = s_res_first_status_code; + break; + } + + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_minor *= 10; + parser->http_minor += ch - '0'; + + if (parser->http_minor > 999) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + break; + } + + case s_res_first_status_code: + { + if (!IS_NUM(ch)) { + if (ch == ' ') { + break; + } + + SET_ERRNO(HPE_INVALID_STATUS); + goto error; + } + parser->status_code = ch - '0'; + state = s_res_status_code; + break; + } + + case s_res_status_code: + { + if (!IS_NUM(ch)) { + switch (ch) { + case ' ': + state = s_res_status; + break; + case CR: + state = s_res_line_almost_done; + break; + case LF: + state = s_header_field_start; + break; + default: + SET_ERRNO(HPE_INVALID_STATUS); + goto error; + } + break; + } + + parser->status_code *= 10; + parser->status_code += ch - '0'; + + if (parser->status_code > 999) { + SET_ERRNO(HPE_INVALID_STATUS); + goto error; + } + + break; + } + + case s_res_status: + /* the human readable status. e.g. "NOT FOUND" */ + MARK(reason); + if (ch == CR) { + state = s_res_line_almost_done; + CALLBACK_DATA(reason); + break; + } + + if (ch == LF) { + state = s_header_field_start; + CALLBACK_DATA(reason); + break; + } + break; + + case s_res_line_almost_done: + STRICT_CHECK(ch != LF); + state = s_header_field_start; + break; + + case s_pre_start_req: + if (ch == CR || ch == LF) { + break; + } + state = s_start_req; + CALLBACK_NOTIFY_NOADVANCE(message_begin); + goto reexecute_byte; + + case s_start_req: + { + parser->flags = 0; + parser->content_length = -1; + + if (!IS_ALPHA(ch)) { + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + + parser->method = (enum http_method) 0; + parser->index = 1; + switch (ch) { + case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break; + case 'D': parser->method = HTTP_DELETE; break; + case 'G': parser->method = HTTP_GET; break; + case 'H': parser->method = HTTP_HEAD; break; + case 'L': parser->method = HTTP_LOCK; break; + case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH */ break; + case 'N': parser->method = HTTP_NOTIFY; break; + case 'O': parser->method = HTTP_OPTIONS; break; + case 'P': parser->method = HTTP_POST; + /* or PROPFIND or PROPPATCH or PUT or PATCH */ + break; + case 'R': parser->method = HTTP_REPORT; break; + case 'S': parser->method = HTTP_SUBSCRIBE; break; + case 'T': parser->method = HTTP_TRACE; break; + case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE */ break; + default: + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + state = s_req_method; + + break; + } + + case s_req_method: + { + if (ch == '\0') { + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + + const char *matcher = method_strings[parser->method]; + if (ch == ' ' && matcher[parser->index] == '\0') { + state = s_req_spaces_before_url; + } else if (ch == matcher[parser->index]) { + ; /* nada */ + } else if (parser->method == HTTP_CONNECT) { + if (parser->index == 1 && ch == 'H') { + parser->method = HTTP_CHECKOUT; + } else if (parser->index == 2 && ch == 'P') { + parser->method = HTTP_COPY; + } else { + goto error; + } + } else if (parser->method == HTTP_MKCOL) { + if (parser->index == 1 && ch == 'O') { + parser->method = HTTP_MOVE; + } else if (parser->index == 1 && ch == 'E') { + parser->method = HTTP_MERGE; + } else if (parser->index == 1 && ch == '-') { + parser->method = HTTP_MSEARCH; + } else if (parser->index == 2 && ch == 'A') { + parser->method = HTTP_MKACTIVITY; + } else { + goto error; + } + } else if (parser->index == 1 && parser->method == HTTP_POST) { + if (ch == 'R') { + parser->method = HTTP_PROPFIND; /* or HTTP_PROPPATCH */ + } else if (ch == 'U') { + parser->method = HTTP_PUT; + } else if (ch == 'A') { + parser->method = HTTP_PATCH; + } else { + goto error; + } + } else if (parser->index == 2 && parser->method == HTTP_UNLOCK && ch == 'S') { + parser->method = HTTP_UNSUBSCRIBE; + } else if (parser->index == 4 && parser->method == HTTP_PROPFIND && ch == 'P') { + parser->method = HTTP_PROPPATCH; + } else { + SET_ERRNO(HPE_INVALID_METHOD); + goto error; + } + + ++parser->index; + break; + } + + case s_req_spaces_before_url: + { + if (ch == ' ') break; + + // CONNECT requests must be followed by a : + if (parser->method == HTTP_CONNECT) { + MARK(url); + state = s_req_host_start; + goto reexecute_byte; + } + + if (ch == '/' || ch == '*') { + MARK(url); + state = s_req_path; + break; + } + + /* Proxied requests are followed by scheme of an absolute URI (alpha). + * All other methods are followed by '/' or '*' (handled above). + */ + if (IS_ALPHA(ch)) { + MARK(url); + state = s_req_schema; + break; + } + + SET_ERRNO(HPE_INVALID_URL); + goto error; + } + + case s_req_schema: + { + if (IS_ALPHA(ch)) break; + + if (ch == ':') { + state = s_req_schema_slash; + break; + } + + SET_ERRNO(HPE_INVALID_URL); + goto error; + } + + case s_req_schema_slash: + STRICT_CHECK(ch != '/'); + state = s_req_schema_slash_slash; + break; + + case s_req_schema_slash_slash: + STRICT_CHECK(ch != '/'); + state = s_req_host_start; + break; + + case s_req_host_start: + if (ch == '[') { + state = s_req_host_ipv6; + break; + } else if (IS_ALPHANUM(ch)) { + state = s_req_host; + break; + } + + SET_ERRNO(HPE_INVALID_HOST); + goto error; + + case s_req_host: + if (IS_HOST_CHAR(ch)) break; + state = s_req_host_done; + goto reexecute_byte; + + case s_req_host_ipv6: + if (IS_HEX(ch) || ch == ':') break; + if (ch == ']') { + state = s_req_host_done; + break; + } + + SET_ERRNO(HPE_INVALID_HOST); + goto error; + + case s_req_host_done: + switch (ch) { + case ':': + state = s_req_port; + break; + case '/': + state = s_req_path; + break; + case ' ': + /* The request line looks like: + * "GET http://foo.bar.com HTTP/1.1" + * That is, there is no path. + */ + state = s_req_http_start; + CALLBACK_DATA(url); + break; + case '?': + state = s_req_query_string_start; + break; + default: + SET_ERRNO(HPE_INVALID_HOST); + goto error; + } + + break; + + case s_req_port: + { + if (IS_NUM(ch)) break; + switch (ch) { + case '/': + state = s_req_path; + break; + case ' ': + /* The request line looks like: + * "GET http://foo.bar.com:1234 HTTP/1.1" + * That is, there is no path. + */ + state = s_req_http_start; + CALLBACK_DATA(url); + break; + case '?': + state = s_req_query_string_start; + break; + default: + SET_ERRNO(HPE_INVALID_PORT); + goto error; + } + break; + } + + case s_req_path: + { + if (IS_URL_CHAR(ch)) break; + + switch (ch) { + case ' ': + state = s_req_http_start; + CALLBACK_DATA(url); + break; + case CR: + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + CALLBACK_DATA(url); + break; + case LF: + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + CALLBACK_DATA(url); + break; + case '?': + state = s_req_query_string_start; + break; + case '#': + state = s_req_fragment_start; + break; + default: + SET_ERRNO(HPE_INVALID_PATH); + goto error; + } + break; + } + + case s_req_query_string_start: + { + if (IS_URL_CHAR(ch)) { + state = s_req_query_string; + break; + } + + switch (ch) { + case '?': + break; /* XXX ignore extra '?' ... is this right? */ + case ' ': + state = s_req_http_start; + CALLBACK_DATA(url); + break; + case CR: + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + CALLBACK_DATA(url); + break; + case LF: + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + CALLBACK_DATA(url); + break; + case '#': + state = s_req_fragment_start; + break; + default: + SET_ERRNO(HPE_INVALID_QUERY_STRING); + goto error; + } + break; + } + + case s_req_query_string: + { + if (IS_URL_CHAR(ch)) break; + + switch (ch) { + case '?': + /* allow extra '?' in query string */ + break; + case ' ': + state = s_req_http_start; + CALLBACK_DATA(url); + break; + case CR: + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + CALLBACK_DATA(url); + break; + case LF: + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + CALLBACK_DATA(url); + break; + case '#': + state = s_req_fragment_start; + break; + default: + SET_ERRNO(HPE_INVALID_QUERY_STRING); + goto error; + } + break; + } + + case s_req_fragment_start: + { + if (IS_URL_CHAR(ch)) { + state = s_req_fragment; + break; + } + + switch (ch) { + case ' ': + state = s_req_http_start; + CALLBACK_DATA(url); + break; + case CR: + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + CALLBACK_DATA(url); + break; + case LF: + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + CALLBACK_DATA(url); + break; + case '?': + state = s_req_fragment; + break; + case '#': + break; + default: + SET_ERRNO(HPE_INVALID_FRAGMENT); + goto error; + } + break; + } + + case s_req_fragment: + { + if (IS_URL_CHAR(ch)) break; + + switch (ch) { + case ' ': + state = s_req_http_start; + CALLBACK_DATA(url); + break; + case CR: + parser->http_major = 0; + parser->http_minor = 9; + state = s_req_line_almost_done; + CALLBACK_DATA(url); + break; + case LF: + parser->http_major = 0; + parser->http_minor = 9; + state = s_header_field_start; + CALLBACK_DATA(url); + break; + case '?': + case '#': + break; + default: + SET_ERRNO(HPE_INVALID_FRAGMENT); + goto error; + } + break; + } + + case s_req_http_start: + switch (ch) { + case 'H': + state = s_req_http_H; + break; + case ' ': + break; + default: + SET_ERRNO(HPE_INVALID_CONSTANT); + goto error; + } + break; + + case s_req_http_H: + STRICT_CHECK(ch != 'T'); + state = s_req_http_HT; + break; + + case s_req_http_HT: + STRICT_CHECK(ch != 'T'); + state = s_req_http_HTT; + break; + + case s_req_http_HTT: + STRICT_CHECK(ch != 'P'); + state = s_req_http_HTTP; + break; + + case s_req_http_HTTP: + STRICT_CHECK(ch != '/'); + state = s_req_first_http_major; + break; + + /* first digit of major HTTP version */ + case s_req_first_http_major: + if (ch < '0' || ch > '9') { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_major = ch - '0'; + state = s_req_http_major; + break; + + /* major HTTP version or dot */ + case s_req_http_major: + { + if (ch == '.') { + state = s_req_first_http_minor; + break; + } + + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_major *= 10; + parser->http_major += ch - '0'; + + if (parser->http_major > 999) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + break; + } + + /* first digit of minor HTTP version */ + case s_req_first_http_minor: + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_minor = ch - '0'; + state = s_req_http_minor; + break; + + /* minor HTTP version or end of request line */ + case s_req_http_minor: + { + if (ch == CR) { + state = s_req_line_almost_done; + break; + } + + if (ch == LF) { + state = s_header_field_start; + break; + } + + /* XXX allow spaces after digit? */ + + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + parser->http_minor *= 10; + parser->http_minor += ch - '0'; + + if (parser->http_minor > 999) { + SET_ERRNO(HPE_INVALID_VERSION); + goto error; + } + + break; + } + + /* end of request line */ + case s_req_line_almost_done: + { + if (ch != LF) { + SET_ERRNO(HPE_LF_EXPECTED); + goto error; + } + + state = s_header_field_start; + break; + } + + case s_header_field_start: + { + if (ch == CR) { + state = s_headers_almost_done; + break; + } + + if (ch == LF) { + /* they might be just sending \n instead of \r\n so this would be + * the second \n to denote the end of headers*/ + state = s_headers_almost_done; + goto reexecute_byte; + } + + c = TOKEN(ch); + + if (!c) { + SET_ERRNO(HPE_INVALID_HEADER_TOKEN); + goto error; + } + + MARK(header_field); + + parser->index = 0; + state = s_header_field; + + switch (c) { + case 'c': + parser->header_state = h_matching_content_length; + break; + + case 't': + parser->header_state = h_matching_transfer_encoding; + break; + + case 'u': + parser->header_state = h_matching_upgrade; + break; + + default: + parser->header_state = h_general; + break; + } + break; + } + + case s_header_field: + { + c = TOKEN(ch); + + if (c) { + switch (parser->header_state) { + case h_general: + + // fast-forwarding, wheeeeeee! + #define MOVE_THE_HEAD do { \ + ++p; \ + if (!TOKEN(*p)) { \ + ch = *p; \ + goto notatoken; \ + } \ + } while(0); + + if (data + len - p >= 9) { + MOVE_THE_HEAD + MOVE_THE_HEAD + MOVE_THE_HEAD + MOVE_THE_HEAD + MOVE_THE_HEAD + MOVE_THE_HEAD + MOVE_THE_HEAD + MOVE_THE_HEAD + } else if (data + len - p >= 4) { + MOVE_THE_HEAD + MOVE_THE_HEAD + MOVE_THE_HEAD + } + + break; + + /* content-length */ + + case h_matching_content_length: + parser->index++; + if (parser->index > sizeof(CONTENT_LENGTH)-1 + || c != CONTENT_LENGTH[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(CONTENT_LENGTH)-2) { + parser->header_state = h_content_length; + } + break; + + /* transfer-encoding */ + + case h_matching_transfer_encoding: + parser->index++; + if (parser->index > sizeof(TRANSFER_ENCODING)-1 + || c != TRANSFER_ENCODING[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(TRANSFER_ENCODING)-2) { + parser->header_state = h_transfer_encoding; + } + break; + + /* upgrade */ + + case h_matching_upgrade: + parser->index++; + if (parser->index > sizeof(UPGRADE)-1 + || c != UPGRADE[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(UPGRADE)-2) { + parser->header_state = h_upgrade; + } + break; + + case h_content_length: + case h_transfer_encoding: + case h_upgrade: + if (ch != ' ') parser->header_state = h_general; + break; + + default: + assert(0 && "Unknown header_state"); + break; + } + break; + } + + notatoken: + if (ch == ':') { + state = s_header_value_start; + CALLBACK_DATA(header_field); + break; + } + + SET_ERRNO(HPE_INVALID_HEADER_TOKEN); + goto error; + } + + case s_header_value_start: + { + if (ch == ' ' || ch == '\t') break; + + MARK(header_value); + + state = s_header_value; + parser->index = 0; + + if (ch == CR) { + STRICT_CHECK(parser->quote != 0); + parser->header_state = h_general; + state = s_header_almost_done; + CALLBACK_DATA(header_value); + break; + } + + if (ch == LF) { + STRICT_CHECK(parser->quote != 0); + state = s_header_field_start; + CALLBACK_DATA(header_value); + break; + } + + c = LOWER(ch); + + switch (parser->header_state) { + case h_upgrade: + parser->flags |= F_UPGRADE; + parser->header_state = h_general; + break; + + case h_transfer_encoding: + /* looking for 'Transfer-Encoding: chunked' */ + if ('c' == c) { + parser->header_state = h_matching_transfer_encoding_chunked; + } else { + parser->header_state = h_general; + } + break; + + case h_content_length: + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); + goto error; + } + + parser->content_length = ch - '0'; + break; + + default: + parser->header_state = ch == QT ? h_general_and_quote : h_general; + break; + } + break; + } + + case s_header_value: + { + cr_or_lf_or_qt: + if (ch == CR && + parser->header_state != h_general_and_quote_and_escape) { + state = s_header_almost_done; + CALLBACK_DATA(header_value); + break; + } + + if (ch == LF && + parser->header_state != h_general_and_quote_and_escape) { + state = s_header_almost_done; + CALLBACK_DATA_NOADVANCE(header_value); + goto reexecute_byte; + } + + switch (parser->header_state) { + case h_general: + if (ch == QT) { + parser->header_state = h_general_and_quote; + } + + // fast-forwarding, wheee! + #define MOVE_FAST do { \ + ++p; \ + ch = *p; \ + if (ch == CR || ch == LF || ch == QT) { \ + goto cr_or_lf_or_qt; \ + } \ + } while(0); + + if (data + len - p >= 12) { + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + } else if (data + len - p >= 5) { + MOVE_FAST + MOVE_FAST + MOVE_FAST + MOVE_FAST + } + + break; + + case h_general_and_quote: + if (ch == QT) { + parser->header_state = h_general; + } else if (ch == BS) { + parser->header_state = h_general_and_quote_and_escape; + } + break; + + case h_general_and_quote_and_escape: + parser->header_state = h_general_and_quote; + break; + + + case h_transfer_encoding: + SET_ERRNO(HPE_INVALID_HEADER_TOKEN); + goto error; + break; + + case h_content_length: + if (ch == ' ') break; + if (!IS_NUM(ch)) { + SET_ERRNO(HPE_INVALID_CONTENT_LENGTH); + goto error; + } + + if (parser->content_length > ((INT64_MAX - 10) / 10)) { + /* overflow */ + SET_ERRNO(HPE_HUGE_CONTENT_LENGTH); + goto error; + } + + parser->content_length *= 10; + parser->content_length += ch - '0'; + break; + + /* Transfer-Encoding: chunked */ + case h_matching_transfer_encoding_chunked: + parser->index++; + if (parser->index > sizeof(CHUNKED)-1 + || LOWER(ch) != CHUNKED[parser->index]) { + parser->header_state = h_general; + } else if (parser->index == sizeof(CHUNKED)-2) { + parser->header_state = h_transfer_encoding_chunked; + } + break; + + case h_transfer_encoding_chunked: + if (ch != ' ') parser->header_state = h_general; + break; + + default: + state = s_header_value; + parser->header_state = h_general; + break; + } + break; + } + + case s_header_almost_done: + { + if (ch == LF) { + state = s_header_value_lws; + } else { + state = s_header_value; + } + + switch (parser->header_state) { + case h_transfer_encoding_chunked: + parser->flags |= F_CHUNKED; + break; + default: + break; + } + + if (ch != LF) { + CALLBACK_SPACE(header_value); + } + + break; + } + + case s_header_value_lws: + { + if (ch == ' ' || ch == '\t') + { + state = s_header_value_start; + CALLBACK_SPACE(header_value); + } + else + { + state = s_header_field_start; + goto reexecute_byte; + } + break; + } + + case s_headers_almost_done: + { + STRICT_CHECK(ch != LF); + + if (parser->flags & F_TRAILING) { + /* End of a chunked request */ + state = s_message_done; + CALLBACK_NOTIFY_NOADVANCE(chunk_complete); + goto reexecute_byte; + } + + state = s_headers_done; + + /* Set this here so that on_headers_complete() callbacks can see it */ + parser->upgrade = + (parser->flags & F_UPGRADE || parser->method == HTTP_CONNECT); + + /* Here we call the headers_complete callback. This is somewhat + * different than other callbacks because if the user returns 1, we + * will interpret that as saying that this message has no body. This + * is needed for the annoying case of receiving a response to a HEAD + * request. + * + * We'd like to use CALLBACK_NOTIFY_NOADVANCE() here but we cannot, so + * we have to simulate it by handling a change in errno below. + */ + size_t header_size = p - data + 1; + switch (settings->on_headers_complete(parser, nullptr, header_size)) { + case 0: + break; + + case 1: + parser->flags |= F_SKIPBODY; + break; + + default: + SET_ERRNO(HPE_CB_headers_complete); + RETURN(p - data); /* Error */ + } + + if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { + RETURN(p - data); + } + + goto reexecute_byte; + } + + case s_headers_done: + { + STRICT_CHECK(ch != LF); + + // we're done parsing headers, reset overflow counters + parser->nread = 0; + // (if we now move to s_body_*, then this is irrelevant) + data_or_header_data_start = p; + + int hasBody = parser->flags & F_CHUNKED || parser->content_length > 0; + if (parser->upgrade && (parser->method == HTTP_CONNECT || + (parser->flags & F_SKIPBODY) || !hasBody)) { + /* Exit, the rest of the message is in a different protocol. */ + state = NEW_MESSAGE(); + CALLBACK_NOTIFY(message_complete); + RETURN((p - data) + 1); + } + + if (parser->flags & F_SKIPBODY) { + state = NEW_MESSAGE(); + CALLBACK_NOTIFY(message_complete); + } else if (parser->flags & F_CHUNKED) { + /* chunked encoding - ignore Content-Length header */ + state = s_chunk_size_start; + } else { + if (parser->content_length == 0) { + /* Content-Length header given but zero: Content-Length: 0\r\n */ + state = NEW_MESSAGE(); + CALLBACK_NOTIFY(message_complete); + } else if (parser->content_length > 0) { + /* Content-Length header given and non-zero */ + state = s_body_identity; + } else { + unsigned short sc = parser->status_code; + if (parser->type == HTTP_REQUEST || + ((100 <= sc && sc <= 199) || sc == 204 || sc == 304)) { + /* Assume content-length 0 - read the next */ + state = NEW_MESSAGE(); + CALLBACK_NOTIFY(message_complete); + } else { + /* Read body until EOF */ + state = s_body_identity_eof; + } + } + } + + break; + } + + case s_body_identity: + { + uint64_t to_read = MIN(parser->content_length, (data + len) - p); + + assert(parser->content_length > 0); + + /* The difference between advancing content_length and p is because + * the latter will automatically advance on the next loop iteration. + * Further, if content_length ends up at 0, we want to see the last + * byte again for our message complete callback. + */ + MARK(body); + parser->content_length -= to_read; + p += to_read - 1; + + if (parser->content_length == 0) { + state = s_message_done; + + /* Mimic CALLBACK_DATA_NOADVANCE() but with one extra byte. + * + * The alternative to doing this is to wait for the next byte to + * trigger the data callback, just as in every other case. The + * problem with this is that this makes it difficult for the test + * harness to distinguish between complete-on-EOF and + * complete-on-length. It's not clear that this distinction is + * important for applications, but let's keep it for now. + */ + _CALLBACK_DATA(body, p - body_mark + 1, p - data); + goto reexecute_byte; + } + + break; + } + + /* read until EOF */ + case s_body_identity_eof: + MARK(body); + p = data + len - 1; + + break; + + case s_message_done: + state = NEW_MESSAGE(); + parser->nread = 0; + data_or_header_data_start = p; + CALLBACK_NOTIFY(message_complete); + if (parser->upgrade) { + /* Exit, the rest of the message is in a different protocol. */ + RETURN((p - data) + 1); + } + break; + + case s_chunk_size_start: + { + assert(parser->flags & F_CHUNKED); + + unhex_val = unhex[(unsigned char)ch]; + if (unhex_val == -1) { + SET_ERRNO(HPE_INVALID_CHUNK_SIZE); + goto error; + } + + parser->content_length = unhex_val; + state = s_chunk_size; + break; + } + + case s_chunk_size: + { + assert(parser->flags & F_CHUNKED); + + if (ch == CR) { + state = s_chunk_size_almost_done; + break; + } + + unhex_val = unhex[(unsigned char)ch]; + + if (unhex_val == -1) { + if (ch == ';' || ch == ' ') { + state = s_chunk_parameters; + break; + } + + SET_ERRNO(HPE_INVALID_CHUNK_SIZE); + goto error; + } + + if (parser->content_length > (INT64_MAX - unhex_val) >> 4) { + /* overflow */ + SET_ERRNO(HPE_HUGE_CHUNK_SIZE); + goto error; + } + parser->content_length *= 16; + parser->content_length += unhex_val; + break; + } + + case s_chunk_parameters: + { + assert(parser->flags & F_CHUNKED); + /* + * just ignore this shit. TODO check for overflow + * TODO: It would be nice to pass this information to the + * on_chunk_header callback. + */ + if (ch == CR) { + state = s_chunk_size_almost_done; + break; + } + break; + } + + case s_chunk_size_almost_done: + { + assert(parser->flags & F_CHUNKED); + STRICT_CHECK(ch != LF); + + if (parser->content_length == 0) { + parser->flags |= F_TRAILING; + state = s_header_field_start; + CALLBACK_NOTIFY(chunk_header); + } else { + state = s_chunk_data; + CALLBACK_NOTIFY(chunk_header); + } + break; + } + + case s_chunk_data: + { + uint64_t to_read = MIN(parser->content_length, (data + len) - p); + + assert(parser->flags & F_CHUNKED); + assert(parser->content_length > 0); + + /* See the explanation in s_body_identity for why the content + * length and data pointers are managed this way. + */ + MARK(body); + parser->content_length -= to_read; + p += to_read - 1; + + if (parser->content_length == 0) { + state = s_chunk_data_almost_done; + } + + break; + } + + case s_chunk_data_almost_done: + assert(parser->flags & F_CHUNKED); + assert(parser->content_length == 0); + STRICT_CHECK(ch != CR); + state = s_chunk_data_done; + CALLBACK_DATA(body); + break; + + case s_chunk_data_done: + assert(parser->flags & F_CHUNKED); + STRICT_CHECK(ch != LF); + state = s_chunk_size_start; + parser->nread = 0; + data_or_header_data_start = p; + CALLBACK_NOTIFY(chunk_complete); + break; + + default: + assert(0 && "unhandled state"); + SET_ERRNO(HPE_INVALID_INTERNAL_STATE); + goto error; + } + } + + /* We can check for overflow here because in Proxygen, len <= ~8KB and so the + * worst thing that can happen is that we catch the overflow at 88KB rather + * than at 80KB. + * In case of chunk encoding, we count the overflow for every + * chunk separately. + * We zero the nread counter (and reset data_or_header_data_start) when we + * start parsing a new message or a new chunk. + */ + if (PARSING_HEADER(state)) { + parser->nread += p - data_or_header_data_start; + if (parser->nread > HTTP_MAX_HEADER_SIZE) { + SET_ERRNO(HPE_HEADER_OVERFLOW); + goto error; + } + } + + /* Run callbacks for any marks that we have leftover after we ran out of + * bytes. There should be at most one of these set, so it's OK to invoke + * them in series (unset marks will not result in callbacks). + * + * We use the NOADVANCE() variety of callbacks here because 'p' has already + * overflowed 'data' and this allows us to correct for the off-by-one that + * we'd otherwise have (since CALLBACK_DATA() is meant to be run with a 'p' + * value that's in-bounds). + */ + + assert(((header_field_mark ? 1 : 0) + + (header_value_mark ? 1 : 0) + + (url_mark ? 1 : 0) + + (reason_mark ? 1 : 0) + + (body_mark ? 1 : 0)) <= 1); + + CALLBACK_DATA_NOADVANCE(header_field); + CALLBACK_DATA_NOADVANCE(header_value); + CALLBACK_DATA_NOADVANCE(url); + CALLBACK_DATA_NOADVANCE(reason); + CALLBACK_DATA_NOADVANCE(body); + + RETURN(len); + +error: + if (HTTP_PARSER_ERRNO(parser) == HPE_OK) { + SET_ERRNO(HPE_UNKNOWN); + } + + RETURN(p - data); +} + + +const char * http_method_str (enum http_method m) +{ + return method_strings[m]; +} + + +void +http_parser_init (http_parser *parser, enum http_parser_type t) +{ + parser->type = t; + parser->state = (t == HTTP_REQUEST ? s_pre_start_req : (t == HTTP_RESPONSE ? s_pre_start_res : s_pre_start_req_or_res)); + parser->nread = 0; + parser->upgrade = 0; + parser->flags = 0; + parser->method = 0; + parser->http_errno = HPE_OK; +} + +const char * +http_errno_name(enum http_errno err) { + assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0]))); + return http_strerror_tab[err].name; +} + +const char * +http_errno_description(enum http_errno err) { + assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0]))); + return http_strerror_tab[err].description; +} + + +static enum http_host_state +http_parse_host_char(enum http_host_state s, const char ch) { + switch(s) { + case s_http_userinfo: + case s_http_userinfo_start: + if (ch == '@') { + return s_http_host_start; + } + + if (IS_USERINFO_CHAR(ch)) { + return s_http_userinfo; + } + break; + + case s_http_host_start: + if (ch == '[') { + return s_http_host_v6_start; + } + + if (IS_HOST_CHAR(ch)) { + return s_http_host; + } + + break; + + case s_http_host: + if (IS_HOST_CHAR(ch)) { + return s_http_host; + } + + /* FALLTHROUGH */ + case s_http_host_v6_end: + if (ch == ':') { + return s_http_host_port_start; + } + + break; + + case s_http_host_v6: + if (ch == ']') { + return s_http_host_v6_end; + } + + /* FALLTHROUGH */ + case s_http_host_v6_start: + if (IS_HEX(ch) || ch == ':' || ch == '.') { + return s_http_host_v6; + } + + break; + + case s_http_host_port: + case s_http_host_port_start: + if (IS_NUM(ch)) { + return s_http_host_port; + } + + break; + + default: + break; + } + return s_http_host_dead; +} + +static int +http_parse_host(const char * buf, struct http_parser_url *u, int found_at) { + enum http_host_state s; + + const char *p; + size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len; + + u->field_data[UF_HOST].len = 0; + + s = found_at ? s_http_userinfo_start : s_http_host_start; + + for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) { + enum http_host_state new_s = http_parse_host_char(s, *p); + + if (new_s == s_http_host_dead) { + return 1; + } + + switch(new_s) { + case s_http_host: + if (s != s_http_host) { + u->field_data[UF_HOST].off = p - buf; + } + u->field_data[UF_HOST].len++; + break; + + case s_http_host_v6: + if (s != s_http_host_v6) { + u->field_data[UF_HOST].off = p - buf; + } + u->field_data[UF_HOST].len++; + break; + + case s_http_host_port: + if (s != s_http_host_port) { + u->field_data[UF_PORT].off = p - buf; + u->field_data[UF_PORT].len = 0; + u->field_set |= (1 << UF_PORT); + } + u->field_data[UF_PORT].len++; + break; + + case s_http_userinfo: + if (s != s_http_userinfo) { + u->field_data[UF_USERINFO].off = p - buf ; + u->field_data[UF_USERINFO].len = 0; + u->field_set |= (1 << UF_USERINFO); + } + u->field_data[UF_USERINFO].len++; + break; + + default: + break; + } + s = new_s; + } + + /* Make sure we don't end somewhere unexpected */ + switch (s) { + case s_http_host_start: + case s_http_host_v6_start: + case s_http_host_v6: + case s_http_host_port_start: + case s_http_userinfo: + case s_http_userinfo_start: + return 1; + default: + break; + } + + return 0; +} + + +int +http_parser_parse_url(const char *buf, size_t buflen, int is_connect, + struct http_parser_url *u) +{ + enum state s; + const char *p; + enum http_parser_url_fields uf, old_uf; + int found_at = 0; + + u->port = u->field_set = 0; + s = is_connect ? s_req_server_start : s_req_spaces_before_url; + uf = old_uf = UF_MAX; + + for (p = buf; p < buf + buflen; p++) { + s = parse_url_char(s, *p); + + /* Figure out the next field that we're operating on */ + switch (s) { + case s_dead: + return 1; + + /* Skip delimeters */ + case s_req_schema_slash: + case s_req_schema_slash_slash: + case s_req_server_start: + case s_req_query_string_start: + case s_req_fragment_start: + continue; + + case s_req_schema: + uf = UF_SCHEMA; + break; + + case s_req_server_with_at: + found_at = 1; + + /* FALLTROUGH */ + case s_req_server: + uf = UF_HOST; + break; + + case s_req_path: + uf = UF_PATH; + break; + + case s_req_query_string: + uf = UF_QUERY; + break; + + case s_req_fragment: + uf = UF_FRAGMENT; + break; + + default: + assert(!"Unexpected state"); + return 1; + } + + /* Nothing's changed; soldier on */ + if (uf == old_uf) { + u->field_data[uf].len++; + continue; + } + + u->field_data[uf].off = p - buf; + u->field_data[uf].len = 1; + + u->field_set |= (1 << uf); + old_uf = uf; + } + + /* host must be present if there is a schema */ + /* parsing http:///toto will fail */ + if ((u->field_set & ((1 << UF_SCHEMA) | (1 << UF_HOST))) != 0) { + if (http_parse_host(buf, u, found_at) != 0) { + return 1; + } + } + + /* CONNECT requests can only contain "hostname:port" */ + if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) { + return 1; + } + + if (u->field_set & (1 << UF_PORT)) { + /* Don't bother with endp; we've already validated the string */ + unsigned long v = strtoul(buf + u->field_data[UF_PORT].off, nullptr, 10); + + /* Ports have a max value of 2^16 */ + if (v > 0xffff) { + return 1; + } + + u->port = (uint16_t) v; + } + + return 0; +} + +void +http_parser_pause(http_parser *parser, int paused) { + /* Users should only be pausing/unpausing a parser that is not in an error + * state. In non-debug builds, there's not much that we can do about this + * other than ignore it. + */ + if (HTTP_PARSER_ERRNO(parser) == HPE_OK || + HTTP_PARSER_ERRNO(parser) == HPE_PAUSED) { + SET_ERRNO((paused) ? HPE_PAUSED : HPE_OK); + } else { + assert(0 && "Attempting to pause parser in error state"); + } +} + +#if __cplusplus +} +#endif /* __cplusplus */ diff --git a/proxygen/external/http_parser/test.c b/proxygen/external/http_parser/test.c new file mode 100644 index 0000000000..034f1dc27d --- /dev/null +++ b/proxygen/external/http_parser/test.c @@ -0,0 +1,3142 @@ +/* Copyright Joyent, Inc. and other Node contributors. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +#include "http_parser.h" +#include +#include +#include +#include /* rand */ +#include +#include + +#undef TRUE +#define TRUE 1 +#undef FALSE +#define FALSE 0 + +#define MAX_HEADERS 13 +#define MAX_ELEMENT_SIZE 500 +#define MAX_CHUNKS 16 + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) + +static http_parser *parser; + +struct message { + const char *name; // for debugging purposes + const char *raw; + enum http_parser_type type; + enum http_method method; + int status_code; + char request_url[MAX_ELEMENT_SIZE]; + char response_reason[MAX_ELEMENT_SIZE]; + size_t response_reason_size; + char body[MAX_ELEMENT_SIZE]; + size_t body_size; + int num_headers; + enum { NONE=0, FIELD, VALUE } last_header_element; + char headers [MAX_HEADERS][2][MAX_ELEMENT_SIZE]; + int should_keep_alive; + + int num_chunks; + int num_chunks_complete; + int chunk_lengths[MAX_CHUNKS]; + + const char *upgrade; // upgraded body + + unsigned short http_major; + unsigned short http_minor; + + int message_begin_cb_called; + int headers_complete_cb_called; + int message_complete_cb_called; + int message_complete_on_eof; +}; + +static int currently_parsing_eof; + +static struct message messages[5]; +static int num_messages; +static http_parser_settings *current_pause_parser; + +/* * R E Q U E S T S * */ +const struct message requests[] = +#define CURL_GET 0 +{ {.name= "curl get" + ,.type= HTTP_REQUEST + ,.raw= "GET /test HTTP/1.1\r\n" + "User-Agent: curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1\r\n" + "Host: 0.0.0.0=5000\r\n" + "Accept: */*\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/test" + ,.num_headers= 3 + ,.headers= + { { "User-Agent", "curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1" } + , { "Host", "0.0.0.0=5000" } + , { "Accept", "*/*" } + } + ,.body= "" + } + +#define FIREFOX_GET 1 +, {.name= "firefox get" + ,.type= HTTP_REQUEST + ,.raw= "GET /favicon.ico HTTP/1.1\r\n" + "Host: 0.0.0.0=5000\r\n" + "User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + "Accept-Language: en-us,en;q=0.5\r\n" + "Accept-Encoding: gzip,deflate\r\n" + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + "Keep-Alive: 300\r\n" + "Connection: keep-alive\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/favicon.ico" + ,.num_headers= 8 + ,.headers= + { { "Host", "0.0.0.0=5000" } + , { "User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0" } + , { "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" } + , { "Accept-Language", "en-us,en;q=0.5" } + , { "Accept-Encoding", "gzip,deflate" } + , { "Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7" } + , { "Keep-Alive", "300" } + , { "Connection", "keep-alive" } + } + ,.body= "" + } + +#define DUMBFUCK 2 +, {.name= "dumbfuck" + ,.type= HTTP_REQUEST + ,.raw= "GET /dumbfuck HTTP/1.1\r\n" + "aaaaaaaaaaaaa:++++++++++\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/dumbfuck" + ,.num_headers= 1 + ,.headers= + { { "aaaaaaaaaaaaa", "++++++++++" } + } + ,.body= "" + } + +#define FRAGMENT_IN_URI 3 +, {.name= "fragment in url" + ,.type= HTTP_REQUEST + ,.raw= "GET /forums/1/topics/2375?page=1#posts-17408 HTTP/1.1\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + /* XXX request url does include fragment? */ + ,.request_url= "/forums/1/topics/2375?page=1#posts-17408" + ,.num_headers= 0 + ,.body= "" + } + +#define GET_NO_HEADERS_NO_BODY 4 +, {.name= "get no headers no body" + ,.type= HTTP_REQUEST + ,.raw= "GET /get_no_headers_no_body/world HTTP/1.1\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE /* would need Connection: close */ + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/get_no_headers_no_body/world" + ,.num_headers= 0 + ,.body= "" + } + +#define GET_ONE_HEADER_NO_BODY 5 +, {.name= "get one header no body" + ,.type= HTTP_REQUEST + ,.raw= "GET /get_one_header_no_body HTTP/1.1\r\n" + "Accept: */*\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE /* would need Connection: close */ + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/get_one_header_no_body" + ,.num_headers= 1 + ,.headers= + { { "Accept" , "*/*" } + } + ,.body= "" + } + +#define GET_FUNKY_CONTENT_LENGTH 6 +, {.name= "get funky content length body hello" + ,.type= HTTP_REQUEST + ,.raw= "GET /get_funky_content_length_body_hello HTTP/1.0\r\n" + "conTENT-Length: 5\r\n" + "\r\n" + "HELLO" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.method= HTTP_GET + ,.request_url= "/get_funky_content_length_body_hello" + ,.num_headers= 1 + ,.headers= + { { "conTENT-Length" , "5" } + } + ,.body= "HELLO" + } + +#define POST_IDENTITY_BODY_WORLD 7 +, {.name= "post identity body world" + ,.type= HTTP_REQUEST + ,.raw= "POST /post_identity_body_world?q=search#hey HTTP/1.1\r\n" + "Accept: */*\r\n" + "Transfer-Encoding: identity\r\n" + "Content-Length: 5\r\n" + "\r\n" + "World" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.request_url= "/post_identity_body_world?q=search#hey" + ,.num_headers= 3 + ,.headers= + { { "Accept", "*/*" } + , { "Transfer-Encoding", "identity" } + , { "Content-Length", "5" } + } + ,.body= "World" + } + +#define POST_CHUNKED_ALL_YOUR_BASE 8 +, {.name= "post - chunked body: all your base are belong to us" + ,.type= HTTP_REQUEST + ,.raw= "POST /post_chunked_all_your_base HTTP/1.1\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "1e\r\nall your base are belong to us\r\n" + "0\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.request_url= "/post_chunked_all_your_base" + ,.num_headers= 1 + ,.headers= + { { "Transfer-Encoding" , "chunked" } + } + ,.body= "all your base are belong to us" + ,.num_chunks= 1 + ,.num_chunks_complete= 2 + ,.chunk_lengths= { 0x1e } + } + +#define TWO_CHUNKS_MULT_ZERO_END 9 +, {.name= "two chunks ; triple zero ending" + ,.type= HTTP_REQUEST + ,.raw= "POST /two_chunks_mult_zero_end HTTP/1.1\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5\r\nhello\r\n" + "6\r\n world\r\n" + "000\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.request_url= "/two_chunks_mult_zero_end" + ,.num_headers= 1 + ,.headers= + { { "Transfer-Encoding", "chunked" } + } + ,.body= "hello world" + ,.num_chunks= 2 + ,.num_chunks_complete= 3 + ,.chunk_lengths= { 5, 6 } + } + +#define CHUNKED_W_TRAILING_HEADERS 10 +, {.name= "chunked with trailing headers. blech." + ,.type= HTTP_REQUEST + ,.raw= "POST /chunked_w_trailing_headers HTTP/1.1\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5\r\nhello\r\n" + "6\r\n world\r\n" + "0\r\n" + "Vary: *\r\n" + "Content-Type: text/plain\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.request_url= "/chunked_w_trailing_headers" + ,.num_headers= 3 + ,.headers= + { { "Transfer-Encoding", "chunked" } + , { "Vary", "*" } + , { "Content-Type", "text/plain" } + } + ,.body= "hello world" + ,.num_chunks= 2 + ,.num_chunks_complete= 3 + ,.chunk_lengths= { 5, 6 } + } + +#define CHUNKED_W_BULLSHIT_AFTER_LENGTH 11 +, {.name= "with bullshit after the length" + ,.type= HTTP_REQUEST + ,.raw= "POST /chunked_w_bullshit_after_length HTTP/1.1\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5; ihatew3;whatthefuck=aretheseparametersfor\r\nhello\r\n" + "6; blahblah; blah\r\n world\r\n" + "0\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.request_url= "/chunked_w_bullshit_after_length" + ,.num_headers= 1 + ,.headers= + { { "Transfer-Encoding", "chunked" } + } + ,.body= "hello world" + ,.num_chunks= 2 + ,.num_chunks_complete= 3 + ,.chunk_lengths= { 5, 6 } + } + +#define WITH_QUOTES 12 +, {.name= "with quotes" + ,.type= HTTP_REQUEST + ,.raw= "GET /with_\"stupid\"_quotes?foo=\"bar\" HTTP/1.1\r\n\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/with_\"stupid\"_quotes?foo=\"bar\"" + ,.num_headers= 0 + ,.headers= { } + ,.body= "" + } + +#define APACHEBENCH_GET 13 +/* The server receiving this request SHOULD NOT wait for EOF + * to know that content-length == 0. + * How to represent this in a unit test? message_complete_on_eof + * Compare with NO_CONTENT_LENGTH_RESPONSE. + */ +, {.name = "apachebench get" + ,.type= HTTP_REQUEST + ,.raw= "GET /test HTTP/1.0\r\n" + "Host: 0.0.0.0:5000\r\n" + "User-Agent: ApacheBench/2.3\r\n" + "Accept: */*\r\n\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.method= HTTP_GET + ,.request_url= "/test" + ,.num_headers= 3 + ,.headers= { { "Host", "0.0.0.0:5000" } + , { "User-Agent", "ApacheBench/2.3" } + , { "Accept", "*/*" } + } + ,.body= "" + } + +#define QUERY_URL_WITH_QUESTION_MARK_GET 14 +/* Some clients include '?' characters in query strings. + */ +, {.name = "query url with question mark" + ,.type= HTTP_REQUEST + ,.raw= "GET /test.cgi?foo=bar?baz HTTP/1.1\r\n\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/test.cgi?foo=bar?baz" + ,.num_headers= 0 + ,.headers= {} + ,.body= "" + } + +#define PREFIX_NEWLINE_GET 15 +/* Some clients, especially after a POST in a keep-alive connection, + * will send an extra CRLF before the next request + */ +, {.name = "newline prefix get" + ,.type= HTTP_REQUEST + ,.raw= "\r\nGET /test HTTP/1.1\r\n\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/test" + ,.num_headers= 0 + ,.headers= { } + ,.body= "" + } + +#define UPGRADE_REQUEST 16 +, {.name = "upgrade request" + ,.type= HTTP_REQUEST + ,.raw= "GET /demo HTTP/1.1\r\n" + "Host: example.com\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Key2: 12998 5 Y3 1 .P00\r\n" + "Sec-WebSocket-Protocol: sample\r\n" + "Upgrade: WebSocket\r\n" + "Sec-WebSocket-Key1: 4 @1 46546xW%0l 1 5\r\n" + "Origin: http://example.com\r\n" + "\r\n" + "Hot diggity dogg" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/demo" + ,.num_headers= 7 + ,.upgrade="Hot diggity dogg" + ,.headers= { { "Host", "example.com" } + , { "Connection", "Upgrade" } + , { "Sec-WebSocket-Key2", "12998 5 Y3 1 .P00" } + , { "Sec-WebSocket-Protocol", "sample" } + , { "Upgrade", "WebSocket" } + , { "Sec-WebSocket-Key1", "4 @1 46546xW%0l 1 5" } + , { "Origin", "http://example.com" } + } + ,.body= "" + } + +#define CONNECT_REQUEST 17 +, {.name = "connect request" + ,.type= HTTP_REQUEST + ,.raw= "CONNECT 0-home0.netscape.com:443 HTTP/1.0\r\n" + "User-agent: Mozilla/1.1N\r\n" + "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n" + "\r\n" + "some data\r\n" + "and yet even more data" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.method= HTTP_CONNECT + ,.request_url= "0-home0.netscape.com:443" + ,.num_headers= 2 + ,.upgrade="some data\r\nand yet even more data" + ,.headers= { { "User-agent", "Mozilla/1.1N" } + , { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" } + } + ,.body= "" + } + +#define REPORT_REQ 18 +, {.name= "report request" + ,.type= HTTP_REQUEST + ,.raw= "REPORT /test HTTP/1.1\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_REPORT + ,.request_url= "/test" + ,.num_headers= 0 + ,.headers= {} + ,.body= "" + } + +#define NO_HTTP_VERSION 19 +, {.name= "request with no http version" + ,.type= HTTP_REQUEST + ,.raw= "GET /\r\n" + "\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 0 + ,.http_minor= 9 + ,.method= HTTP_GET + ,.request_url= "/" + ,.num_headers= 0 + ,.headers= {} + ,.body= "" + } + +#define MSEARCH_REQ 20 +, {.name= "m-search request" + ,.type= HTTP_REQUEST + ,.raw= "M-SEARCH * HTTP/1.1\r\n" + "HOST: 239.255.255.250:1900\r\n" + "MAN: \"ssdp:discover\"\r\n" + "ST: \"ssdp:all\"\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_MSEARCH + ,.request_url= "*" + ,.num_headers= 3 + ,.headers= { { "HOST", "239.255.255.250:1900" } + , { "MAN", "\"ssdp:discover\"" } + , { "ST", "\"ssdp:all\"" } + } + ,.body= "" + } + +#define LINE_FOLDING_IN_HEADER 20 +, {.name= "line folding in header value" + ,.type= HTTP_REQUEST + ,.raw= "GET / HTTP/1.1\r\n" + "Line1: abc\r\n" + "\tdef\r\n" + " ghi\r\n" + "\t\tjkl\r\n" + " mno \r\n" + "\t \tqrs\r\n" + "Line2: \t line2\t\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/" + ,.num_headers= 2 + ,.headers= { { "Line1", "abc def ghi jkl mno qrs" } + , { "Line2", "line2\t" } + } + ,.body= "" + } + + +#define QUERY_TERMINATED_HOST 21 +, {.name= "host terminated by a query string" + ,.type= HTTP_REQUEST + ,.raw= "GET http://hypnotoad.org?hail=all HTTP/1.1\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "http://hypnotoad.org?hail=all" + ,.num_headers= 0 + ,.headers= { } + ,.body= "" + } + +#define QUERY_TERMINATED_HOSTPORT 22 +, {.name= "host:port terminated by a query string" + ,.type= HTTP_REQUEST + ,.raw= "GET http://hypnotoad.org:1234?hail=all HTTP/1.1\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "http://hypnotoad.org:1234?hail=all" + ,.num_headers= 0 + ,.headers= { } + ,.body= "" + } + +#define SPACE_TERMINATED_HOSTPORT 23 +, {.name= "host:port terminated by a space" + ,.type= HTTP_REQUEST + ,.raw= "GET http://hypnotoad.org:1234 HTTP/1.1\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "http://hypnotoad.org:1234" + ,.num_headers= 0 + ,.headers= { } + ,.body= "" + } + +#if !HTTP_PARSER_STRICT +#define UTF8_PATH_REQ 24 +, {.name= "utf-8 path request" + ,.type= HTTP_REQUEST + ,.raw= "GET /δ¶/δt/pope?q=1#narf HTTP/1.1\r\n" + "Host: github.com\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_GET + ,.request_url= "/δ¶/δt/pope?q=1#narf" + ,.num_headers= 1 + ,.headers= { {"Host", "github.com" } + } + ,.body= "" + } + +#define HOSTNAME_UNDERSCORE 25 +, {.name = "hostname underscore" + ,.type= HTTP_REQUEST + ,.raw= "CONNECT home_0.netscape.com:443 HTTP/1.0\r\n" + "User-agent: Mozilla/1.1N\r\n" + "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n" + "\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.method= HTTP_CONNECT + ,.request_url= "home_0.netscape.com:443" + ,.num_headers= 2 + ,.upgrade="" + ,.headers= { { "User-agent", "Mozilla/1.1N" } + , { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" } + } + ,.body= "" + } +#endif /* !HTTP_PARSER_STRICT */ + +#define PATCH_REQ 26 +, {.name = "PATCH request" + ,.type= HTTP_REQUEST + ,.raw= "PATCH /file.txt HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Content-Type: application/example\r\n" + "If-Match: \"e0023aa4e\"\r\n" + "Content-Length: 10\r\n" + "\r\n" + "cccccccccc" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_PATCH + ,.request_url= "/file.txt" + ,.num_headers= 4 + ,.headers= { { "Host", "www.example.com" } + , { "Content-Type", "application/example" } + , { "If-Match", "\"e0023aa4e\"" } + , { "Content-Length", "10" } + } + ,.body= "cccccccccc" + } + +#define CONNECT_CAPS_REQUEST 27 +, {.name = "connect caps request" + ,.type= HTTP_REQUEST + ,.raw= "CONNECT HOME0.NETSCAPE.COM:443 HTTP/1.0\r\n" + "User-agent: Mozilla/1.1N\r\n" + "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n" + "\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.method= HTTP_CONNECT + ,.request_url= "HOME0.NETSCAPE.COM:443" + ,.num_headers= 2 + ,.upgrade="" + ,.headers= { { "User-agent", "Mozilla/1.1N" } + , { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" } + } + ,.body= "" + } + +#define IPV6_LITERAL_URI_GET 28 +, {.name = "IPv6 literal URI GET" + ,.type= HTTP_REQUEST + ,.raw= "GET http://[2a03:2880:2050:1f01:face:b00c:0:9]/ HTTP/1.0\r\n" + "\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.method= HTTP_GET + ,.request_url= "http://[2a03:2880:2050:1f01:face:b00c:0:9]/" + ,.num_headers= 0 + ,.headers= {} + ,.body= "" + } + +#define IPV6_LITERAL_URI_CONNECT 29 +, {.name = "IPv6 literal URI CONNECT" + ,.type= HTTP_REQUEST + ,.raw= "CONNECT [2a03:2880:2050:1f01:face:b00c:0:9]:443 HTTP/1.0\r\n" + "\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.method= HTTP_CONNECT + ,.request_url= "[2a03:2880:2050:1f01:face:b00c:0:9]:443" + ,.num_headers= 0 + ,.upgrade="" + ,.headers= {} + ,.body= "" + } + +#define UPGRADE_POST_REQUEST 30 +, {.name = "upgrade post request" + ,.type= HTTP_REQUEST + ,.raw= "POST /demo HTTP/1.1\r\n" + "Host: example.com\r\n" + "Connection: Upgrade\r\n" + "Upgrade: HTTP/2.0\r\n" + "Content-Length: 15\r\n" + "\r\n" + "sweet post body" + "Hot diggity dogg" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.method= HTTP_POST + ,.request_url= "/demo" + ,.num_headers= 4 + ,.upgrade="Hot diggity dogg" + ,.headers= { { "Host", "example.com" } + , { "Connection", "Upgrade" } + , { "Upgrade", "HTTP/2.0" } + , { "Content-Length", "15" } + } + ,.body= "sweet post body" + } + +#define CONNECT_WITH_BODY_REQUEST 31 +, {.name = "connect with body request" + ,.type= HTTP_REQUEST + ,.raw= "CONNECT foo.bar.com:443 HTTP/1.0\r\n" + "User-agent: Mozilla/1.1N\r\n" + "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n" + "Content-Length: 10\r\n" + "\r\n" + "blarfcicle" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.method= HTTP_CONNECT + ,.request_url= "foo.bar.com:443" + ,.num_headers= 3 + ,.upgrade="blarfcicle" + ,.headers= { { "User-agent", "Mozilla/1.1N" } + , { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" } + , { "Content-Length", "10" } + } + ,.body= "" + } + +, {.name= NULL } /* sentinel */ +}; + +/* * R E S P O N S E S * */ +const struct message responses[] = +#define GOOGLE_301 0 +{ {.name= "google 301" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 301 Moved Permanently\r\n" + "Location: http://www.google.com/\r\n" + "Content-Type: text/html; charset=UTF-8\r\n" + "Date: Sun, 26 Apr 2009 11:11:49 GMT\r\n" + "Expires: Tue, 26 May 2009 11:11:49 GMT\r\n" + "X-$PrototypeBI-Version: 1.6.0.3\r\n" /* $ char in header field */ + "Cache-Control: public, max-age=2592000\r\n" + "Server: gws\r\n" + "Content-Length: 219 \r\n" + "\r\n" + "\n" + "301 Moved\n" + "

301 Moved

\n" + "The document has moved\n" + "here.\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 301 + ,.response_reason= "Moved Permanently" + ,.num_headers= 8 + ,.headers= + { { "Location", "http://www.google.com/" } + , { "Content-Type", "text/html; charset=UTF-8" } + , { "Date", "Sun, 26 Apr 2009 11:11:49 GMT" } + , { "Expires", "Tue, 26 May 2009 11:11:49 GMT" } + , { "X-$PrototypeBI-Version", "1.6.0.3" } + , { "Cache-Control", "public, max-age=2592000" } + , { "Server", "gws" } + , { "Content-Length", "219 " } + } + ,.body= "\n" + "301 Moved\n" + "

301 Moved

\n" + "The document has moved\n" + "here.\r\n" + "\r\n" + } + +#define NO_CONTENT_LENGTH_RESPONSE 1 +/* The client should wait for the server's EOF. That is, when content-length + * is not specified, and "Connection: close", the end of body is specified + * by the EOF. + * Compare with APACHEBENCH_GET + */ +, {.name= "no content-length response" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + "Date: Tue, 04 Aug 2009 07:59:32 GMT\r\n" + "Server: Apache\r\n" + "X-Powered-By: Servlet/2.5 JSP/2.1\r\n" + "Content-Type: text/xml; charset=utf-8\r\n" + "Connection: close\r\n" + "\r\n" + "\n" + "\n" + " \n" + " \n" + " SOAP-ENV:Client\n" + " Client Error\n" + " \n" + " \n" + "" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= TRUE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_reason= "OK" + ,.num_headers= 5 + ,.headers= + { { "Date", "Tue, 04 Aug 2009 07:59:32 GMT" } + , { "Server", "Apache" } + , { "X-Powered-By", "Servlet/2.5 JSP/2.1" } + , { "Content-Type", "text/xml; charset=utf-8" } + , { "Connection", "close" } + } + ,.body= "\n" + "\n" + " \n" + " \n" + " SOAP-ENV:Client\n" + " Client Error\n" + " \n" + " \n" + "" + } + +#define NO_HEADERS_NO_BODY_404 2 +, {.name= "404 no headers no body" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 404 Not Found\r\n\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= TRUE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 404 + ,.response_reason= "Not Found" + ,.num_headers= 0 ,.headers= {} + ,.body_size= 0 + ,.body= "" + } + +#define NO_REASON_PHRASE 3 +, {.name= "304 no response phrase" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 304\r\n\r\n" + ,.should_keep_alive = TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 304 + ,.response_reason= "" + ,.num_headers= 0 + ,.headers= {} + ,.body= "" + } + +#define TRAILING_SPACE_ON_CHUNKED_BODY 4 +, {.name="200 trailing space on chunked body" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + "Content-Type: text/plain\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "25 \r\n" + "This is the data in the first chunk\r\n" + "\r\n" + "1C\r\n" + "and this is the second one\r\n" + "\r\n" + "0 \r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_reason= "OK" + ,.num_headers= 2 + ,.headers= + { {"Content-Type", "text/plain" } + , {"Transfer-Encoding", "chunked" } + } + ,.body_size = 37+28 + ,.body = + "This is the data in the first chunk\r\n" + "and this is the second one\r\n" + ,.num_chunks= 2 + ,.num_chunks_complete= 3 + ,.chunk_lengths= { 0x25, 0x1c } + + } + +#define NO_CARRIAGE_RET 5 +, {.name="no carriage ret" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\n" + "Content-Type: text/html; charset=utf-8\n" + "Connection: close\n" + "\n" + "these headers are from http://news.ycombinator.com/" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= TRUE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_reason= "OK" + ,.num_headers= 2 + ,.headers= + { {"Content-Type", "text/html; charset=utf-8" } + , {"Connection", "close" } + } + ,.body= "these headers are from http://news.ycombinator.com/" + } + +#define PROXY_CONNECTION 6 +, {.name="proxy connection" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html; charset=UTF-8\r\n" + "Content-Length: 11\r\n" + "Proxy-Connection: close\r\n" + "Date: Thu, 31 Dec 2009 20:55:48 +0000\r\n" + "\r\n" + "hello world" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_reason= "OK" + ,.num_headers= 4 + ,.headers= + { {"Content-Type", "text/html; charset=UTF-8" } + , {"Content-Length", "11" } + , {"Proxy-Connection", "close" } + , {"Date", "Thu, 31 Dec 2009 20:55:48 +0000"} + } + ,.body= "hello world" + } + +#define UNDERSTORE_HEADER_KEY 7 + // shown by + // curl -o /dev/null -v "http://ad.doubleclick.net/pfadx/DARTSHELLCONFIGXML;dcmt=text/xml;" +, {.name="underscore header key" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + "Server: DCLK-AdSvr\r\n" + "Content-Type: text/xml\r\n" + "Content-Length: 0\r\n" + "DCLK_imp: v7;x;114750856;0-0;0;17820020;0/0;21603567/21621457/1;;~okv=;dcmt=text/xml;;~cs=o\r\n\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_reason= "OK" + ,.num_headers= 4 + ,.headers= + { {"Server", "DCLK-AdSvr" } + , {"Content-Type", "text/xml" } + , {"Content-Length", "0" } + , {"DCLK_imp", "v7;x;114750856;0-0;0;17820020;0/0;21603567/21621457/1;;~okv=;dcmt=text/xml;;~cs=o" } + } + ,.body= "" + } + +#define BONJOUR_MADAME_FR 8 +/* The client should not merge two headers fields when the first one doesn't + * have a value. + */ +, {.name= "bonjourmadame.fr" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.0 301 Moved Permanently\r\n" + "Date: Thu, 03 Jun 2010 09:56:32 GMT\r\n" + "Server: Apache/2.2.3 (Red Hat)\r\n" + "Cache-Control: public\r\n" + "Pragma: \r\n" + "Location: http://www.bonjourmadame.fr/\r\n" + "Vary: Accept-Encoding\r\n" + "Content-Length: 0\r\n" + "Content-Type: text/html; charset=UTF-8\r\n" + "Connection: keep-alive\r\n" + "\r\n" + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.status_code= 301 + ,.response_reason= "Moved Permanently" + ,.num_headers= 9 + ,.headers= + { { "Date", "Thu, 03 Jun 2010 09:56:32 GMT" } + , { "Server", "Apache/2.2.3 (Red Hat)" } + , { "Cache-Control", "public" } + , { "Pragma", "" } + , { "Location", "http://www.bonjourmadame.fr/" } + , { "Vary", "Accept-Encoding" } + , { "Content-Length", "0" } + , { "Content-Type", "text/html; charset=UTF-8" } + , { "Connection", "keep-alive" } + } + ,.body= "" + } + +#define SPACE_IN_FIELD_RES 9 +/* Should handle spaces in header fields */ +, {.name= "field space" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + "Server: Microsoft-IIS/6.0\r\n" + "X-Powered-By: ASP.NET\r\n" + "en-US Content-Type: text/xml\r\n" /* this is the problem */ + "Content-Type: text/xml\r\n" + "Content-Length: 16\r\n" + "Date: Fri, 23 Jul 2010 18:45:38 GMT\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "hello" /* fake body */ + ,.should_keep_alive= TRUE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_reason= "OK" + ,.num_headers= 7 + ,.headers= + { { "Server", "Microsoft-IIS/6.0" } + , { "X-Powered-By", "ASP.NET" } + , { "en-US Content-Type", "text/xml" } + , { "Content-Type", "text/xml" } + , { "Content-Length", "16" } + , { "Date", "Fri, 23 Jul 2010 18:45:38 GMT" } + , { "Connection", "keep-alive" } + } + ,.body= "hello" + } + + +#define RES_FIELD_UNDERSCORE 10 +/* Should handle spaces in header fields */ +, {.name= "field underscore" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + "Date: Tue, 28 Sep 2010 01:14:13 GMT\r\n" + "Server: Apache\r\n" + "Cache-Control: no-cache, must-revalidate\r\n" + "Expires: Mon, 26 Jul 1997 05:00:00 GMT\r\n" + ".et-Cookie: PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com\r\n" + "Vary: Accept-Encoding\r\n" + "_eep-Alive: timeout=45\r\n" /* semantic value ignored */ + "_onnection: Keep-Alive\r\n" /* semantic value ignored */ + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/html\r\n" + "Connection: close\r\n" + "\r\n" + "0\r\n\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 200 + ,.response_reason= "OK" + ,.num_headers= 11 + ,.headers= + { { "Date", "Tue, 28 Sep 2010 01:14:13 GMT" } + , { "Server", "Apache" } + , { "Cache-Control", "no-cache, must-revalidate" } + , { "Expires", "Mon, 26 Jul 1997 05:00:00 GMT" } + , { ".et-Cookie", "PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com" } + , { "Vary", "Accept-Encoding" } + , { "_eep-Alive", "timeout=45" } + , { "_onnection", "Keep-Alive" } + , { "Transfer-Encoding", "chunked" } + , { "Content-Type", "text/html" } + , { "Connection", "close" } + } + ,.body= "" + ,.num_chunks= 0 + ,.num_chunks_complete= 1 + ,.chunk_lengths= {} + } + +#define NON_ASCII_IN_STATUS_LINE 11 +/* Should handle non-ASCII in status line */ +, {.name= "non-ASCII in status line" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 500 Oriëntatieprobleem\r\n" + "Date: Fri, 5 Nov 2010 23:07:12 GMT+2\r\n" + "Content-Length: 0\r\n" + "Connection: close\r\n" + "\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 1 + ,.status_code= 500 + ,.response_reason= "Oriëntatieprobleem" + ,.num_headers= 3 + ,.headers= + { { "Date", "Fri, 5 Nov 2010 23:07:12 GMT+2" } + , { "Content-Length", "0" } + , { "Connection", "close" } + } + ,.body= "" + } + + +, {.name= NULL } /* sentinel */ +}; + +/* Test for execution of on_message_callback regardless of the request/response + * correctness */ +/* Invalid response */ +const struct message on_message_begin_cb_test[] = +{ {.name= "Invalid response" + ,.type= HTTP_RESPONSE + ,.raw= "ZZZZZ\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.body= "" + }, + +/* Invalid request */ + {.name= "Invalid request" + ,.type= HTTP_REQUEST + ,.raw= "ZZZZZ\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.body= "" + }, + +/* Invalid unspecified request/response */ + {.name= "Invalid 'both' 'request/response'" + ,.type= HTTP_BOTH + ,.raw= "ZZZZZ\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.body= "" + }, + +/* Valid 200 response */ + {.name= "Simple valid 200 reponse" + ,.type= HTTP_RESPONSE + ,.raw= "HTTP/1.1 200 OK\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.body= "" + }, + +/* Valid request */ + {.name= "Simple valid request" + ,.type= HTTP_REQUEST + ,.raw= "GET / HTTP/1.1\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.body= "" + }, + +/* Valid unspecified request */ + {.name= "Simple valid 'both' request" + ,.type= HTTP_BOTH + ,.raw= "GET / HTTP/1.1\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.body= "" + }, + +/* Valid unspecified response */ + {.name= "Simple valid 'both' response" + ,.type= HTTP_BOTH + ,.raw= "GET / HTTP/1.1\r\n" + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.body= "" + } +, {.name= NULL } /* sentinel */ +}; + +int +request_url_cb (http_parser *p, const char *buf, size_t len) +{ + assert(p == parser); + strncat(messages[num_messages].request_url, buf, len); + return 0; +} + +int +header_field_cb (http_parser *p, const char *buf, size_t len) +{ + assert(p == parser); + struct message *m = &messages[num_messages]; + + if (m->last_header_element != FIELD) + m->num_headers++; + + strncat(m->headers[m->num_headers-1][0], buf, len); + + m->last_header_element = FIELD; + + return 0; +} + +int +header_value_cb (http_parser *p, const char *buf, size_t len) +{ + assert(p == parser); + struct message *m = &messages[num_messages]; + + const size_t cursize = strlen(m->headers[m->num_headers-1][1]); + if (cursize + len + 1 > MAX_ELEMENT_SIZE) { + len = MAX_ELEMENT_SIZE - (cursize + 1); + } + + strncat(m->headers[m->num_headers-1][1], buf, len); + + m->last_header_element = VALUE; + + return 0; +} + +int +body_cb (http_parser *p, const char *buf, size_t len) +{ + assert(p == parser); + strncat(messages[num_messages].body, buf, len); + messages[num_messages].body_size += len; + // printf("body_cb: '%s'\n", requests[num_messages].body); + return 0; +} + +int +count_body_cb (http_parser *p, const char *buf, size_t len) +{ + assert(p == parser); + assert(buf); + messages[num_messages].body_size += len; + return 0; +} + +int +message_begin_cb (http_parser *p) +{ + assert(p == parser); + messages[num_messages].message_begin_cb_called++; + return 0; +} + +int +headers_complete_cb (http_parser *p, const char *buf, size_t len) +{ + assert(p == parser); + assert(!buf); + messages[num_messages].method = parser->method; + messages[num_messages].status_code = parser->status_code; + messages[num_messages].http_major = parser->http_major; + messages[num_messages].http_minor = parser->http_minor; + messages[num_messages].headers_complete_cb_called = TRUE; + return 0; +} + +int +message_complete_cb (http_parser *p) +{ + assert(p == parser); + messages[num_messages].message_complete_cb_called = TRUE; + + messages[num_messages].message_complete_on_eof = currently_parsing_eof; + + num_messages++; + return 0; +} + +int +response_reason_cb (http_parser *p, const char *buf, size_t len) +{ + assert(p == parser); + strncat(messages[num_messages].response_reason, buf, len); + return 0; +} + +int +chunk_header_cb (http_parser *p) +{ + assert(p == parser); + if (p->content_length == 0) { + // Terminating chunk. Don't increment num_chunks in this case. + return 0; + } + + int chunk_idx = messages[num_messages].num_chunks; + ++messages[num_messages].num_chunks; + if (chunk_idx < MAX_CHUNKS) { + messages[num_messages].chunk_lengths[chunk_idx] = p->content_length; + } + + return 0; +} + +int +chunk_complete_cb (http_parser *p) +{ + assert(p == parser); + ++messages[num_messages].num_chunks_complete; + return 0; +} + +/* These dontcall_* callbacks exist so that we can verify that when we're + * paused, no additional callbacks are invoked */ +int +dontcall_message_begin_cb (http_parser *p) +{ + if (p) { } // gcc + fprintf(stderr, "\n\n*** on_message_begin() called on paused parser ***\n\n"); + exit(1); +} + +int +dontcall_header_field_cb (http_parser *p, const char *buf, size_t len) +{ + if (p || buf || len) { } // gcc + fprintf(stderr, "\n\n*** on_header_field() called on paused parser ***\n\n"); + exit(1); +} + +int +dontcall_header_value_cb (http_parser *p, const char *buf, size_t len) +{ + if (p || buf || len) { } // gcc + fprintf(stderr, "\n\n*** on_header_value() called on paused parser ***\n\n"); + exit(1); +} + +int +dontcall_request_url_cb (http_parser *p, const char *buf, size_t len) +{ + if (p || buf || len) { } // gcc + fprintf(stderr, "\n\n*** on_request_url() called on paused parser ***\n\n"); + exit(1); +} + +int +dontcall_body_cb (http_parser *p, const char *buf, size_t len) +{ + if (p || buf || len) { } // gcc + fprintf(stderr, "\n\n*** on_body_cb() called on paused parser ***\n\n"); + exit(1); +} + +int +dontcall_headers_complete_cb (http_parser *p, const char *buf, size_t len) +{ + if (p || buf || len) { } // gcc + fprintf(stderr, "\n\n*** on_headers_complete() called on paused " + "parser ***\n\n"); + exit(1); +} + +int +dontcall_message_complete_cb (http_parser *p) +{ + if (p) { } // gcc + fprintf(stderr, "\n\n*** on_message_complete() called on paused " + "parser ***\n\n"); + exit(1); +} + +int +dontcall_response_reason_cb (http_parser *p, const char *buf, size_t len) +{ + if (p || buf || len) { } // gcc + fprintf(stderr, "\n\n*** on_reason() called on paused parser ***\n\n"); + exit(1); +} + +int +dontcall_chunk_header_cb (http_parser *p) +{ + if (p) { } // gcc + fprintf(stderr, "\n\n*** on_chunk_header() called on paused parser ***\n\n"); + exit(1); +} + +int +dontcall_chunk_complete_cb (http_parser *p) +{ + if (p) { } // gcc + fprintf(stderr, "\n\n*** on_chunk_complete() " + "called on paused parser ***\n\n"); + exit(1); +} + +static http_parser_settings settings_dontcall = + {.on_message_begin = dontcall_message_begin_cb + ,.on_header_field = dontcall_header_field_cb + ,.on_header_value = dontcall_header_value_cb + ,.on_url = dontcall_request_url_cb + ,.on_body = dontcall_body_cb + ,.on_headers_complete = dontcall_headers_complete_cb + ,.on_message_complete = dontcall_message_complete_cb + ,.on_reason = dontcall_response_reason_cb + ,.on_chunk_header = dontcall_chunk_header_cb + ,.on_chunk_complete = dontcall_chunk_complete_cb + }; + +/* These pause_* callbacks always pause the parser and just invoke the regular + * callback that tracks content. Before returning, we overwrite the parser + * settings to point to the _dontcall variety so that we can verify that + * the pause actually did, you know, pause. */ +int +pause_message_begin_cb (http_parser *p) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return message_begin_cb(p); +} + +int +pause_header_field_cb (http_parser *p, const char *buf, size_t len) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return header_field_cb(p, buf, len); +} + +int +pause_header_value_cb (http_parser *p, const char *buf, size_t len) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return header_value_cb(p, buf, len); +} + +int +pause_request_url_cb (http_parser *p, const char *buf, size_t len) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return request_url_cb(p, buf, len); +} + +int +pause_body_cb (http_parser *p, const char *buf, size_t len) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return body_cb(p, buf, len); +} + +int +pause_headers_complete_cb (http_parser *p, const char *buf, size_t len) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return headers_complete_cb(p, buf, len); +} + +int +pause_message_complete_cb (http_parser *p) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return message_complete_cb(p); +} + +int +pause_response_reason_cb (http_parser *p, const char *buf, size_t len) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return response_reason_cb(p, buf, len); +} + +int +pause_chunk_header_cb (http_parser *p) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return chunk_header_cb(p); +} + +int +pause_chunk_complete_cb (http_parser *p) +{ + http_parser_pause(p, 1); + *current_pause_parser = settings_dontcall; + return chunk_complete_cb(p); +} + +int empty_cb (http_parser *p) { return 0; } +int empty_data_cb (http_parser *p, const char *buf, size_t len) { return 0; } + +static http_parser_settings settings_pause = + {.on_message_begin = pause_message_begin_cb + ,.on_header_field = pause_header_field_cb + ,.on_header_value = pause_header_value_cb + ,.on_url = pause_request_url_cb + ,.on_body = pause_body_cb + ,.on_headers_complete = pause_headers_complete_cb + ,.on_message_complete = pause_message_complete_cb + ,.on_reason = pause_response_reason_cb + ,.on_chunk_header = pause_chunk_header_cb + ,.on_chunk_complete = pause_chunk_complete_cb + }; + +static http_parser_settings settings = + {.on_message_begin = message_begin_cb + ,.on_header_field = header_field_cb + ,.on_header_value = header_value_cb + ,.on_url = request_url_cb + ,.on_body = body_cb + ,.on_headers_complete = headers_complete_cb + ,.on_message_complete = message_complete_cb + ,.on_reason = response_reason_cb + ,.on_chunk_header = chunk_header_cb + ,.on_chunk_complete = chunk_complete_cb + }; + +static http_parser_settings settings_count_body = + {.on_message_begin = message_begin_cb + ,.on_header_field = header_field_cb + ,.on_header_value = header_value_cb + ,.on_url = request_url_cb + ,.on_body = count_body_cb + ,.on_headers_complete = headers_complete_cb + ,.on_message_complete = message_complete_cb + ,.on_reason = response_reason_cb + ,.on_chunk_header = chunk_header_cb + ,.on_chunk_complete = chunk_complete_cb + }; + +static http_parser_settings settings_null = + {.on_message_begin = empty_cb + ,.on_header_field = empty_data_cb + ,.on_header_value = empty_data_cb + ,.on_url = empty_data_cb + ,.on_body = empty_data_cb + ,.on_headers_complete = empty_data_cb + ,.on_message_complete = empty_cb + ,.on_reason = empty_data_cb + ,.on_chunk_header = empty_cb + ,.on_chunk_complete = empty_cb + }; + +void +parser_init (enum http_parser_type type) +{ + num_messages = 0; + + assert(parser == NULL); + + parser = malloc(sizeof(http_parser)); + + http_parser_init(parser, type); + + memset(&messages, 0, sizeof messages); + +} + +void +parser_free () +{ + assert(parser); + free(parser); + parser = NULL; +} + +size_t parse (const char *buf, size_t len) +{ + size_t nparsed; + currently_parsing_eof = (len == 0); + nparsed = http_parser_execute(parser, &settings, buf, len); + return nparsed; +} + +size_t parse_count_body (const char *buf, size_t len) +{ + size_t nparsed; + currently_parsing_eof = (len == 0); + nparsed = http_parser_execute(parser, &settings_count_body, buf, len); + return nparsed; +} + +size_t parse_pause (const char *buf, size_t len) +{ + size_t nparsed; + http_parser_settings s = settings_pause; + + currently_parsing_eof = (len == 0); + current_pause_parser = &s; + nparsed = http_parser_execute(parser, current_pause_parser, buf, len); + return nparsed; +} + +static inline int +check_str_eq (const struct message *m, + const char *prop, + const char *expected, + const char *found) { + if ((expected == NULL) != (found == NULL)) { + printf("\n*** Error: %s in '%s' ***\n\n", prop, m->name); + printf("expected %s\n", (expected == NULL) ? "NULL" : expected); + printf(" found %s\n", (found == NULL) ? "NULL" : found); + return 0; + } + if (expected != NULL && 0 != strcmp(expected, found)) { + printf("\n*** Error: %s in '%s' ***\n\n", prop, m->name); + printf("expected '%s'\n", expected); + printf(" found '%s'\n", found); + return 0; + } + return 1; +} + +static inline int +check_num_eq (const struct message *m, + const char *prop, + int expected, + int found) { + if (expected != found) { + printf("\n*** Error: %s in '%s' ***\n\n", prop, m->name); + printf("expected %d\n", expected); + printf(" found %d\n", found); + return 0; + } + return 1; +} + +#define MESSAGE_CHECK_STR_EQ(expected, found, prop) \ + if (!check_str_eq(expected, #prop, expected->prop, found->prop)) return 0 + +#define MESSAGE_CHECK_NUM_EQ(expected, found, prop) \ + if (!check_num_eq(expected, #prop, expected->prop, found->prop)) return 0 + + +int +message_eq (int index, const struct message *expected) +{ + int i; + struct message *m = &messages[index]; + + MESSAGE_CHECK_NUM_EQ(expected, m, http_major); + MESSAGE_CHECK_NUM_EQ(expected, m, http_minor); + + if (expected->type == HTTP_REQUEST) { + MESSAGE_CHECK_NUM_EQ(expected, m, method); + } else { + MESSAGE_CHECK_NUM_EQ(expected, m, status_code); + MESSAGE_CHECK_STR_EQ(expected, m, response_reason); + } + + MESSAGE_CHECK_NUM_EQ(expected, m, message_complete_on_eof); + + assert(m->message_begin_cb_called); + assert(m->headers_complete_cb_called); + assert(m->message_complete_cb_called); + + + MESSAGE_CHECK_STR_EQ(expected, m, request_url); + if (expected->body_size) { + MESSAGE_CHECK_NUM_EQ(expected, m, body_size); + } else { + MESSAGE_CHECK_STR_EQ(expected, m, body); + } + + MESSAGE_CHECK_NUM_EQ(expected, m, num_chunks); + MESSAGE_CHECK_NUM_EQ(expected, m, num_chunks_complete); + for (i = 0; i < m->num_chunks && i < MAX_CHUNKS; i++) { + MESSAGE_CHECK_NUM_EQ(expected, m, chunk_lengths[i]); + } + + MESSAGE_CHECK_NUM_EQ(expected, m, num_headers); + + int r; + for (i = 0; i < m->num_headers; i++) { + r = check_str_eq(expected, "header field", expected->headers[i][0], m->headers[i][0]); + if (!r) return 0; + r = check_str_eq(expected, "header value", expected->headers[i][1], m->headers[i][1]); + if (!r) return 0; + } + + MESSAGE_CHECK_STR_EQ(expected, m, upgrade); + + return 1; +} + +/* Given a sequence of varargs messages, return the number of them that the + * parser should successfully parse, taking into account that upgraded + * messages prevent all subsequent messages from being parsed. + */ +size_t +count_parsed_messages(const size_t nmsgs, ...) { + size_t i; + va_list ap; + + va_start(ap, nmsgs); + + for (i = 0; i < nmsgs; i++) { + struct message *m = va_arg(ap, struct message *); + + if (m->upgrade) { + va_end(ap); + return i + 1; + } + } + + va_end(ap); + return nmsgs; +} + +/* Given a sequence of bytes and the number of these that we were able to + * parse, verify that upgrade bodies are correct. + */ +void +upgrade_message_fix(char *body, const size_t nread, const size_t nmsgs, ...) { + va_list ap; + size_t i; + size_t off = 0; + + va_start(ap, nmsgs); + + for (i = 0; i < nmsgs; i++) { + struct message *m = va_arg(ap, struct message *); + + off += strlen(m->raw); + + if (m->upgrade) { + off -= strlen(m->upgrade); + + /* Check the portion of the response after its specified upgrade */ + if (!check_str_eq(m, "upgrade", body + off, body + nread)) { + exit(1); + } + + /* Fix up the response so that message_eq() will verify the beginning + * of the upgrade */ + *(body + nread + strlen(m->upgrade)) = '\0'; + messages[num_messages -1 ].upgrade = body + nread; + + va_end(ap); + return; + } + } + + va_end(ap); + printf("\n\n*** Error: expected a message with upgrade ***\n"); + + exit(1); +} + +static void +print_error (const char *raw, size_t error_location) +{ + fprintf(stderr, "\n*** %s:%d -- %s ***\n\n", + "http_parser.c", HTTP_PARSER_ERRNO_LINE(parser), + http_errno_description(HTTP_PARSER_ERRNO(parser))); + + int this_line = 0, char_len = 0; + size_t i, j, len = strlen(raw), error_location_line = 0; + for (i = 0; i < len; i++) { + if (i == error_location) this_line = 1; + switch (raw[i]) { + case '\r': + char_len = 2; + fprintf(stderr, "\\r"); + break; + + case '\n': + char_len = 2; + fprintf(stderr, "\\n\n"); + + if (this_line) goto print; + + error_location_line = 0; + continue; + + default: + char_len = 1; + fputc(raw[i], stderr); + break; + } + if (!this_line) error_location_line += char_len; + } + + fprintf(stderr, "[eof]\n"); + + print: + for (j = 0; j < error_location_line; j++) { + fputc(' ', stderr); + } + fprintf(stderr, "^\n\nerror location: %u\n", (unsigned int)error_location); +} + + +struct url_test { + const char *name; + const char *url; + int is_connect; + struct http_parser_url u; + int rv; +}; + +const struct url_test url_tests[] = +{ {.name="proxy request" + ,.url="http://hostname/" + ,.is_connect=0 + ,.u= + {.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PATH) + ,.port=0 + ,.field_data= + {{ 0, 4 } /* UF_SCHEMA */ + ,{ 7, 8 } /* UF_HOST */ + ,{ 0, 0 } /* UF_PORT */ + ,{ 15, 1 } /* UF_PATH */ + ,{ 0, 0 } /* UF_QUERY */ + ,{ 0, 0 } /* UF_FRAGMENT */ + ,{ 0, 0 } /* UF_USERINFO */ + } + } + ,.rv=0 + } + +, {.name="proxy request with port" + ,.url="http://hostname:444/" + ,.is_connect=0 + ,.u= + {.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PORT) | (1 << UF_PATH) + ,.port=444 + ,.field_data= + {{ 0, 4 } /* UF_SCHEMA */ + ,{ 7, 8 } /* UF_HOST */ + ,{ 16, 3 } /* UF_PORT */ + ,{ 19, 1 } /* UF_PATH */ + ,{ 0, 0 } /* UF_QUERY */ + ,{ 0, 0 } /* UF_FRAGMENT */ + ,{ 0, 0 } /* UF_USERINFO */ + } + } + ,.rv=0 + } + +, {.name="CONNECT request" + ,.url="hostname:443" + ,.is_connect=1 + ,.u= + {.field_set=(1 << UF_HOST) | (1 << UF_PORT) + ,.port=443 + ,.field_data= + {{ 0, 0 } /* UF_SCHEMA */ + ,{ 0, 8 } /* UF_HOST */ + ,{ 9, 3 } /* UF_PORT */ + ,{ 0, 0 } /* UF_PATH */ + ,{ 0, 0 } /* UF_QUERY */ + ,{ 0, 0 } /* UF_FRAGMENT */ + ,{ 0, 0 } /* UF_USERINFO */ + } + } + ,.rv=0 + } + +, {.name="CONNECT request but not connect" + ,.url="hostname:443" + ,.is_connect=0 + ,.rv=1 + } + +, {.name="proxy ipv6 request" + ,.url="http://[1:2::3:4]/" + ,.is_connect=0 + ,.u= + {.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PATH) + ,.port=0 + ,.field_data= + {{ 0, 4 } /* UF_SCHEMA */ + ,{ 8, 8 } /* UF_HOST */ + ,{ 0, 0 } /* UF_PORT */ + ,{ 17, 1 } /* UF_PATH */ + ,{ 0, 0 } /* UF_QUERY */ + ,{ 0, 0 } /* UF_FRAGMENT */ + ,{ 0, 0 } /* UF_USERINFO */ + } + } + ,.rv=0 + } + +, {.name="proxy ipv6 request with port" + ,.url="http://[1:2::3:4]:67/" + ,.is_connect=0 + ,.u= + {.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PORT) | (1 << UF_PATH) + ,.port=67 + ,.field_data= + {{ 0, 4 } /* UF_SCHEMA */ + ,{ 8, 8 } /* UF_HOST */ + ,{ 18, 2 } /* UF_PORT */ + ,{ 20, 1 } /* UF_PATH */ + ,{ 0, 0 } /* UF_QUERY */ + ,{ 0, 0 } /* UF_FRAGMENT */ + ,{ 0, 0 } /* UF_USERINFO */ + } + } + ,.rv=0 + } + +, {.name="CONNECT ipv6 address" + ,.url="[1:2::3:4]:443" + ,.is_connect=1 + ,.u= + {.field_set=(1 << UF_HOST) | (1 << UF_PORT) + ,.port=443 + ,.field_data= + {{ 0, 0 } /* UF_SCHEMA */ + ,{ 1, 8 } /* UF_HOST */ + ,{ 11, 3 } /* UF_PORT */ + ,{ 0, 0 } /* UF_PATH */ + ,{ 0, 0 } /* UF_QUERY */ + ,{ 0, 0 } /* UF_FRAGMENT */ + ,{ 0, 0 } /* UF_USERINFO */ + } + } + ,.rv=0 + } + +, {.name="ipv4 in ipv6 address" + ,.url="http://[2001:0000:0000:0000:0000:0000:1.9.1.1]/" + ,.is_connect=0 + ,.u= + {.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PATH) + ,.port=0 + ,.field_data= + {{ 0, 4 } /* UF_SCHEMA */ + ,{ 8, 37 } /* UF_HOST */ + ,{ 0, 0 } /* UF_PORT */ + ,{ 46, 1 } /* UF_PATH */ + ,{ 0, 0 } /* UF_QUERY */ + ,{ 0, 0 } /* UF_FRAGMENT */ + ,{ 0, 0 } /* UF_USERINFO */ + } + } + ,.rv=0 + } + +, {.name="extra ? in query string" + ,.url="http://a.tbcdn.cn/p/fp/2010c/??fp-header-min.css,fp-base-min.css," + "fp-channel-min.css,fp-product-min.css,fp-mall-min.css,fp-category-min.css," + "fp-sub-min.css,fp-gdp4p-min.css,fp-css3-min.css,fp-misc-min.css?t=20101022.css" + ,.is_connect=0 + ,.u= + {.field_set=(1<field_set, u->port); + for (i = 0; i < UF_MAX; i++) { + if ((u->field_set & (1 << i)) == 0) { + printf("\tfield_data[%u]: unset\n", i); + continue; + } + + printf("\tfield_data[%u]: off: %u len: %u part: \"%.*s\n\"", + i, + u->field_data[i].off, + u->field_data[i].len, + u->field_data[i].len, + url + u->field_data[i].off); + } +} + +void +test_parse_url (void) +{ + struct http_parser_url u; + const struct url_test *test; + unsigned int i; + int rv; + + for (i = 0; i < (sizeof(url_tests) / sizeof(url_tests[0])); i++) { + test = &url_tests[i]; + memset(&u, 0, sizeof(u)); + + rv = http_parser_parse_url(test->url, + strlen(test->url), + test->is_connect, + &u); + + if (test->rv == 0) { + if (rv != 0) { + printf("\n*** http_parser_parse_url(\"%s\") \"%s\" test failed, " + "unexpected rv %d ***\n\n", test->url, test->name, rv); + abort(); + } + + if (memcmp(&u, &test->u, sizeof(u)) != 0) { + printf("\n*** http_parser_parse_url(\"%s\") \"%s\" failed ***\n", + test->url, test->name); + + printf("target http_parser_url:\n"); + dump_url(test->url, &test->u); + printf("result http_parser_url:\n"); + dump_url(test->url, &u); + + abort(); + } + } else { + /* test->rv != 0 */ + if (rv == 0) { + printf("\n*** http_parser_parse_url(\"%s\") \"%s\" test failed, " + "unexpected rv %d ***\n\n", test->url, test->name, rv); + abort(); + } + } + } +} + + +void +test_message (const struct message *message) +{ + size_t raw_len = strlen(message->raw); + size_t msg1len; + for (msg1len = 0; msg1len < raw_len; msg1len++) { + parser_init(message->type); + + size_t read; + const char *msg1 = message->raw; + const char *msg2 = msg1 + msg1len; + size_t msg2len = raw_len - msg1len; + + if (msg1len) { + read = parse(msg1, msg1len); + + if (message->upgrade && parser->upgrade && num_messages > 0) { + messages[num_messages - 1].upgrade = msg1 + read; + goto test; + } + + if (read != msg1len) { + print_error(msg1, read); + exit(1); + } + } + + + read = parse(msg2, msg2len); + + if (message->upgrade && parser->upgrade) { + messages[num_messages - 1].upgrade = msg2 + read; + goto test; + } + + if (read != msg2len) { + print_error(msg2, read); + exit(1); + } + + read = parse(NULL, 0); + + if (read != 0) { + print_error(message->raw, read); + exit(1); + } + + test: + + if (num_messages != 1) { + printf("\n*** num_messages != 1 after testing '%s' ***\n\n", message->name); + exit(1); + } + + if(!message_eq(0, message)) exit(1); + + parser_free(); + } +} + +void +test_message_count_body (const struct message *message) +{ + parser_init(message->type); + + size_t read; + size_t l = strlen(message->raw); + size_t i, toread; + size_t chunk = 4024; + + for (i = 0; i < l; i+= chunk) { + toread = MIN(l-i, chunk); + read = parse_count_body(message->raw + i, toread); + if (read != toread) { + print_error(message->raw, read); + exit(1); + } + } + + + read = parse_count_body(NULL, 0); + if (read != 0) { + print_error(message->raw, read); + exit(1); + } + + if (num_messages != 1) { + printf("\n*** num_messages != 1 after testing '%s' ***\n\n", message->name); + exit(1); + } + + if(!message_eq(0, message)) exit(1); + + parser_free(); +} + +void +test_simple (const char *buf, enum http_errno err_expected) +{ + parser_init(HTTP_REQUEST); + + size_t parsed; + int pass; + enum http_errno err; + + parsed = parse(buf, strlen(buf)); + pass = (parsed == strlen(buf)); + err = HTTP_PARSER_ERRNO(parser); + parsed = parse(NULL, 0); + pass &= (parsed == 0); + + parser_free(); + + /* In strict mode, allow us to pass with an unexpected HPE_STRICT as + * long as the caller isn't expecting success. + */ +#if HTTP_PARSER_STRICT + if (err_expected != err && err_expected != HPE_OK && err != HPE_STRICT) { +#else + if (err_expected != err) { +#endif + fprintf(stderr, "\n*** test_simple expected %s, but saw %s ***\n\n%s\n", + http_errno_name(err_expected), http_errno_name(err), buf); + exit(1); + } +} + +void +test_header_overflow_error (int req) +{ + http_parser parser; + http_parser_init(&parser, req ? HTTP_REQUEST : HTTP_RESPONSE); + size_t parsed; + const char *buf; + buf = req ? "GET / HTTP/1.1\r\n" : "HTTP/1.0 200 OK\r\n"; + parsed = http_parser_execute(&parser, &settings_null, buf, strlen(buf)); + assert(parsed == strlen(buf)); + + buf = "header-key: header-value\r\n"; + size_t buflen = strlen(buf); + + int i; + for (i = 0; i < 10000; i++) { + parsed = http_parser_execute(&parser, &settings_null, buf, buflen); + if (parsed != buflen) { + //fprintf(stderr, "error found on iter %d\n", i); + assert(HTTP_PARSER_ERRNO(&parser) == HPE_HEADER_OVERFLOW); + return; + } + } + + fprintf(stderr, "\n*** Error expected but none in header overflow test ***\n"); + exit(1); +} + +void +test_no_overflow_long_body (int req, size_t length) +{ + http_parser parser; + http_parser_init(&parser, req ? HTTP_REQUEST : HTTP_RESPONSE); + size_t parsed; + size_t i; + char buf1[3000]; + size_t buf1len = sprintf(buf1, "%s\r\nConnection: Keep-Alive\r\nContent-Length: %zu\r\n\r\n", + req ? "POST / HTTP/1.0" : "HTTP/1.0 200 OK", length); + parsed = http_parser_execute(&parser, &settings_null, buf1, buf1len); + if (parsed != buf1len) + goto err; + + for (i = 0; i < length; i++) { + char foo = 'a'; + parsed = http_parser_execute(&parser, &settings_null, &foo, 1); + if (parsed != 1) + goto err; + } + + parsed = http_parser_execute(&parser, &settings_null, buf1, buf1len); + if (parsed != buf1len) goto err; + return; + + err: + fprintf(stderr, + "\n*** error in test_no_overflow_long_body %s of length %zu ***\n", + req ? "REQUEST" : "RESPONSE", + length); + exit(1); +} + +void +test_multiple3 (const struct message *r1, const struct message *r2, const struct message *r3) +{ + int message_count = count_parsed_messages(3, r1, r2, r3); + + char total[ strlen(r1->raw) + + strlen(r2->raw) + + strlen(r3->raw) + + 1 + ]; + total[0] = '\0'; + + strcat(total, r1->raw); + strcat(total, r2->raw); + strcat(total, r3->raw); + + parser_init(r1->type); + + size_t read; + + read = parse(total, strlen(total)); + + if (parser->upgrade) { + upgrade_message_fix(total, read, 3, r1, r2, r3); + goto test; + } + + if (read != strlen(total)) { + print_error(total, read); + exit(1); + } + + read = parse(NULL, 0); + + if (read != 0) { + print_error(total, read); + exit(1); + } + +test: + + if (message_count != num_messages) { + fprintf(stderr, "\n\n*** Parser didn't see 3 messages only %d *** \n", num_messages); + exit(1); + } + + if (!message_eq(0, r1)) exit(1); + if (message_count > 1 && !message_eq(1, r2)) exit(1); + if (message_count > 2 && !message_eq(2, r3)) exit(1); + + parser_free(); +} + +/* SCAN through every possible breaking to make sure the + * parser can handle getting the content in any chunks that + * might come from the socket + */ +void +test_scan (const struct message *r1, const struct message *r2, const struct message *r3) +{ + char total[80*1024] = "\0"; + char buf1[80*1024] = "\0"; + char buf2[80*1024] = "\0"; + char buf3[80*1024] = "\0"; + + strcat(total, r1->raw); + strcat(total, r2->raw); + strcat(total, r3->raw); + + size_t read; + + int total_len = strlen(total); + + int total_ops = 2 * (total_len - 1) * (total_len - 2) / 2; + int ops = 0 ; + + size_t buf1_len, buf2_len, buf3_len; + int message_count = count_parsed_messages(3, r1, r2, r3); + + int i,j,type_both; + for (type_both = 0; type_both < 2; type_both ++ ) { + for (j = 2; j < total_len; j ++ ) { + for (i = 1; i < j; i ++ ) { + + if (ops % 1000 == 0) { + printf("\b\b\b\b%3.0f%%", 100 * (float)ops /(float)total_ops); + fflush(stdout); + } + ops += 1; + + parser_init(type_both ? HTTP_BOTH : r1->type); + + buf1_len = i; + strncpy(buf1, total, buf1_len); + buf1[buf1_len] = 0; + + buf2_len = j - i; + strncpy(buf2, total+i, buf2_len); + buf2[buf2_len] = 0; + + buf3_len = total_len - j; + strncpy(buf3, total+j, buf3_len); + buf3[buf3_len] = 0; + + read = parse(buf1, buf1_len); + + if (parser->upgrade) goto test; + + if (read != buf1_len) { + print_error(buf1, read); + goto error; + } + + read += parse(buf2, buf2_len); + + if (parser->upgrade) goto test; + + if (read != buf1_len + buf2_len) { + print_error(buf2, read); + goto error; + } + + read += parse(buf3, buf3_len); + if (parser->upgrade) goto test; + + if (read != buf1_len + buf2_len + buf3_len) { + print_error(buf3, read); + goto error; + } + + parse(NULL, 0); + +test: + if (parser->upgrade) { + upgrade_message_fix(total, read, 3, r1, r2, r3); + } + + if (message_count != num_messages) { + fprintf(stderr, "\n\nParser didn't see %d messages only %d\n", + message_count, num_messages); + goto error; + } + + if (!message_eq(0, r1)) { + fprintf(stderr, "\n\nError matching messages[0] in test_scan.\n"); + goto error; + } + + if (message_count > 1 && !message_eq(1, r2)) { + fprintf(stderr, "\n\nError matching messages[1] in test_scan.\n"); + goto error; + } + + if (message_count > 2 && !message_eq(2, r3)) { + fprintf(stderr, "\n\nError matching messages[2] in test_scan.\n"); + goto error; + } + + parser_free(); + } + } + } + puts("\b\b\b\b100%"); + return; + + error: + fprintf(stderr, "i=%d j=%d\n", i, j); + fprintf(stderr, "buf1 (%u) %s\n\n", (unsigned int)buf1_len, buf1); + fprintf(stderr, "buf2 (%u) %s\n\n", (unsigned int)buf2_len , buf2); + fprintf(stderr, "buf3 (%u) %s\n", (unsigned int)buf3_len, buf3); + exit(1); +} + +// user required to free the result +// string terminated by \0 +char * +create_large_chunked_message (int body_size_in_kb, const char* headers) +{ + int i; + size_t wrote = 0; + size_t headers_len = strlen(headers); + size_t bufsize = headers_len + (5+1024+2)*body_size_in_kb + 6; + char * buf = malloc(bufsize); + + memcpy(buf, headers, headers_len); + wrote += headers_len; + + for (i = 0; i < body_size_in_kb; i++) { + // write 1kb chunk into the body. + memcpy(buf + wrote, "400\r\n", 5); + wrote += 5; + memset(buf + wrote, 'C', 1024); + wrote += 1024; + strcpy(buf + wrote, "\r\n"); + wrote += 2; + } + + memcpy(buf + wrote, "0\r\n\r\n", 6); + wrote += 6; + assert(wrote == bufsize); + + return buf; +} + +/* Verify that we can pause parsing at any of the bytes in the + * message and still get the result that we're expecting. */ +void +test_message_pause (const struct message *msg) +{ + char *buf = (char*) msg->raw; + size_t buflen = strlen(msg->raw); + size_t nread; + + parser_init(msg->type); + + do { + nread = parse_pause(buf, buflen); + + // We can only set the upgrade buffer once we've gotten our message + // completion callback. + if (messages[0].message_complete_cb_called && + msg->upgrade && + parser->upgrade) { + messages[0].upgrade = buf + nread; + goto test; + } + + if (nread < buflen) { + + // Not much do to if we failed a strict-mode check + if (HTTP_PARSER_ERRNO(parser) == HPE_STRICT) { + parser_free(); + return; + } + + assert (HTTP_PARSER_ERRNO(parser) == HPE_PAUSED); + } + + buf += nread; + buflen -= nread; + http_parser_pause(parser, 0); + } while (buflen > 0); + + nread = parse_pause(NULL, 0); + assert (nread == 0); + +test: + if (num_messages != 1) { + printf("\n*** num_messages != 1 after testing '%s' ***\n\n", msg->name); + exit(1); + } + + if(!message_eq(0, msg)) exit(1); + + parser_free(); +} + +/* Verify that on_message_begin callback is called regardless of correctness of + * the message */ +void +test_on_message_begin_cb(const struct message *message) +{ + parser_init(message->type); + assert(messages[num_messages].message_begin_cb_called == 0); + parse(message->raw, strlen(message->raw)); + assert(messages[num_messages].message_begin_cb_called == 1); + parser_free(); +} + +int +main (void) +{ + parser = NULL; + int i, j, k; + int request_count; + int response_count; + int on_message_begin_cb_count; + + printf("sizeof(http_parser) = %u\n", (unsigned int)sizeof(http_parser)); + + for (request_count = 0; requests[request_count].name; request_count++); + for (response_count = 0; responses[response_count].name; response_count++); + for (on_message_begin_cb_count = 0; + on_message_begin_cb_test[on_message_begin_cb_count].name; + on_message_begin_cb_count++); + + //// API + test_parse_url(); + + /// TESTING ON_MESSAGE_BEGIN CALLS + for (i = 0 ; i < on_message_begin_cb_count; i++) { + test_on_message_begin_cb(&on_message_begin_cb_test[i]); + } + + //// OVERFLOW CONDITIONS + + test_header_overflow_error(HTTP_REQUEST); + test_no_overflow_long_body(HTTP_REQUEST, 1000); + test_no_overflow_long_body(HTTP_REQUEST, 100000); + + test_header_overflow_error(HTTP_RESPONSE); + test_no_overflow_long_body(HTTP_RESPONSE, 1000); + test_no_overflow_long_body(HTTP_RESPONSE, 100000); + + //// RESPONSES + + for (i = 0; i < response_count; i++) { + test_message(&responses[i]); + } + + for (i = 0; i < response_count; i++) { + test_message_pause(&responses[i]); + } + + for (i = 0; i < response_count; i++) { + if (!responses[i].should_keep_alive) continue; + for (j = 0; j < response_count; j++) { + if (!responses[j].should_keep_alive) continue; + for (k = 0; k < response_count; k++) { + test_multiple3(&responses[i], &responses[j], &responses[k]); + } + } + } + + test_message_count_body(&responses[NO_HEADERS_NO_BODY_404]); + test_message_count_body(&responses[TRAILING_SPACE_ON_CHUNKED_BODY]); + + // test very large chunked response + { + char * msg = create_large_chunked_message(31337, + "HTTP/1.0 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "Content-Type: text/plain\r\n" + "\r\n"); + struct message large_chunked = + {.name= "large chunked" + ,.type= HTTP_RESPONSE + ,.raw= msg + ,.should_keep_alive= FALSE + ,.message_complete_on_eof= FALSE + ,.http_major= 1 + ,.http_minor= 0 + ,.status_code= 200 + ,.response_reason= "OK" + ,.num_headers= 2 + ,.headers= + { { "Transfer-Encoding", "chunked" } + , { "Content-Type", "text/plain" } + } + ,.body_size= 31337*1024 + ,.num_chunks= 31337 + ,.num_chunks_complete= 31338 + }; + for (i = 0; i < MAX_CHUNKS; ++i) { + large_chunked.chunk_lengths[i] = 1024; + } + test_message_count_body(&large_chunked); + free(msg); + } + + + + printf("response scan 1/2 "); + test_scan( &responses[TRAILING_SPACE_ON_CHUNKED_BODY] + , &responses[NO_REASON_PHRASE] + , &responses[NO_HEADERS_NO_BODY_404] + ); + + printf("response scan 2/2 "); + test_scan( &responses[BONJOUR_MADAME_FR] + , &responses[UNDERSTORE_HEADER_KEY] + , &responses[NO_CARRIAGE_RET] + ); + + puts("responses okay"); + + + /// REQUESTS + + test_simple("hello world", HPE_INVALID_METHOD); + test_simple("GET / HTP/1.1\r\n\r\n", HPE_INVALID_VERSION); + + + test_simple("ASDF / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD); + test_simple("PROPPATCHA / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD); + test_simple("GETA / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD); + + // Well-formed but incomplete + test_simple("GET / HTTP/1.1\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: 6\r\n" + "\r\n" + "fooba", + HPE_OK); + + static const char *all_methods[] = { + "DELETE", + "GET", + "HEAD", + "POST", + "PUT", + //"CONNECT", //CONNECT can't be tested like other methods, it's a tunnel + "OPTIONS", + "TRACE", + "COPY", + "LOCK", + "MKCOL", + "MOVE", + "PROPFIND", + "PROPPATCH", + "UNLOCK", + "REPORT", + "MKACTIVITY", + "CHECKOUT", + "MERGE", + "M-SEARCH", + "NOTIFY", + "SUBSCRIBE", + "UNSUBSCRIBE", + "PATCH", + 0 }; + const char **this_method; + for (this_method = all_methods; *this_method; this_method++) { + char buf[200]; + sprintf(buf, "%s / HTTP/1.1\r\n\r\n", *this_method); + test_simple(buf, HPE_OK); + } + + static const char *bad_methods[] = { + "C******", + "M****", + 0 }; + for (this_method = bad_methods; *this_method; this_method++) { + char buf[200]; + sprintf(buf, "%s / HTTP/1.1\r\n\r\n", *this_method); + test_simple(buf, HPE_UNKNOWN); + } + + const char *dumbfuck2 = + "GET / HTTP/1.1\r\n" + "X-SSL-Bullshit: -----BEGIN CERTIFICATE-----\r\n" + "\tMIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx\r\n" + "\tETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT\r\n" + "\tAkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu\r\n" + "\tdWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV\r\n" + "\tSzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV\r\n" + "\tBAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB\r\n" + "\tBQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF\r\n" + "\tW51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR\r\n" + "\tgW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL\r\n" + "\t0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP\r\n" + "\tu2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR\r\n" + "\twgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG\r\n" + "\tA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs\r\n" + "\tBglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD\r\n" + "\tVR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj\r\n" + "\tloCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj\r\n" + "\taWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG\r\n" + "\t9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE\r\n" + "\tIjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO\r\n" + "\tBgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1\r\n" + "\tcHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4QgEDBDAWLmh0\r\n" + "\tdHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC5jcmwwPwYD\r\n" + "\tVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv\r\n" + "\tY3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3\r\n" + "\tXCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8\r\n" + "\tUO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk\r\n" + "\thTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK\r\n" + "\twTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu\r\n" + "\tYhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3\r\n" + "\tRA==\r\n" + "\t-----END CERTIFICATE-----\r\n" + "\r\n"; + test_simple(dumbfuck2, HPE_OK); + + const char *corrupted_header_name = + "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "X-Some-Header\r\033\065\325eep-Alive\r\n" + "Accept-Encoding: gzip\r\n" + "\r\n"; + test_simple(corrupted_header_name, HPE_INVALID_HEADER_TOKEN); + +#if 0 + // NOTE(Wed Nov 18 11:57:27 CET 2009) this seems okay. we just read body + // until EOF. + // + // no content-length + // error if there is a body without content length + const char *bad_get_no_headers_no_body = "GET /bad_get_no_headers_no_body/world HTTP/1.1\r\n" + "Accept: */*\r\n" + "\r\n" + "HELLO"; + test_simple(bad_get_no_headers_no_body, 0); +#endif + /* TODO sending junk and large headers gets rejected */ + + + /* check to make sure our predefined requests are okay */ + for (i = 0; requests[i].name; i++) { + test_message(&requests[i]); + } + + for (i = 0; i < request_count; i++) { + test_message_pause(&requests[i]); + } + + for (i = 0; i < request_count; i++) { + if (!requests[i].should_keep_alive) continue; + for (j = 0; j < request_count; j++) { + if (!requests[j].should_keep_alive) continue; + for (k = 0; k < request_count; k++) { + test_multiple3(&requests[i], &requests[j], &requests[k]); + } + } + } + + printf("request scan 1/4 "); + test_scan( &requests[GET_NO_HEADERS_NO_BODY] + , &requests[GET_ONE_HEADER_NO_BODY] + , &requests[GET_NO_HEADERS_NO_BODY] + ); + + printf("request scan 2/4 "); + test_scan( &requests[POST_CHUNKED_ALL_YOUR_BASE] + , &requests[POST_IDENTITY_BODY_WORLD] + , &requests[GET_FUNKY_CONTENT_LENGTH] + ); + + printf("request scan 3/4 "); + test_scan( &requests[TWO_CHUNKS_MULT_ZERO_END] + , &requests[CHUNKED_W_TRAILING_HEADERS] + , &requests[CHUNKED_W_BULLSHIT_AFTER_LENGTH] + ); + + printf("request scan 4/4 "); + test_scan( &requests[QUERY_URL_WITH_QUESTION_MARK_GET] + , &requests[PREFIX_NEWLINE_GET ] + , &requests[CONNECT_REQUEST] + ); + + puts("requests okay"); + + return 0; +} diff --git a/proxygen/httpserver/Filters.h b/proxygen/httpserver/Filters.h new file mode 100644 index 0000000000..66f7c4018d --- /dev/null +++ b/proxygen/httpserver/Filters.h @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/RequestHandler.h" +#include "proxygen/httpserver/ResponseHandler.h" + +namespace proxygen { + +/** + * Filters are a way to add functionality to HTTPServer without complicating app + * specific RequestHandler. The basic idea is + * + * App-handler <=> Filter-1 <=> Filter-2 <=> Client + * + * The data flows through these filters between client and handler. They can do + * things like modify the data flowing through them, can update stats, create + * traces and even send direct response if they feel like. + * + * The default implementation just lets everything pass through. + */ +class Filter : public RequestHandler, public ResponseHandler { + public: + explicit Filter(RequestHandler* upstream) + : ResponseHandler(upstream) { + } + + // Request handler + void setResponseHandler(ResponseHandler* handler) noexcept override { + // Save downstream handler and pass ourselves as downstream handler + downstream_ = handler; + upstream_->setResponseHandler(this); + } + + void onRequest(std::unique_ptr headers) noexcept override { + upstream_->onRequest(std::move(headers)); + } + + void onBody(std::unique_ptr body) noexcept override { + upstream_->onBody(std::move(body)); + } + + void onUpgrade(UpgradeProtocol protocol) noexcept override { + upstream_->onUpgrade(protocol); + } + + void onEOM() noexcept override { + upstream_->onEOM(); + } + + void requestComplete() noexcept override { + downstream_ = nullptr; + upstream_->requestComplete(); + delete this; + } + + void onError(ProxygenError err) noexcept override { + downstream_ = nullptr; + upstream_->onError(err); + delete this; + } + + void onEgressPaused() noexcept { + upstream_->onEgressPaused(); + } + + void onEgressResumed() noexcept { + upstream_->onEgressResumed(); + } + + // Response handler + void sendHeaders(HTTPMessage& msg) noexcept override { + downstream_->sendHeaders(msg); + } + + void sendChunkHeader(size_t len) noexcept override { + downstream_->sendChunkHeader(len); + } + + void sendBody(std::unique_ptr body) noexcept override { + downstream_->sendBody(std::move(body)); + } + + void sendChunkTerminator() noexcept override { + downstream_->sendChunkTerminator(); + } + + void sendEOM() noexcept override { + downstream_->sendEOM(); + } + + void sendAbort() noexcept override { + downstream_->sendAbort(); + } + + void refreshTimeout() noexcept override { + downstream_->refreshTimeout(); + } + + void pauseIngress() noexcept override { + downstream_->pauseIngress(); + } + + void resumeIngress() noexcept override { + downstream_->resumeIngress(); + } + + const TransportInfo& getSetupTransportInfo() const noexcept override { + return downstream_->getSetupTransportInfo(); + } + + void getCurrentTransportInfo(TransportInfo* tinfo) const override { + downstream_->getCurrentTransportInfo(tinfo); + } + +}; + +} diff --git a/proxygen/httpserver/HTTPServer.cpp b/proxygen/httpserver/HTTPServer.cpp new file mode 100644 index 0000000000..01f25ada5b --- /dev/null +++ b/proxygen/httpserver/HTTPServer.cpp @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/httpserver/HTTPServer.h" + +#include "proxygen/httpserver/HTTPServerAcceptor.h" +#include "proxygen/httpserver/SignalHandler.h" +#include "proxygen/httpserver/filters/RejectConnectFilter.h" + +#include +#include +#include + +using apache::thrift::async::TAsyncServerSocket; +using folly::EventBase; +using folly::EventBaseManager; +using folly::SocketAddress; + +namespace proxygen { + +HTTPServer::HTTPServer(HTTPServerOptions options): + options_(std::move(options)) { + + // Insert a filter to fail all the CONNECT request, if required + if (!options_.supportsConnect) { + options_.handlerFactories.insert( + options_.handlerFactories.begin(), + folly::make_unique()); + } +} + +HTTPServer::~HTTPServer() { + CHECK(!mainEventBase_) << "Forgot to stop() server?"; +} + +void HTTPServer::bind(std::vector& addrs) { + CHECK(serverSockets_.empty()) << "Server sockets are already bound"; + + // Set a scope guard that brings us to unintialized state + folly::ScopeGuard g = folly::makeGuard([&] { + serverSockets_.clear(); + }); + + auto evb = EventBaseManager::get()->getEventBase(); + for (auto& addr: addrs) { + serverSockets_.emplace_back(new TAsyncServerSocket(evb)); + serverSockets_.back()->bind(addr.address); + + // Use might have asked to register with some ephemeral port + serverSockets_.back()->getAddress(&addr.address); + } + + // If everything succeeded, dismiss the cleanup guard + g.dismiss(); + addresses_ = addrs; +} + +void HTTPServer::start(std::function onSuccess, + std::function onError) { + // Step 1: Check that server sockets are bound + CHECK(!serverSockets_.empty()) << "Need to call `bind()` before `start()`"; + + // Global Event base manager that will be used to create all the worker + // threads eventbases + auto manager = EventBaseManager::get(); + mainEventBase_ = manager->getEventBase(); + + // Will be used to make sure that all the event loops are running + // before we proceed + boost::barrier barrier(2); + + // Step 2: Setup handler threads + handlerThreads_ = std::vector(options_.threads); + + std::vector accConfigs; + FOR_EACH_RANGE (i, 0, addresses_.size()) { + accConfigs.emplace_back(HTTPServerAcceptor::makeConfig(addresses_[i], + options_)); + } + + for (auto& handlerThread: handlerThreads_) { + handlerThread.thread = std::thread([&] () { + folly::setThreadName("http-worker"); + handlerThread.eventBase = manager->getEventBase(); + barrier.wait(); + + handlerThread.eventBase->loopForever(); + + // Call loop() again to drain all the events + handlerThread.eventBase->loop(); + }); + + // Wait for eventbase pointer to be set + barrier.wait(); + + // Make sure event loop is running before we proceed + handlerThread.eventBase->runInEventBaseThread([&] () { + barrier.wait(); + for (auto& factory: options_.handlerFactories) { + factory->onServerStart(); + } + }); + barrier.wait(); + + // Create acceptors + FOR_EACH_RANGE (i, 0, accConfigs.size()) { + auto acc = HTTPServerAcceptor::make(accConfigs[i], options_); + acc->init(serverSockets_[i].get(), handlerThread.eventBase); + ++handlerThread.acceptorsRunning; + + // Set completion callback such that it invokes onServerStop when + // all acceptors are drained + HandlerThread* ptr = &handlerThread; + acc->setCompletionCallback([ptr, this] () { + --ptr->acceptorsRunning; + + if (ptr->acceptorsRunning == 0) { + for (auto& factory: options_.handlerFactories) { + factory->onServerStop(); + } + } + }); + + handlerThread.acceptors.push_back(std::move(acc)); + } + } + + // Step 3: Install signal handler if required + if (!options_.shutdownOn.empty()) { + signalHandler_ = folly::make_unique(this); + signalHandler_->install(options_.shutdownOn); + } + + // Step 4: Switch the server socket eventbase (bind may have been invoked + // in a separate thread). + for (auto& serverSocket: serverSockets_) { + serverSocket->detachEventBase(); + serverSocket->attachEventBase(mainEventBase_); + } + + // Step 5: Start listening for connections. This can throw if somebody else + // binds to same address and calls listen before this. + try { + for (auto& serverSocket: serverSockets_) { + serverSocket->listen(options_.listenBacklog); + serverSocket->startAccepting(); + } + } catch (...) { + stop(); + + if (onError) { + onError(std::current_exception()); + return; + } + + throw; + } + + // Step 6: Start the main event loop + if (onSuccess) { + mainEventBase_->runInEventBaseThread([onSuccess] () { + onSuccess(); + }); + } + + mainEventBase_->loopForever(); +} + +void HTTPServer::stop() { + CHECK(mainEventBase_); + + if (!mainEventBase_->isInEventBaseThread()) { + auto barrier = std::make_shared(2); + mainEventBase_->runInEventBaseThread([this, barrier] () { + stop(); + barrier->wait(); + }); + + barrier->wait(); + return; + } + + signalHandler_.reset(); + serverSockets_.clear(); + mainEventBase_->terminateLoopSoon(); + mainEventBase_ = nullptr; + + for (auto& handlerThread: handlerThreads_) { + handlerThread.eventBase->terminateLoopSoon(); + } + + for (auto& handlerThread: handlerThreads_) { + handlerThread.thread.join(); + } + + handlerThreads_.clear(); +} + +} diff --git a/proxygen/httpserver/HTTPServer.h b/proxygen/httpserver/HTTPServer.h new file mode 100644 index 0000000000..12b4780018 --- /dev/null +++ b/proxygen/httpserver/HTTPServer.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/HTTPServerOptions.h" +#include "proxygen/lib/ssl/SSLContextConfig.h" + +#include +#include +#include + +namespace proxygen { + +class SignalHandler; +class HTTPServerAcceptor; + +/** + * HTTPServer based on proxygen http libraries + */ +class HTTPServer final { + public: + /** + * For each IP you can specify HTTP protocol to use. You can use plain old + * HTTP/1.1 protocol or SPDY/3.1 for now. + */ + enum class Protocol: uint8_t { + HTTP, + SPDY, + }; + + struct IPConfig { + IPConfig(folly::SocketAddress a, + Protocol p) + : address(a), + protocol(p) {} + + folly::SocketAddress address; + Protocol protocol; + std::vector sslConfigs; + }; + + /** + * Create a new HTTPServer + */ + explicit HTTPServer(HTTPServerOptions options); + ~HTTPServer(); + + /** + * Bind server to the following addresses. Can be called from any thread. + * + * Throws exception on error (say port is already busy). You can try binding + * to different set of ports. Though once it succeeds, it is a FATAL to call + * it again. + * + * The list is updated in-place to contain final port server bound to if + * ephemeral port was given. If the call fails, the list might be partially + * updated. + */ + void bind(std::vector& addrs); + + /** + * Start HTTPServer. + * + * Note this is a blocking call and the current thread will be used to listen + * for incoming connections. Throws exception if something goes wrong (say + * somebody else is already listening on that socket). + * + * `onSuccess` callback will be invoked from the event loop which shows that + * all the setup was successfully done. + * + * `onError` callback will be invoked if some errors occurs while starting the + * server instead of throwing exception. + */ + void start(std::function onSuccess = nullptr, + std::function onError = nullptr); + + /** + * Stop HTTPServer. + * + * Can be called from any thread. Server will stop listening for new + * connections and will wait for running requests to finish. + * + * TODO: Separate method to do hard shutdown? + */ + void stop(); + + /** + * Get the list of addresses server is listening on. Empty if sockets are not + * bound yet. + */ + std::vector addresses() const { + return addresses_; + } + + private: + HTTPServerOptions options_; + + /** + * Event base in which we binded server sockets. + */ + folly::EventBase* mainEventBase_{nullptr}; + + /** + * Server socket + */ + std::vector + serverSockets_; + + /** + * Struct to hold info about handle threads + */ + struct HandlerThread { + std::thread thread; + + folly::EventBase* eventBase{nullptr}; + + std::vector> acceptors; + + uint32_t acceptorsRunning{0}; + }; + + std::vector handlerThreads_; + + /** + * Optional signal handlers on which we should shutdown server + */ + std::unique_ptr signalHandler_; + + /** + * Addresses we are listening on + */ + std::vector addresses_; +}; + +} diff --git a/proxygen/httpserver/HTTPServerAcceptor.cpp b/proxygen/httpserver/HTTPServerAcceptor.cpp new file mode 100644 index 0000000000..2f18bcc42a --- /dev/null +++ b/proxygen/httpserver/HTTPServerAcceptor.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/httpserver/HTTPServerAcceptor.h" + +#include "proxygen/httpserver/RequestHandlerAdaptor.h" +#include "proxygen/httpserver/RequestHandlerFactory.h" +#include "proxygen/lib/http/codec/HTTP1xCodec.h" +#include "proxygen/lib/http/session/HTTPDownstreamSession.h" + +using folly::SocketAddress; + +namespace proxygen { + +AcceptorConfiguration HTTPServerAcceptor::makeConfig( + const HTTPServer::IPConfig& ipConfig, + const HTTPServerOptions& opts) { + + AcceptorConfiguration conf; + conf.bindAddress = ipConfig.address; + conf.connectionIdleTimeout = opts.idleTimeout; + conf.transactionIdleTimeout = opts.idleTimeout; + + if (ipConfig.protocol == HTTPServer::Protocol::SPDY) { + conf.plaintextProtocol = "spdy/3.1"; + } + + conf.sslContextConfigs = ipConfig.sslConfigs; + return conf; +} + +std::unique_ptr HTTPServerAcceptor::make( + const AcceptorConfiguration& conf, + const HTTPServerOptions& opts) { + // Create a copy of the filter chain in reverse order since we need to create + // Handlers in that order. + std::vector handlerFactories; + for (auto& f: opts.handlerFactories) { + handlerFactories.push_back(f.get()); + } + std::reverse(handlerFactories.begin(), handlerFactories.end()); + + return std::unique_ptr( + new HTTPServerAcceptor(conf, handlerFactories)); +} + +HTTPServerAcceptor::HTTPServerAcceptor( + const AcceptorConfiguration& conf, + std::vector handlerFactories) + : HTTPSessionAcceptor(conf), + handlerFactories_(handlerFactories) { +} + +void HTTPServerAcceptor::setCompletionCallback(std::function f) { + completionCallback_ = f; +} + +HTTPServerAcceptor::~HTTPServerAcceptor() { +} + +HTTPTransactionHandler* HTTPServerAcceptor::newHandler( + HTTPTransaction& txn, + HTTPMessage* msg) noexcept { + + SocketAddress clientAddr, vipAddr; + txn.getPeerAddress(clientAddr); + txn.getLocalAddress(vipAddr); + msg->setClientAddress(clientAddr); + msg->setDstAddress(vipAddr); + + // Create filters chain + RequestHandler* h = nullptr; + for (auto& factory: handlerFactories_) { + h = factory->onRequest(h, msg); + } + + return new RequestHandlerAdaptor(h); +} + +void HTTPServerAcceptor::onConnectionsDrained() { + if (completionCallback_) { + completionCallback_(); + } +} + +} diff --git a/proxygen/httpserver/HTTPServerAcceptor.h b/proxygen/httpserver/HTTPServerAcceptor.h new file mode 100644 index 0000000000..0708c90e97 --- /dev/null +++ b/proxygen/httpserver/HTTPServerAcceptor.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/HTTPServer.h" +#include "proxygen/httpserver/HTTPServerOptions.h" +#include "proxygen/lib/http/session/HTTPSessionAcceptor.h" + +namespace proxygen { + +class HTTPServerAcceptor final : public HTTPSessionAcceptor { + public: + static AcceptorConfiguration makeConfig( + const HTTPServer::IPConfig& ipConfig, + const HTTPServerOptions& opts); + + static std::unique_ptr make( + const AcceptorConfiguration& conf, + const HTTPServerOptions& opts); + + /** + * Invokes the given method when all the connections are drained + */ + void setCompletionCallback(std::function f); + + ~HTTPServerAcceptor(); + + private: + HTTPServerAcceptor(const AcceptorConfiguration& conf, + std::vector handlerFactories); + + // HTTPSessionAcceptor + HTTPTransaction::Handler* newHandler(HTTPTransaction& txn, + HTTPMessage* msg) noexcept override; + void onConnectionsDrained() override; + + std::function completionCallback_; + const std::vector handlerFactories_{nullptr}; +}; + +} diff --git a/proxygen/httpserver/HTTPServerOptions.h b/proxygen/httpserver/HTTPServerOptions.h new file mode 100644 index 0000000000..5b5825b7b7 --- /dev/null +++ b/proxygen/httpserver/HTTPServerOptions.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/Filters.h" +#include "proxygen/httpserver/RequestHandlerFactory.h" + +#include +#include + +namespace proxygen { + +/** + * Configuration options for HTTPServer + * + * XXX: Provide a helper that can convert thrift/json to this config + * directly. We keep this object type-safe. + */ +class HTTPServerOptions { + public: + /** + * Number of threads to start to handle requests. Note that this excludes + * the thread you call `HTTPServer.start()` in. + * + * XXX: Put some perf numbers to help user decide how many threads to + * create. + * XXX: Maybe support not creating any more worker threads and doing all + * the work in same thread when `threads == 0`. + */ + size_t threads = 1; + + /** + * Chain of RequestHandlerFactory that are used to create RequestHandler + * which handles requests. + * + * You can do something like - + * + * handlerFactories = RequestHandlerChain() + * .addThen() + * .addThen() + * .addThen() + * .addThen() + * .build(); + */ + std::vector> handlerFactories; + + /** + * This idle timeout serves two purposes - + * + * 1. How long to keep idle connections around before throwing them away. + * + * 2. If it takes client more than this time to send request, we fail the + * request. + * + * XXX: Maybe have separate time limit for both? + */ + std::chrono::milliseconds idleTimeout{60000}; + + /** + * TCP server socket backlog to start with. + */ + uint32_t listenBacklog{1024}; + + /** + * Signals on which to shutdown the server. Mostly you will want + * {SIGINT, SIGTERM}. Note, if you have multiple deamons running or you want + * to have a separate signal handler, leave this empty and handle signals + * yourself. + */ + std::vector shutdownOn{}; + + /** + * Set to true if you want to support CONNECT request. Most likely you + * don't want that. + */ + bool supportsConnect{false}; +}; + +} diff --git a/proxygen/httpserver/Makefile.am b/proxygen/httpserver/Makefile.am new file mode 100644 index 0000000000..7acbbfdbd9 --- /dev/null +++ b/proxygen/httpserver/Makefile.am @@ -0,0 +1,27 @@ +SUBDIRS = . samples tests + +lib_LTLIBRARIES = libproxygenhttpserver.la + +libproxygenhttpserverdir = $(includedir)/proxygen/httpserver +nobase_libproxygenhttpserver_HEADERS = \ + Filters.h \ + HTTPServer.h \ + HTTPServerAcceptor.h \ + HTTPServerOptions.h \ + Mocks.h \ + RequestHandler.h \ + RequestHandlerAdaptor.h \ + RequestHandlerFactory.h \ + ResponseBuilder.h \ + ResponseHandler.h \ + ScopedHTTPServer.h \ + SignalHandler.h + +libproxygenhttpserver_la_SOURCES = \ + HTTPServer.cpp \ + HTTPServerAcceptor.cpp \ + RequestHandlerAdaptor.cpp \ + SignalHandler.cpp + +libproxygenhttpserver_la_LIBADD = \ + ../lib/libproxygenlib.la diff --git a/proxygen/httpserver/Mocks.h b/proxygen/httpserver/Mocks.h new file mode 100644 index 0000000000..09b310a69c --- /dev/null +++ b/proxygen/httpserver/Mocks.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/RequestHandler.h" +#include "proxygen/httpserver/ResponseHandler.h" + +#include + +namespace proxygen { + +/** + * Mocks to help with testing + */ + +class MockResponseHandler : public ResponseHandler { + public: + explicit MockResponseHandler(RequestHandler* h): ResponseHandler(h) { + } + + GMOCK_METHOD1_(, noexcept,, sendHeaders, void(HTTPMessage&)); + GMOCK_METHOD1_(, noexcept,, sendChunkHeader, void(size_t)); + GMOCK_METHOD1_(, noexcept,, sendBody, void(std::shared_ptr)); + GMOCK_METHOD0_(, noexcept,, sendChunkTerminator, void()); + GMOCK_METHOD0_(, noexcept,, sendEOM, void()); + GMOCK_METHOD0_(, noexcept,, sendAbort, void()); + GMOCK_METHOD0_(, noexcept,, refreshTimeout, void()); + GMOCK_METHOD0_(, noexcept,, pauseIngress, void()); + GMOCK_METHOD0_(, noexcept,, resumeIngress, void()); + const TransportInfo& getSetupTransportInfo() const noexcept { + return transportInfo; + } + MOCK_CONST_METHOD1(getCurrentTransportInfo, void(TransportInfo*)); + + void sendBody(std::unique_ptr body) noexcept override { + if (body) { + sendBody(std::shared_ptr(body.release())); + } else { + sendBody(std::shared_ptr(nullptr)); + } + } + + TransportInfo transportInfo; +}; + +class MockRequestHandler : public RequestHandler { + public: + GMOCK_METHOD1_(, noexcept,, setResponseHandler, void(ResponseHandler*)); + GMOCK_METHOD1_(, noexcept,, onRequest, void(std::shared_ptr)); + GMOCK_METHOD1_(, noexcept,, onBody, void(std::shared_ptr)); + GMOCK_METHOD1_(, noexcept,, onUpgrade, void(UpgradeProtocol)); + GMOCK_METHOD0_(, noexcept,, onEOM, void()); + GMOCK_METHOD0_(, noexcept,, requestComplete, void()); + GMOCK_METHOD1_(, noexcept,, onError, void(ProxygenError)); + GMOCK_METHOD0_(, noexcept,, onEgressPaused, void()); + GMOCK_METHOD0_(, noexcept,, onEgressResumed, void()); + + void onRequest(std::unique_ptr headers) noexcept override { + onRequest(std::shared_ptr(headers.release())); + } + + void onBody(std::unique_ptr body) noexcept override { + if (body) { + onBody(std::shared_ptr(body.release())); + } else { + onBody(std::shared_ptr(nullptr)); + } + } +}; + +} diff --git a/proxygen/httpserver/RequestHandler.h b/proxygen/httpserver/RequestHandler.h new file mode 100644 index 0000000000..7e285d4d88 --- /dev/null +++ b/proxygen/httpserver/RequestHandler.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPTransaction.h" + +namespace proxygen { + +class ResponseHandler; + +/** + * Interface to be implemented by objects that handle requests from + * client. ResponseHandler acts as the client for these objects and + * provides methods to send back the response + */ +class RequestHandler { + public: + /** + * Saves the downstream handle with itself. Implementations of this + * interface should use downstream_ to send back response. + * + * XXX: Only override this method if you are ABSOLUTELY sure what you + * are doing. If possible, just use downstream_ variable and dont + * mess with these things. + */ + virtual void setResponseHandler(ResponseHandler* handler) noexcept { + downstream_ = CHECK_NOTNULL(handler); + } + + /** + * Invoked when we have successfully fetched headers from client. This will + * always be the first callback invoked on your handler. + */ + virtual void onRequest(std::unique_ptr headers) noexcept = 0; + + /** + * Invoked when we get part of body for the request. + */ + virtual void onBody(std::unique_ptr body) noexcept = 0; + + /** + * Invoked when the session has been upgraded to a different protocol + */ + virtual void onUpgrade(proxygen::UpgradeProtocol prot) noexcept = 0; + + /** + * Invoked when we finish receiving the body. + */ + virtual void onEOM() noexcept = 0; + + /** + * Invoked when request processing has been completed and nothing more + * needs to be done. This may be a good place to log some stats and + * clean up resources. This is distinct from onEOM() because it is + * invoked after the response is fully sent. Once this callback has been + * received, `downstream_` should be considered invalid. + */ + virtual void requestComplete() noexcept = 0; + + /** + * Request failed. Maybe because of read/write error on socket or client + * not being able to send request in time. + * + * NOTE: Can be invoked at any time (except for before onRequest). + * + * No more callbacks will be invoked after this. You should clean up after + * yourself. + */ + virtual void onError(ProxygenError err) noexcept = 0; + + /** + * Signals from HTTP layer when client queue is full or empty. If you are + * sending a streaming response, consider implementing these and acting + * accordingly. Saves your server from running out of memory. + */ + virtual void onEgressPaused() noexcept { + } + + virtual void onEgressResumed() noexcept { + } + + virtual ~RequestHandler() {} + + protected: + /** + * A place designated for the response handler. You can use this to send back + * the response in your RequestHandler. + */ + ResponseHandler* downstream_{nullptr}; +}; + +} diff --git a/proxygen/httpserver/RequestHandlerAdaptor.cpp b/proxygen/httpserver/RequestHandlerAdaptor.cpp new file mode 100644 index 0000000000..e1c1deb2e3 --- /dev/null +++ b/proxygen/httpserver/RequestHandlerAdaptor.cpp @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/httpserver/RequestHandlerAdaptor.h" + +#include "proxygen/httpserver/RequestHandler.h" +#include "proxygen/httpserver/ResponseBuilder.h" + +#include + +namespace proxygen { + +RequestHandlerAdaptor::RequestHandlerAdaptor(RequestHandler* requestHandler) + : ResponseHandler(requestHandler) { +} + +void RequestHandlerAdaptor::setTransaction(HTTPTransaction* txn) noexcept { + txn_ = txn; + + // We become that transparent layer + upstream_->setResponseHandler(this); +} + +void RequestHandlerAdaptor::detachTransaction() noexcept { + if (err_ == kErrorNone) { + upstream_->requestComplete(); + } + + // Otherwise we would have got some error call back and invoked onError + // on RequestHandler + delete this; +} + +void RequestHandlerAdaptor::onHeadersComplete(std::unique_ptr msg) + noexcept { + if (msg->getHeaders().exists(HTTP_HEADER_EXPECT)) { + auto expectation = msg->getHeaders().getSingleOrEmpty(HTTP_HEADER_EXPECT); + if (!boost::iequals(expectation, "100-continue")) { + setError(kErrorUnsupportedExpectation); + + ResponseBuilder(this) + .status(417, "Expectation Failed") + .closeConnection() + .sendWithEOM(); + } else { + ResponseBuilder(this) + .status(100, "Continue") + .send(); + + } + } + + // Only in case of no error + if (err_ == kErrorNone) { + upstream_->onRequest(std::move(msg)); + } +} + +void RequestHandlerAdaptor::onBody(std::unique_ptr c) noexcept { + upstream_->onBody(std::move(c)); +} + +void RequestHandlerAdaptor::onChunkHeader(size_t length) noexcept { +} + +void RequestHandlerAdaptor::onChunkComplete() noexcept { +} + +void RequestHandlerAdaptor::onTrailers(std::unique_ptr trailers) + noexcept { + // XXX: Support trailers +} + +void RequestHandlerAdaptor::onEOM() noexcept { + if (err_ == kErrorNone) { + upstream_->onEOM(); + } +} + +void RequestHandlerAdaptor::onUpgrade(UpgradeProtocol protocol) noexcept { + upstream_->onUpgrade(protocol); +} + +void RequestHandlerAdaptor::onError(const HTTPException& error) noexcept { + if (err_ != kErrorNone) { + // we have already handled an error and upstream would have been deleted + return; + } + + if (error.getProxygenError() == kErrorTimeout) { + setError(kErrorTimeout); + + if (responseStarted_) { + sendAbort(); + } else { + ResponseBuilder(this) + .status(408, "Request Timeout") + .closeConnection() + .sendWithEOM(); + } + } else if (error.getDirection() == HTTPException::Direction::INGRESS) { + setError(kErrorRead); + + if (responseStarted_) { + sendAbort(); + } else { + ResponseBuilder(this) + .status(400, "Bad Request") + .closeConnection() + .sendWithEOM(); + } + + } else { + setError(kErrorWrite); + } + + // Wait for detachTransaction to clean up +} + +void RequestHandlerAdaptor::onEgressPaused() noexcept { + upstream_->onEgressPaused(); +} + +void RequestHandlerAdaptor::onEgressResumed() noexcept { + upstream_->onEgressResumed(); +} + +void RequestHandlerAdaptor::sendHeaders(HTTPMessage& msg) noexcept { + responseStarted_ = true; + txn_->sendHeaders(msg); +} + +void RequestHandlerAdaptor::sendChunkHeader(size_t len) noexcept { + txn_->sendChunkHeader(len); +} + +void RequestHandlerAdaptor::sendBody(std::unique_ptr b) noexcept { + txn_->sendBody(std::move(b)); +} + +void RequestHandlerAdaptor::sendChunkTerminator() noexcept { + txn_->sendChunkTerminator(); +} + +void RequestHandlerAdaptor::sendEOM() noexcept { + txn_->sendEOM(); +} + +void RequestHandlerAdaptor::sendAbort() noexcept { + txn_->sendAbort(); +} + +void RequestHandlerAdaptor::refreshTimeout() noexcept { + txn_->refreshTimeout(); +} + +void RequestHandlerAdaptor::pauseIngress() noexcept { + txn_->pauseIngress(); +} + +void RequestHandlerAdaptor::resumeIngress() noexcept { + txn_->resumeIngress(); +} + +const TransportInfo& +RequestHandlerAdaptor::getSetupTransportInfo() const noexcept { + return txn_->getSetupTransportInfo(); +} + +void RequestHandlerAdaptor::getCurrentTransportInfo( + TransportInfo* tinfo) const { + txn_->getCurrentTransportInfo(tinfo); +} + +void RequestHandlerAdaptor::setError(ProxygenError err) noexcept { + err_ = err; + upstream_->onError(err); +} + +} diff --git a/proxygen/httpserver/RequestHandlerAdaptor.h b/proxygen/httpserver/RequestHandlerAdaptor.h new file mode 100644 index 0000000000..d7303d3a80 --- /dev/null +++ b/proxygen/httpserver/RequestHandlerAdaptor.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/ResponseHandler.h" +#include "proxygen/lib/http/session/HTTPTransaction.h" + +namespace proxygen { + +class RequestHandler; + +/** + * An adaptor that converts HTTPTransactionHandler to RequestHandler. + * Apart from that it also - + * + * - Handles all the error cases itself as described below. It makes a terminal + * call onError(...) where you are expected to log something and stop + * processing the request if you have started so. + * + * onError - Send a direct response back if no response has started and + * writing is still possible. Otherwise sends an abort. + * + * - Handles 100-continue case for you (by sending Continue response) + */ +class RequestHandlerAdaptor + : public HTTPTransactionHandler, + public ResponseHandler { + public: + explicit RequestHandlerAdaptor(RequestHandler* requestHandler); + + private: + // HTTPTransactionHandler + void setTransaction(HTTPTransaction* txn) noexcept override; + void detachTransaction() noexcept override; + void onHeadersComplete(std::unique_ptr msg) noexcept override; + void onBody(std::unique_ptr chain) noexcept override; + void onChunkHeader(size_t length) noexcept override; + void onChunkComplete() noexcept override; + void onTrailers(std::unique_ptr trailers) noexcept override; + void onEOM() noexcept override; + void onUpgrade(UpgradeProtocol protocol) noexcept override; + void onError(const HTTPException& error) noexcept override; + void onEgressPaused() noexcept override; + void onEgressResumed() noexcept override; + + // ResponseHandler + void sendHeaders(HTTPMessage& msg) noexcept override; + void sendChunkHeader(size_t len) noexcept override; + void sendBody(std::unique_ptr body) noexcept override; + void sendChunkTerminator() noexcept override; + void sendEOM() noexcept override; + void sendAbort() noexcept override; + void refreshTimeout() noexcept override; + void pauseIngress() noexcept override; + void resumeIngress() noexcept override; + const TransportInfo& getSetupTransportInfo() const noexcept override; + void getCurrentTransportInfo(TransportInfo* tinfo) const override; + + // Helper method + void setError(ProxygenError err) noexcept; + + HTTPTransaction* txn_{nullptr}; + ProxygenError err_{kErrorNone}; + bool responseStarted_{false}; +}; + +} diff --git a/proxygen/httpserver/RequestHandlerFactory.h b/proxygen/httpserver/RequestHandlerFactory.h new file mode 100644 index 0000000000..a1a1c2d443 --- /dev/null +++ b/proxygen/httpserver/RequestHandlerFactory.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/RequestHandler.h" + +namespace proxygen { + +/** + * Factory for RequestHandlers. + */ +class RequestHandlerFactory { + public: + virtual ~RequestHandlerFactory() {} + + /** + * Invoked in each thread server is going to handle requests + * before we start handling requests. Can be used to setup + * thread-local setup for each thread (stats and such). + */ + virtual void onServerStart() noexcept = 0; + + /** + * Invoked in each handler thread after all the connections are + * drained from that thread. Can be used to tear down thread-local setup. + */ + virtual void onServerStop() noexcept = 0; + + /** + * Invoked for each new request server handles. HTTPMessage is provided + * so that user can potentially choose among several implementation of + * handler based on URL or something. No, need to save/copy this + * HTTPMessage. RequestHandler will be given the HTTPMessage + * in a separate callback. + * + * Some request handlers don't handle the request themselves (think filters). + * They do take some actions based on request/response but otherwise they + * just hand-off request to some other RequestHandler. This upstream + * RequestHandler is given as first parameter. For the terminal RequestHandler + * this will by nullptr. + */ + virtual RequestHandler* onRequest(RequestHandler*, HTTPMessage*) noexcept = 0; +}; + +/** + * Helper class to help beautify the way we make chains of these filters + */ +class RequestHandlerChain { + public: + std::vector> build() { + return std::move(chain_); + } + + template + RequestHandlerChain& addThen(Args&&... args) { + chain_.push_back(folly::make_unique(std::forward(args)...)); + return *this; + } + + RequestHandlerChain& addThen(std::unique_ptr h) { + chain_.push_back(std::move(h)); + return *this; + } + + private: + std::vector> chain_; +}; + +} diff --git a/proxygen/httpserver/ResponseBuilder.h b/proxygen/httpserver/ResponseBuilder.h new file mode 100644 index 0000000000..786c0c3bf1 --- /dev/null +++ b/proxygen/httpserver/ResponseBuilder.h @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/ResponseHandler.h" + +#include + +namespace proxygen { + +/** + * Helps you make responses and send them on demand. + * + * NOTE: We don't do any correctness checks here, we depend on + * state machine in HTTPTransaction to tell us when an + * error occurs + * + * Three expected use cases are + * + * 1. Send all response at once. If this is an error + * response, most probably you also want 'closeConnection'. + * + * ResponseBuilder(handler) + * .status(200, "OK") + * .body(...) + * .sendWithEOM(); + * + * 2. Sending back response in chunks. + * + * ResponseBuilder(handler) + * .status(200, "OK") + * .body(...) + * .send(); // Without `WithEOM` we make it chunked + * + * 1 or more time + * + * ResponseBuilder(handler) + * .body(...) + * .send(); + * + * At last + * + * ResponseBuilder(handler) + * .body(...) + * .sendWithEOM(); + * + * 3. Accept or reject Upgrade Requests + * + * ResponseBuilder(handler) + * .acceptUpgradeRequest() // send '200 OK' without EOM + * + * or + * + * ResponseBuilder(handler) + * .rejectUpgradeRequest() // send '400 Bad Request' + * + */ +class ResponseBuilder { + public: + explicit ResponseBuilder(ResponseHandler* txn): txn_(txn) { + } + + ResponseBuilder& status(uint16_t code, std::string message) { + headers_ = folly::make_unique(); + headers_->setHTTPVersion(1, 1); + headers_->setStatusCode(code); + headers_->setStatusMessage(message); + return *this; + } + + template + ResponseBuilder& header(const std::string& headerIn, const T& value) { + CHECK(headers_) << "You need to call `status` before adding headers"; + headers_->getHeaders().add(headerIn, value); + return *this; + } + + template + ResponseBuilder& header(HTTPHeaderCode code, const T& value) { + CHECK(headers_) << "You need to call `status` before adding headers"; + headers_->getHeaders().add(code, value); + return *this; + } + + ResponseBuilder& body(std::unique_ptr bodyIn) { + if (bodyIn) { + if (body_) { + body_->prependChain(std::move(bodyIn)); + } else { + body_ = std::move(bodyIn); + } + } + + return *this; + } + + template + ResponseBuilder& body(T&& t) { + return body(folly::IOBuf::maybeCopyBuffer( + folly::to(std::forward(t)))); + } + + ResponseBuilder& closeConnection() { + return header(HTTP_HEADER_CONNECTION, "close"); + } + + void sendWithEOM() { + sendEOM_ = true; + send(); + } + + void send() { + // Once we send them, we don't want to send them again + SCOPE_EXIT { headers_.reset(); }; + + // By default, chunked + bool chunked = true; + + // If we have complete response, we can use Content-Length and get done + if (headers_ && sendEOM_) { + chunked = false; + } + + if (headers_) { + // We don't need to add Content-Length or Encoding for 1xx responses + if (headers_->getStatusCode() >= 200) { + if (chunked) { + headers_->setIsChunked(true); + } else { + const auto len = body_ ? body_->computeChainDataLength() : 0; + headers_->getHeaders().add( + HTTP_HEADER_CONTENT_LENGTH, + folly::to(len)); + } + } + + txn_->sendHeaders(*headers_); + } + + if (body_) { + if (chunked) { + txn_->sendChunkHeader(body_->computeChainDataLength()); + txn_->sendBody(std::move(body_)); + txn_->sendChunkTerminator(); + } else { + txn_->sendBody(std::move(body_)); + } + } + + if (sendEOM_) { + txn_->sendEOM(); + } + } + + enum class UpgradeType { + CONNECT_REQUEST = 0, + HTTP_UPGRADE, + }; + + void acceptUpgradeRequest(UpgradeType upgradeType, + const std::string upgradeProtocol = "") { + headers_ = folly::make_unique(); + if (upgradeType == UpgradeType::CONNECT_REQUEST) { + headers_->constructDirectResponse({1, 1}, 200, "OK"); + } else { + CHECK(!upgradeProtocol.empty()); + headers_->constructDirectResponse({1, 1}, 101, "Switching Protocols"); + headers_->getHeaders().add(HTTP_HEADER_UPGRADE, upgradeProtocol); + headers_->getHeaders().add(HTTP_HEADER_CONNECTION, "Upgrade"); + } + txn_->sendHeaders(*headers_); + } + + void rejectUpgradeRequest() { + headers_ = folly::make_unique(); + headers_->constructDirectResponse({1, 1}, 400, "Bad Request"); + txn_->sendHeaders(*headers_); + txn_->sendEOM(); + } + + private: + ResponseHandler* const txn_{nullptr}; + + std::unique_ptr headers_; + std::unique_ptr body_; + + // If true, sends EOM. + bool sendEOM_{false}; +}; + +} diff --git a/proxygen/httpserver/ResponseHandler.h b/proxygen/httpserver/ResponseHandler.h new file mode 100644 index 0000000000..cfadf33ddc --- /dev/null +++ b/proxygen/httpserver/ResponseHandler.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPTransaction.h" + +namespace proxygen { + +class RequestHandler; + +/** + * Interface that acts as client for RequestHandler. It also has a hook + * for the RequestHandler so that it is easy to chain these Request/Response + * handlers and be able to modify these chains. + * + * The names are pretty much self explanatory. You only need + * to get into details about this interface if you are implementing filters. + * + * NOTE: All the writes are done at the end of the event loop. So this is safe + * to do in your RequestHandler. + * + * { + * ... + * downstream_->sendHeader(...); + * downstream_->sendEOM(); + * } + * + * You dont need to worry about any callbacks being invoked after + * sendHeader. + * + * Consider using proxygen/httpserver/ResponseBuilder to send back the + * response. It will take care of chunking response if required and + * everything. + */ +class ResponseHandler { + public: + explicit ResponseHandler(RequestHandler* upstream) + : upstream_(CHECK_NOTNULL(upstream)) { + } + + virtual ~ResponseHandler() {} + + /** + * NOTE: We take response message as non-const reference, to allow filters + * between your handler and client to be able to modify response + * if they want to. + * + * eg. a compression filter might want to change the content-encoding + */ + virtual void sendHeaders(HTTPMessage& msg) noexcept = 0; + + virtual void sendChunkHeader(size_t len) noexcept = 0; + + virtual void sendBody(std::unique_ptr body) noexcept = 0; + + virtual void sendChunkTerminator() noexcept = 0; + + virtual void sendEOM() noexcept = 0; + + virtual void sendAbort() noexcept = 0; + + virtual void refreshTimeout() noexcept = 0; + + virtual void pauseIngress() noexcept = 0; + + virtual void resumeIngress() noexcept = 0; + + // Accessors for Transport/Connection information + virtual const TransportInfo& getSetupTransportInfo() const noexcept = 0; + + virtual void getCurrentTransportInfo(TransportInfo* tinfo) const = 0; + + protected: + RequestHandler* upstream_{nullptr}; +}; + +} diff --git a/proxygen/httpserver/ScopedHTTPServer.h b/proxygen/httpserver/ScopedHTTPServer.h new file mode 100644 index 0000000000..5f5376ce2d --- /dev/null +++ b/proxygen/httpserver/ScopedHTTPServer.h @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/HTTPServer.h" +#include "proxygen/httpserver/ResponseBuilder.h" + +#include +#include + +namespace proxygen { + +template +class ScopedHandler : public RequestHandler { + public: + explicit ScopedHandler(HandlerType* ptr): handlerPtr_(ptr) { + } + + void onRequest(std::unique_ptr headers) noexcept { + request_ = std::move(headers); + } + + void onBody(std::unique_ptr body) noexcept { + requestBody_.append(std::move(body)); + } + + void onUpgrade(proxygen::UpgradeProtocol proto) noexcept { + } + + void onEOM() noexcept { + try { + ResponseBuilder r(downstream_); + (*handlerPtr_)(*request_, requestBody_.move(), r); + r.sendWithEOM(); + } catch (const std::exception& ex) { + ResponseBuilder(downstream_) + .status(500, "Internal Server Error") + .body(ex.what()) + .sendWithEOM(); + } catch (...) { + ResponseBuilder(downstream_) + .status(500, "Internal Server Error") + .body("Unknown exception thrown") + .sendWithEOM(); + } + } + + void requestComplete() noexcept { + delete this; + } + + void onError(ProxygenError err) noexcept { + delete this; + } + private: + HandlerType* const handlerPtr_{nullptr}; + + std::unique_ptr request_; + folly::IOBufQueue requestBody_; +}; + +template +class ScopedHandlerFactory : public RequestHandlerFactory { + public: + explicit ScopedHandlerFactory(HandlerType handler): handler_(handler) { + } + + void onServerStart() noexcept override { + } + + void onServerStop() noexcept override { + } + + RequestHandler* onRequest(RequestHandler*, HTTPMessage*) noexcept { + return new ScopedHandler(&handler_); + } + private: + HandlerType handler_; +}; + +/** + * A basic server that can be used for testing http clients. Since most such + * servers are short lived, this server takes care of starting and stopping + * automatically. + */ +class ScopedHTTPServer final { + public: + /** + * Start a server listening on the requested `port`. + * If `port` is 0, it will choose a random port. + */ + template + static std::unique_ptr start(HandlerType handler, + int port = 0, + int numThreads = 4); + + /** + * Get the port the server is listening on. This is helpful if the port was + * randomly chosen. + */ + int getPort() const { + auto addresses = server_->addresses(); + CHECK(!addresses.empty()); + return addresses[0].address.getPort(); + } + + ~ScopedHTTPServer() { + server_->stop(); + thread_.join(); + } + + private: + ScopedHTTPServer(std::thread thread, + std::unique_ptr server) + : thread_(std::move(thread)), + server_(std::move(server)) { + } + + std::thread thread_; + std::unique_ptr server_; +}; + +template +inline std::unique_ptr ScopedHTTPServer::start( + HandlerType handler, + int port, + int numThreads) { + + std::unique_ptr f = + folly::make_unique>(handler); + return start(std::move(f), port, numThreads); +} + +template <> +inline std::unique_ptr +ScopedHTTPServer::start>( + std::unique_ptr f, + int port, + int numThreads) { + + // This will handle both IPv4 and IPv6 cases + folly::SocketAddress addr; + addr.setFromLocalPort(port); + + std::vector IPs = { + {addr, HTTPServer::Protocol::HTTP} + }; + + HTTPServerOptions options; + options.threads = numThreads; + + options.handlerFactories.push_back(std::move(f)); + + auto server = folly::make_unique(std::move(options)); + server->bind(IPs); + + // Start the server + std::exception_ptr eptr; + auto barrier = std::make_shared(2); + + std::thread t = std::thread([&, barrier] () { + server->start( + [&, barrier] () { + folly::setThreadName("http-acceptor"); + barrier->wait(); + }, + [&, barrier] (std::exception_ptr ex) { + eptr = ex; + barrier->wait(); + }); + }); + + // Wait for server to start + barrier->wait(); + if (eptr) { + t.join(); + + std::rethrow_exception(eptr); + } + + return std::unique_ptr( + new ScopedHTTPServer(std::move(t), std::move(server))); +} + +} diff --git a/proxygen/httpserver/SignalHandler.cpp b/proxygen/httpserver/SignalHandler.cpp new file mode 100644 index 0000000000..f0b31b12f0 --- /dev/null +++ b/proxygen/httpserver/SignalHandler.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/httpserver/SignalHandler.h" + +#include "proxygen/httpserver/HTTPServer.h" + +#include +#include + +using folly::EventBaseManager; + +namespace proxygen { + +SignalHandler::SignalHandler(HTTPServer* server) + : TAsyncSignalHandler(EventBaseManager::get()->getEventBase()), + server_(server) { +} + +void SignalHandler::install(const std::vector& signals) { + for (const int& signal: signals) { + registerSignalHandler(signal); + } +} + +void SignalHandler::signalReceived(int signum) noexcept { + server_->stop(); +} + +} diff --git a/proxygen/httpserver/SignalHandler.h b/proxygen/httpserver/SignalHandler.h new file mode 100644 index 0000000000..c35e4e5a52 --- /dev/null +++ b/proxygen/httpserver/SignalHandler.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +class HTTPServer; + +/** + * Installs signal handler which will stop HTTPServer when the user presses + * Ctrl-C. To be used if HTTPServer is the main process. + * + * Note: Should only be created from the thread invoking `HTTPServer::start()`. + */ +class SignalHandler: private apache::thrift::async::TAsyncSignalHandler { + public: + explicit SignalHandler(HTTPServer* server); + + void install(const std::vector& signals); + private: + // TAsyncSignalHandler + void signalReceived(int signum) noexcept override; + + HTTPServer* const server_{nullptr}; +}; + +} diff --git a/proxygen/httpserver/filters/DirectResponseHandler.h b/proxygen/httpserver/filters/DirectResponseHandler.h new file mode 100644 index 0000000000..a082d82e1a --- /dev/null +++ b/proxygen/httpserver/filters/DirectResponseHandler.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/RequestHandler.h" + +namespace proxygen { + +/** + * Handler that sends back a fixed response back. + */ +class DirectResponseHandler : public RequestHandler { + public: + DirectResponseHandler(int code, + std::string message, + std::string body) + : code_(code), + message_(message), + body_(folly::IOBuf::copyBuffer(body)) { + } + + void onRequest(std::unique_ptr headers) noexcept override { + } + + void onBody(std::unique_ptr body) noexcept override { + } + + void onUpgrade(proxygen::UpgradeProtocol prot) noexcept override { + } + + void onEOM() noexcept override { + ResponseBuilder(downstream_) + .status(code_, message_) + .body(std::move(body_)) + .sendWithEOM(); + } + + void requestComplete() noexcept override { + delete this; + } + + void onError(ProxygenError err) noexcept override { + delete this; + } + + private: + const int code_; + const std::string message_; + std::unique_ptr body_; +}; + +} diff --git a/proxygen/httpserver/filters/RejectConnectFilter.h b/proxygen/httpserver/filters/RejectConnectFilter.h new file mode 100644 index 0000000000..9fe42953c2 --- /dev/null +++ b/proxygen/httpserver/filters/RejectConnectFilter.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/Filters.h" +#include "proxygen/httpserver/RequestHandlerFactory.h" +#include "proxygen/httpserver/ResponseBuilder.h" + +namespace proxygen { + +/** + * A filter that rejects CONNECT/UPGRADE requests. + */ +class RejectConnectFilter : public Filter { + public: + explicit RejectConnectFilter(RequestHandler* upstream): Filter(upstream) { + } + + void onRequest(std::unique_ptr msg) noexcept override { + upstream_->onError(kErrorMethodNotSupported); + upstream_ = nullptr; + + ResponseBuilder(downstream_).rejectUpgradeRequest(); + } + + void onBody(std::unique_ptr body) noexcept override { + } + + void onUpgrade(UpgradeProtocol protocol) noexcept override { + } + + void onEOM() noexcept override { + } + + void requestComplete() noexcept override { + CHECK(!upstream_); + delete this; + } + + void onError(ProxygenError err) noexcept override { + // If onError is invoked before we forward the error + if (upstream_) { + upstream_->onError(err); + upstream_ = nullptr; + } + + delete this; + } + + void onEgressPaused() noexcept override { + } + + void onEgressResumed() noexcept override { + } + + // Response handler + void sendHeaders(HTTPMessage& msg) noexcept override { + } + + void sendChunkHeader(size_t len) noexcept override { + } + + void sendBody(std::unique_ptr body) noexcept override { + } + + void sendChunkTerminator() noexcept override { + } + + void sendEOM() noexcept override { + } + + void sendAbort() noexcept override { + } + + void refreshTimeout() noexcept override { + } +}; + +class RejectConnectFilterFactory : public RequestHandlerFactory { + public: + void onServerStart() noexcept override { + } + + void onServerStop() noexcept override { + } + + RequestHandler* onRequest(RequestHandler* h, HTTPMessage* msg) + noexcept override { + + if (msg->getMethod() == HTTPMethod::CONNECT) { + return new RejectConnectFilter(h); + } + + // No need to insert this filter + return h; + } +}; + +} diff --git a/proxygen/httpserver/samples/Makefile.am b/proxygen/httpserver/samples/Makefile.am new file mode 100644 index 0000000000..27b2a7478e --- /dev/null +++ b/proxygen/httpserver/samples/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = echo diff --git a/proxygen/httpserver/samples/echo/EchoHandler.cpp b/proxygen/httpserver/samples/echo/EchoHandler.cpp new file mode 100644 index 0000000000..9e685e3d2e --- /dev/null +++ b/proxygen/httpserver/samples/echo/EchoHandler.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "EchoHandler.h" + +#include "EchoStats.h" +#include "proxygen/httpserver/RequestHandler.h" +#include "proxygen/httpserver/ResponseBuilder.h" + +using namespace proxygen; + +namespace EchoService { + +EchoHandler::EchoHandler(EchoStats* stats): stats_(stats) { +} + +void EchoHandler::onRequest(std::unique_ptr headers) noexcept { + stats_->recordRequest(); +} + +void EchoHandler::onBody(std::unique_ptr body) noexcept { + if (body_) { + body_->prependChain(std::move(body)); + } else { + body_ = std::move(body); + } +} + +void EchoHandler::onEOM() noexcept { + ResponseBuilder(downstream_) + .status(200, "OK") + .header("Request-Number", + folly::to(stats_->getRequestCount())) + .body(std::move(body_)) + .sendWithEOM(); +} + +void EchoHandler::onUpgrade(UpgradeProtocol protocol) noexcept { + // handler doesn't support upgrades +} + +void EchoHandler::requestComplete() noexcept { + delete this; +} + +void EchoHandler::onError(ProxygenError err) noexcept { + delete this; +} + +} diff --git a/proxygen/httpserver/samples/echo/EchoHandler.h b/proxygen/httpserver/samples/echo/EchoHandler.h new file mode 100644 index 0000000000..57a168f346 --- /dev/null +++ b/proxygen/httpserver/samples/echo/EchoHandler.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/httpserver/RequestHandler.h" + +#include + +namespace proxygen { +class ResponseHandler; +} + +namespace EchoService { + +class EchoStats; + +class EchoHandler : public proxygen::RequestHandler { + public: + explicit EchoHandler(EchoStats* stats); + + void onRequest(std::unique_ptr headers) + noexcept override; + + void onBody(std::unique_ptr body) noexcept override; + + void onEOM() noexcept override; + + void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override; + + void requestComplete() noexcept override; + + void onError(proxygen::ProxygenError err) noexcept override; + + private: + EchoStats* const stats_{nullptr}; + + std::unique_ptr body_; +}; + +} diff --git a/proxygen/httpserver/samples/echo/EchoServer.cpp b/proxygen/httpserver/samples/echo/EchoServer.cpp new file mode 100644 index 0000000000..54f55511da --- /dev/null +++ b/proxygen/httpserver/samples/echo/EchoServer.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "EchoHandler.h" +#include "EchoStats.h" +#include "proxygen/httpserver/HTTPServer.h" +#include "proxygen/httpserver/RequestHandlerFactory.h" + +#include +#include +#include + +using namespace EchoService; +using namespace proxygen; + +using folly::EventBase; +using folly::EventBaseManager; +using folly::SocketAddress; + +using Protocol = HTTPServer::Protocol; + +DEFINE_int32(http_port, 11000, "Port to listen on with HTTP protocol"); +DEFINE_int32(spdy_port, 11001, "Port to listen on with SPDY protocol"); +DEFINE_int32(thrift_port, 10000, "Port to listen on for thrift"); +DEFINE_string(ip, "localhost", "IP/Hostname to bind to"); +DEFINE_int32(threads, 0, "Number of threads to listen on. Numbers <= 0 " + "will use the number of cores on this machine."); + +class EchoHandlerFactory : public RequestHandlerFactory { + public: + void onServerStart() noexcept override { + stats_.reset(new EchoStats); + } + + void onServerStop() noexcept override { + stats_.reset(); + } + + RequestHandler* onRequest(RequestHandler*, HTTPMessage*) noexcept override { + return new EchoHandler(stats_.get()); + } + + private: + folly::ThreadLocalPtr stats_; +}; + +int main(int argc, char* argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + google::InstallFailureSignalHandler(); + + std::vector IPs = { + {SocketAddress(FLAGS_ip, FLAGS_http_port, true), Protocol::HTTP}, + {SocketAddress(FLAGS_ip, FLAGS_spdy_port, true), Protocol::SPDY}, + }; + + if (FLAGS_threads <= 0) { + FLAGS_threads = sysconf(_SC_NPROCESSORS_ONLN); + CHECK(FLAGS_threads > 0); + } + + HTTPServerOptions options; + options.threads = static_cast(FLAGS_threads); + options.idleTimeout = std::chrono::milliseconds(60000); + options.shutdownOn = {SIGINT, SIGTERM}; + options.handlerFactories = RequestHandlerChain() + .addThen() + .build(); + + HTTPServer server(std::move(options)); + server.bind(IPs); + + // Start HTTPServer mainloop in a separate thread + std::thread t([&] () { + server.start(); + }); + + t.join(); + return 0; +} diff --git a/proxygen/httpserver/samples/echo/EchoStats.h b/proxygen/httpserver/samples/echo/EchoStats.h new file mode 100644 index 0000000000..fbf825f6f5 --- /dev/null +++ b/proxygen/httpserver/samples/echo/EchoStats.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace EchoService { + +/** + * Just some dummy class containing request count. Since we keep + * one instance of this in each class, there is no need of + * synchronization + */ +class EchoStats { + public: + virtual ~EchoStats() { + } + + // NOTE: We make the following methods `virtual` so that we can + // mock them using Gmock for our C++ unit-tests. EchoStats + // is an external dependency to handler and we should be + // able to mock it. + + virtual void recordRequest() { + ++reqCount_; + } + + virtual uint64_t getRequestCount() { + return reqCount_; + } + + private: + uint64_t reqCount_{0}; +}; + +} diff --git a/proxygen/httpserver/samples/echo/Makefile.am b/proxygen/httpserver/samples/echo/Makefile.am new file mode 100644 index 0000000000..d7a1902b71 --- /dev/null +++ b/proxygen/httpserver/samples/echo/Makefile.am @@ -0,0 +1,10 @@ +SUBDIRS = . + +noinst_PROGRAMS = echo_server + +echo_server_SOURCES = \ + EchoHandler.cpp \ + EchoServer.cpp + +echo_server_LDADD = \ + ../../libproxygenhttpserver.la diff --git a/proxygen/httpserver/samples/echo/test/EchoHandlerTest.cpp b/proxygen/httpserver/samples/echo/test/EchoHandlerTest.cpp new file mode 100644 index 0000000000..210a36b5d6 --- /dev/null +++ b/proxygen/httpserver/samples/echo/test/EchoHandlerTest.cpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/httpserver/Mocks.h" +#include "proxygen/httpserver/samples/echo/EchoHandler.h" +#include "proxygen/httpserver/samples/echo/EchoStats.h" + +#include +#include + +using namespace EchoService; +using namespace proxygen; +using namespace testing; + +class MockEchoStats : public EchoStats { + public: + MOCK_METHOD0(recordRequest, void()); + MOCK_METHOD0(getRequestCount, uint64_t()); +}; + +class EchoHandlerFixture : public testing::Test { + public: + void SetUp() { + handler = new EchoHandler(&stats); + responseHandler = folly::make_unique(handler); + handler->setResponseHandler(responseHandler.get()); + } + + void TearDown() { + Mock::VerifyAndClear(&stats); + Mock::VerifyAndClear(responseHandler.get()); + + // Since there is no easy way to verify that handler has deleted + // itself, its advised to run test binary under AddressSanitzer + // or valgrind to verify that. + } + + protected: + EchoHandler* handler{nullptr}; + StrictMock stats; + std::unique_ptr responseHandler; +}; + +TEST_F(EchoHandlerFixture, OnProperRequestSendsResponse) { + EXPECT_CALL(stats, recordRequest()).WillOnce(Return()); + EXPECT_CALL(stats, getRequestCount()).WillOnce(Return(5)); + + HTTPMessage response; + EXPECT_CALL(*responseHandler, sendHeaders(_)).WillOnce( + DoAll(SaveArg<0>(&response), Return())); + EXPECT_CALL(*responseHandler, sendEOM()).WillOnce(Return()); + + // Since we know we dont touch request, its ok to pass `nullptr` here. + handler->onRequest(nullptr); + handler->onEOM(); + handler->requestComplete(); + + EXPECT_EQ("5", response.getHeaders().getSingleOrEmpty("Request-Number")); + EXPECT_EQ(200, response.getStatusCode()); +} + +TEST_F(EchoHandlerFixture, ReplaysBodyProperly) { + EXPECT_CALL(stats, recordRequest()).WillOnce(Return()); + EXPECT_CALL(stats, getRequestCount()).WillOnce(Return(5)); + + HTTPMessage response; + folly::fbstring body; + + EXPECT_CALL(*responseHandler, sendHeaders(_)).WillOnce( + DoAll(SaveArg<0>(&response), Return())); + + EXPECT_CALL(*responseHandler, sendBody(_)).WillRepeatedly( + DoAll( + Invoke([&] (std::shared_ptr b) { + body += b->moveToFbString(); + }), + Return())); + + EXPECT_CALL(*responseHandler, sendEOM()).WillOnce(Return()); + + // Since we know we dont touch request, its ok to pass `nullptr` here. + handler->onRequest(nullptr); + handler->onBody(folly::IOBuf::copyBuffer("part1")); + handler->onBody(folly::IOBuf::copyBuffer("part2")); + handler->onEOM(); + handler->requestComplete(); + + EXPECT_EQ("5", response.getHeaders().getSingleOrEmpty("Request-Number")); + EXPECT_EQ(200, response.getStatusCode()); + EXPECT_EQ("part1part2", body); +} diff --git a/proxygen/httpserver/tests/HTTPServerTest.cpp b/proxygen/httpserver/tests/HTTPServerTest.cpp new file mode 100644 index 0000000000..c04efbde23 --- /dev/null +++ b/proxygen/httpserver/tests/HTTPServerTest.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/httpserver/HTTPServer.h" +#include "proxygen/httpserver/ResponseBuilder.h" +#include "proxygen/lib/utils/TestUtils.h" + +#include +#include +#include +#include + +using namespace proxygen; +using namespace testing; + +using apache::thrift::async::TAsyncSSLSocket; +using apache::thrift::transport::SSLContext; +using folly::AsyncServerSocket; +using folly::EventBaseManager; +using folly::SocketAddress; + +namespace { + +const std::string kTestDir = getContainingDirectory(__FILE__).str(); + +} + +class ServerThread { + private: + boost::barrier barrier_{2}; + std::thread t_; + HTTPServer* server_{nullptr}; + + public: + + explicit ServerThread(HTTPServer* server) : server_(server) {} + ~ServerThread() { + if (server_) { + server_->stop(); + } + t_.join(); + } + + bool start() { + bool throws = false; + t_ = std::thread([&] () { + server_->start( + [&] () { + barrier_.wait(); + }, + [&] (std::exception_ptr ex) { + throws = true; + server_ = nullptr; + barrier_.wait(); + }); + }); + barrier_.wait(); + return !throws; + } +}; + +TEST(MultiBind, HandlesListenFailures) { + SocketAddress addr("127.0.0.1", 0); + + auto evb = EventBaseManager::get()->getEventBase(); + AsyncServerSocket::UniquePtr socket(new AsyncServerSocket(evb)); + socket->bind(addr); + + // Get the ephemeral port + socket->getAddress(&addr); + int port = addr.getPort(); + + std::vector ips = { + { + folly::SocketAddress("127.0.0.1", port), + HTTPServer::Protocol::HTTP + } + }; + + HTTPServerOptions options; + options.threads = 4; + + auto server = folly::make_unique(std::move(options)); + + // We have to bind both the sockets before listening on either + server->bind(ips); + + // On kernel 2.6 trying to listen on a FD that another socket + // has bound to fails. While in kernel 3.2 only when one socket tries + // to listen on a FD that another socket is listening on fails. + try { + socket->listen(1024); + } catch (const std::exception& ex) { + return; + } + + ServerThread st(server.get()); + EXPECT_FALSE(st.start()); +} + +TEST(SSL, SSLTest) { + HTTPServer::IPConfig cfg{ + folly::SocketAddress("127.0.0.1", 0), + HTTPServer::Protocol::HTTP}; + SSLContextConfig sslCfg; + sslCfg.isDefault = true; + sslCfg.setCertificate( + kTestDir + "certs/test_cert1.pem", + kTestDir + "certs/test_key1.pem", + ""); + cfg.sslConfigs.push_back(sslCfg); + + HTTPServerOptions options; + options.threads = 4; + + auto server = folly::make_unique(std::move(options)); + + std::vector ips{cfg}; + server->bind(ips); + + ServerThread st(server.get()); + EXPECT_TRUE(st.start()); + + // Make an SSL connection to the server + class Cb : public apache::thrift::async::TAsyncSocket::ConnectCallback { + public: + explicit Cb(TAsyncSSLSocket* sock) : sock_(sock) {} + void connectSuccess() noexcept override { + success = true; + sock_->close(); + } + void connectError(const apache::thrift::transport::TTransportException&) + noexcept override { + success = false; + } + + bool success{false}; + TAsyncSSLSocket* sock_{nullptr}; + }; + + folly::EventBase evb; + auto ctx = std::make_shared(); + TAsyncSSLSocket::UniquePtr sock(new TAsyncSSLSocket(ctx, &evb)); + Cb cb(sock.get()); + sock->connect(&cb, server->addresses().front().address, 1000); + evb.loop(); + EXPECT_TRUE(cb.success); +} diff --git a/proxygen/httpserver/tests/Makefile.am b/proxygen/httpserver/tests/Makefile.am new file mode 100644 index 0000000000..99267f3ae9 --- /dev/null +++ b/proxygen/httpserver/tests/Makefile.am @@ -0,0 +1,11 @@ +SUBDIRS = . + +check_PROGRAMS = HTTPServerTests +HTTPServerTests_SOURCES = \ + HTTPServerTest.cpp + +HTTPServerTests_LDADD = \ + ../libproxygenhttpserver.la \ + ../../lib/test/libtestmain.la + +TESTS = HTTPServerTests diff --git a/proxygen/httpserver/tests/certs/ca_cert.pem b/proxygen/httpserver/tests/certs/ca_cert.pem new file mode 100644 index 0000000000..2a9649ccd9 --- /dev/null +++ b/proxygen/httpserver/tests/certs/ca_cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDWTCCAkGgAwIBAgIJAP6wIodJOj4eMA0GCSqGSIb3DQEBBQUAMEMxCzAJBgNV +BAYTAlVTMQ0wCwYDVQQKDARBc294MSUwIwYDVQQDDBxBc294IENlcnRpZmljYXRp +b24gQXV0aG9yaXR5MB4XDTE0MDkxOTIyMTM0NloXDTQyMDIwNDIyMTM0NlowQzEL +MAkGA1UEBhMCVVMxDTALBgNVBAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlm +aWNhdGlvbiBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQC2Sk5jEnq4NcoVCVDKrepazJfntSPcgL7DIQnGA3jH4+PA6oCUhZZtKw8R6EL2 +rHpqwh1jQoZKEgxnj9Uf2VuS7cjRoCJxNh8l+XYqNZfNwUrno6UlHzu6Yd9ki6E2 +PqK5xLVM/aem1/fAjlcSFZfua4BvtAqW3JqpafWpigSp8n7QckQSDW9tA8Fg2EX0 +CG+x3TieYEiA2Vr5493sir/cmDY4D1aBiPPlkRSeW21ZMLN0JDUFc2NgfquFTyfg +vP4EKj/XZvwo1oR6fUsPIUr9S3LL2tiJEtIiiQPl/kKp8f6NiWMZKh630BfPcFkq +fpTj3XjsIavD7t3e2jQ/73C9AgMBAAGjUDBOMB0GA1UdDgQWBBR1I164CbUAkL2q +gbLyppcm6ODFmjAfBgNVHSMEGDAWgBR1I164CbUAkL2qgbLyppcm6ODFmjAMBgNV +HRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQCwIQ692yKqSKjGwLthRWyYtwAC +wH4YTX9LERF+CwzkpKcFlkg0g4rfGHP3qwRn0F9x485Hny8Yq9My7MWko9Oqu5Im +N62ixU7LJE+S0MOHbCcfCgVVaaUL7y0Nx3zYyPdURmyRjou534O9/BVjZkU+1S+L +iljuShPcJc3TxNc26s912RkTN5QZkVUOmamP+by5cK+yb3ja2ymIFipO9VuuBixo +s9WcsgaDWeKCaOIp3xLd5Tnm5Q5R1QbEe4okr+5CLF49C23Oei2sR6/VsxQwG2rC +cmdgbTFUDrz2lmwxh/YI0G60QFllCPVWkJFxyNJJ+0duxUfT4jLRzUKNgGrw +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/ca_key.pem b/proxygen/httpserver/tests/certs/ca_key.pem new file mode 100644 index 0000000000..8ce874cc3e --- /dev/null +++ b/proxygen/httpserver/tests/certs/ca_key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAtkpOYxJ6uDXKFQlQyq3qWsyX57Uj3IC+wyEJxgN4x+PjwOqA +lIWWbSsPEehC9qx6asIdY0KGShIMZ4/VH9lbku3I0aAicTYfJfl2KjWXzcFK56Ol +JR87umHfZIuhNj6iucS1TP2nptf3wI5XEhWX7muAb7QKltyaqWn1qYoEqfJ+0HJE +Eg1vbQPBYNhF9Ahvsd04nmBIgNla+ePd7Iq/3Jg2OA9WgYjz5ZEUnlttWTCzdCQ1 +BXNjYH6rhU8n4Lz+BCo/12b8KNaEen1LDyFK/Utyy9rYiRLSIokD5f5CqfH+jYlj +GSoet9AXz3BZKn6U49147CGrw+7d3to0P+9wvQIDAQABAoIBAHKltTs2KhylJ92n +KDrwus40ku1VzaInDtMmekEhedsuBtYUJp5CjmNGi4nVrBf8TlnKkDUXZ+I6C7cu +jPok+CUmjADbWA4f3eNCTAEsB7eOdA/PqlP4mtYULC3Oa6v0JN/1SZmMht62QcnH +PBfRoOaAkhyu/WH4iQU38RuaBGjlW+29AM4h3oP807Su/7A8Wb5lki02WTSmzQf6 +l2t68aHX3Lnt4VPyHE8XFiLt52flqxYf/LklMRx9IA8OGzXc6nJuq0iomW+f8ZCI +Ciz9e+U476WfcbBXXQl8ailXGI3ZAPAHuNS5qV/swHzdWQHTTUJhdJLhA7lBpWEf +IX9kLzkCgYEA7ErlT3sidqNx0Hr0/JbIKfG5hXaZ/f5fpkxTr4b4OLI/vPaV2shC +qw20ellNpAkymRjNhGnj6alxPdP6tX/PkAvdze6hSEHJa20PFpxVIIJlO1HGlMx+ +klpDT/Z6pjclYNal1ojpIXkCLlWK1bN3R6IUAe+dV/LmC9REKIGCTlsCgYEAxX5m +/bzAFoBJKfEWf8nqllnGqETgIC66VpDfDPtW2NI3ZBcjLnlRgyGNJTtaKmoiO1NO +uOX6z21aSnnUFCW5fgXbMpMstz4bUDoaAXzsYen7XZVWkFSdaGiackA5Dm/iBrvk +BzaQ0sDI6njnYKoRTpaMcXTmEgqptAAyx56kGMcCgYEA0+4o1ay/MGFQB4kAijxC +szwXBVlmrKSl7WWv+VK490EIYddYeK38/aaBJOtL88A8HYxdaFIBFOXgp2+lAXzt +EWlTOwy4ozI+EZfzXHhC8bGCUj36OiNfsqw6i1Gql8IGSGC8xTpuvpLmHeCjcSBR +73GzODlNikBVjG6J4zqlQNkCgYEAvtGk/WFUT+lfx9CTpqEXsnHHymnSDAZaMK4F +deubPB/ROTpJ2euKYKMYV3MDaZvmu1+A3pIHRkgoR3FzAox4r1VFN5aQS/UMOvYI +jot/chO7te5HF2lKNclsARwghNyBjXQZnQaR47A18KclGHb6Be7cf/stGR2IXs0a +Q1n6v50CgYEA2oA29lP/x4e5iULI5uwCA2H/fzFkm9oEwwOBqjjwS6CBipQH9WZB +1jI4ddvW9kzcXWO6l2mdeT9b+jFtgIUBqeN2pgD4RXmtuP0xbkUDZgHmWv2aOSgH +IEC8l9oRus88p+5yf7mQ+/WF37Oe/RezDuM6/+ehtoUTtJ1g4XIXICc= +-----END RSA PRIVATE KEY----- diff --git a/proxygen/httpserver/tests/certs/test_cert1.pem b/proxygen/httpserver/tests/certs/test_cert1.pem new file mode 100644 index 0000000000..49b0008ada --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_cert1.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5zCCAc8CAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwOTE5MjIxMzQ2WhcNNDIwMjA0MjIxMzQ2WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAprvPkBYr6LTFOldRjTQ9zKf86tBu9kG1a4CLu4c1 +2twWTf04b8lfpG5qUMt13IeI5CH9ygjkLz6gZjsGDICVokG5P5fd9k+3eIv6m0K5 +rgNUXeSYJJTbfIhEw99fdI4tpu5irtnWLGGsDApF1O2NDLYh6U0+eB1OWOGhqrSU +AMPibief2jtLsETaRZrYSFknPfgrNjzcIfhnAv3rMnkEc55knV8l7UZCLgUaRPfS +4ZcTe1VJghHPCbbfQ6AEcHZaXhOlX0voAXesB5RVuyPMuhQzBfasBstjITIpdbQI +AlnFuF/vo8JRhqJKjOWek6DJyH7yjw9ZtvXsMTJCun9M7wIDAQABMA0GCSqGSIb3 +DQEBBQUAA4IBAQCdghgh4hK0HUgvr+Ue2xUgAkEhQK7nvBlxw42l64zWNIkVrg3C +sGBx1/ZV7sVrrz5P8LkoZmKcgSoaZQRhiZ9P+nBj4hUz8oFYJ2xTl2Bo1UmEoz+r +z63WerLLb48HQLrGJN/V1Uodjb/eVRwY16qw0JoaRg3BGbO2k19jeNIfpp00atic +xvgxZsHuRrax4PkL6ObrASILj78AOzPmKOlMk2cbS+Ol4WJNzbqFDQaR3QXv4WSR +6td3LlJtSyMWjMnkYOOidLYsSQ5bVnWbnP/bj/apRXxX9wi7ez739Gqc4bylJgW5 +Ym+TCytFhaK6z05whWCDcD6CrXyFGer/Cqfv +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/test_cert2.pem b/proxygen/httpserver/tests/certs/test_cert2.pem new file mode 100644 index 0000000000..378e72a0f0 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_cert2.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5zCCAc8CAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwOTE5MjIxMzQ2WhcNNDIwMjA0MjIxMzQ2WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA8uWxaUPDq8Ydzmofy2hD8XQnfniY+VnkIe53aMIT +JV+iMU1G8cguXxwIK6P+iyNDMMOG6tuObng/9yfS7Zt7Ov4F4tf/S0qHNK2yfZZj +n0QISkwtFboA0nso60y0QhlKeySsVnd8Bry53kNAb5CVO9u/RH/YHP7ykqUFGln5 +tX9gpCN0Iw1LUZvABH4rxDP/9mYrKRDgBkYfKJ2xLxG0IyIdM9P0F8RLs/lqENVQ +w1X4TNkFL94VYEKl/5OiepY4cx2zGpqlSYcNXEqfR9wBdCpH0+mMf8FFxE8m7W26 +9BefNn5zg7U2dJXIm1YV0lnjRt92P1RiUvKOkm4Og7WpowIDAQABMA0GCSqGSIb3 +DQEBBQUAA4IBAQCT/OwW7fWCWrfOgkDQNERhE2RPz2s/39I6jHYJfnQaboPkSukB +ESFvk10bhQpJww5C7XHH180EneY48VqIBeSUwmr+KL2YkETwktj5hg4i/KvaW5In +Gb38GB2o10OrMNmM2TNi4eLb81FNZtdm01npH97My6XAkbxHr3e3nNInFUETxmhE +HCIp45Ug2I7vxtXMVqI8WcCBLuKXTkQqBCDXMwov83GllzAkczbkQKdeMgouEtou +Z9zooiMOiPQCg6/3xG0idBIyCRH79pPR079o2q09hY2wDy211e3tAnPygWhSivtO +P9I6H4OY2ZDGgT6HLbmT03RfVbPulsuaiBG+ +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/test_cert3.pem b/proxygen/httpserver/tests/certs/test_cert3.pem new file mode 100644 index 0000000000..e99d979e41 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_cert3.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5zCCAc8CAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwOTE5MjIxMzQ2WhcNNDIwMjA0MjIxMzQ2WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAvGL/otOV3fqmBgLFBctGT25NID4silHADcZfRs4p +vJDb8d9PrKyKkz3HfWzaY/5MwNbkuyTfiqX1hm5fSGCFTp7wS569s1s3neixTtTQ +zDgW+m2oMIOapY8OL9rgU5TCUXqy3lv4MsFbcj2z9Gbnp+kLbsLGlxdO1MGzEwKJ +GsfCjmbtIFc4+6gtmm6SwY9WZ6y64OUmyBxbYKZmaYn8XM8l6NvXVeo3MzGkQS8m +jQCWfP+QSiCwm/GjsG/uRGChp9rvKDbf3D2pB79el4q00AWi9pH/rheuO8NJpuuq +Sd4QffzbDU6qrcux3Nb2N5cDPi4pq4xawgbLOcaUaQj4CQIDAQABMA0GCSqGSIb3 +DQEBBQUAA4IBAQB/LT0Ufq79Km7V7+bmrpH0wD1n5tCXC411lz+G2rJnNLuYPJSI +xXZLXv8r3zv43hf4BonhsgFDk7UuXVAFDIk5N9VWvnVgZ80q51NLl7UOr6XMwgMw +NtCarnDFpvZRcsWAnG/4MKPOh7IZibf/3hHqPpeU2gY7Puxre/17RAXA8dwPx048 +XGz/V9kyMSQ7Rv19LoPqxPWCP9/tdYhyEjh+ywBaZtmmMyutnlrrL1CJ4Pcai9w2 +otb/n2dTjvaJtV8Nh0LWQnltD3IFHQAXwD8ipNQKQErfhhpAeWiZp1SlrmheIKj6 +Zu2u5yeK3oy7FIwa17+6dYmQGKiPdhPMkHt/ +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/test_cert4.pem b/proxygen/httpserver/tests/certs/test_cert4.pem new file mode 100644 index 0000000000..0280f9b111 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_cert4.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5zCCAc8CAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwOTE5MjIxMzQ3WhcNNDIwMjA0MjIxMzQ3WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAtMslNMNsNTMtnUgSJ5s1WDkUNUwU0uM0lTW9rdYW +T0ZUmYBDCG5IQotW7oQkJOcaBhXna6RDpXW9zMW0qae080whJwzXwVBUqJtt6UMr +MDPtgITzL8b8ZOVgPFWlPnMX1/hY48iL74J5dDziE/KlNvkGk+B9pthdge6syqaV +kKoGAuhrORoYS3z4/5yXAJJKjvxxZSUmSdjamuD9Th43lJyqbATeFfRmQKbJGUez +T9NUC4MdCISKxMbd5e1zC0XBSSGSR1oxS+lqq+xslkY8OfJ+gs9FQnq7goCxhG7k +J46ys59dZtDit0t4k53InzoBN5pBoh3NUMXGPGzLgtjoaQIDAQABMA0GCSqGSIb3 +DQEBBQUAA4IBAQBz8q9EIIlkTcBq+HmlLoqjOeL4DG7BsV6PaZHGeqana/jD+QGF +fHY8hYbW/B7eRiAkKPyUI5/N8hrrYz+pnPIKxZXB/RNf6jwMmOLne7MB6dtacCeW +shNIA123SkOaCKpan6zzeKoPt7W/jKf14MNVYru58gJYwEs6WIGBt+TPEV4Rx7gj +VsGRfU5pVFlWTe//nQmWa5IvKgpxIbJf9/2A7iuHQtl0n5slKp7EEk7BV+X2YeGM +Z4W27CiRGSMvmP9Dq+r61ZNDS9xD3azbThcYNL9p55YfTOjFmQvFOdWGDLDA4CDE +qEbhmsYv74Q0oOEcHgH5dxxKhZ4Kh1s2vc6E +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/test_cert5.pem b/proxygen/httpserver/tests/certs/test_cert5.pem new file mode 100644 index 0000000000..a55838ef46 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_cert5.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5zCCAc8CAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwOTE5MjIxMzQ3WhcNNDIwMjA0MjIxMzQ3WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA3+ThdcTL2XjqQ0KeOGiV9HyDHCXl7U63XjYzBzQe +HA/ZEPlqCTfd2iOWbVpIFgDOyxDwJ+tXv9QuJnpj8sQso/BVN1m4lrT/MpjB1O+X +AUcRSlgjwXBEEkGTqi2FgQIMIhyKtlIq7+uhY7pXbwQrzweEbhhWiYC32sIGVCN2 +qyXIYokLODXYFiHkMGx2BrLmvaUbFH7zOnjvGUeYLDDcS0/u9l/mpKeePsiVTZ/d +diLNEfzfapD60h0e0o0f6Nf5DScllCQLoVEIgMc7IcySUI/CMuWgCA+IPumFFSEg +HldFZhIONszJtYkUCAiDKOAA7ro9nY9HtPYoAxSbiv6xqQIDAQABMA0GCSqGSIb3 +DQEBBQUAA4IBAQCvI81KpgGROxFFVII5EIPuWhH0DsTBlfjO/fzrjhL6dqphDZpA +SaD19XSia/NiPQO/4GhkFmx3w7SmNisJ33ecBcmR1HszmtVsJVoUOU1hDmLaxyJc +vhSRpCyIN9FPTACZmgWbRN6L90dQRoKPwYiLBwDX7Y20c0QF+Nvf6SwJBSqVHh5Y +hvwzhtYZwBR1Ufhk8g+CuGtn6ogsLx37AoG2Ysdhx7kqrSvPvahLw1Kgap5XObIZ +e/YvDqWpPC+ImXguFhSKTqL6m7cdDC43tKV8iSbkAdweAvs9XpgqoX0IZ9xKtGai +0+TX2AcoBPvQHCujYWnV2N6roDbYes8vLC9X +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/test_ecdsa_cert1.pem b/proxygen/httpserver/tests/certs/test_ecdsa_cert1.pem new file mode 100644 index 0000000000..3151a8565c --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_ecdsa_cert1.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHDCCAQQCAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwOTE5MjIxMzQ3WhcNNDIwMjA0MjIxMzQ3WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEz2PAexkUGpUHzTaRNL8OtwoYurZpsCRaxTEvw0GTUOv5bpe0 +ituqBHvSfQEdSeN2jfdHOmeQ82KTSpqiNbW2ljANBgkqhkiG9w0BAQUFAAOCAQEA +VQ0Pf2YpNwXIxFPKmGykZKlVH+L4m0La6B7luho+orMKdKOKuIoDJZJQbGYwFn10 +kJ/hbVNeaYffB5b763ECPzuHIYokaPRGuXyixSlI5d4rzy/+GRV1ehSrRf3hXzR5 +0SKPg1yxkwQZKbTD1R2YT3FzBQ7G5fueWhzh5DXFjgoT4eBvOZ+J9zySyL3AiEg7 +tMLinp+vQC/wKt4tFCSqGdfpU/Q/irF70LX+8/iMVbq2mmtWYNCWcrWF4Rv0WMPO ++bB/C9/Mk9w4+J/N0caiCq4v4XJuABDHCEqZF1/FtuyZSTAaLbALdpKnLAk8nVkf ++a7gK3RK4Wl/4CTi9joQKw== +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/test_ecdsa_cert2.pem b/proxygen/httpserver/tests/certs/test_ecdsa_cert2.pem new file mode 100644 index 0000000000..1eb9be7a3a --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_ecdsa_cert2.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHDCCAQQCAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwOTE5MjIxMzQ3WhcNNDIwMjA0MjIxMzQ3WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEhPeKFAokkp4KmIVE7WqKc8w2k0YOwudHXXZ2fAQNxZcGHjml +5Mj9evH8Ezd9OoJ/u7cnuFKorBce4ypHZNA8zzANBgkqhkiG9w0BAQUFAAOCAQEA +L85cwJHc0QlvTqnngGS3F+1yElCROVUbdqLBXw1lyToNH280VtiPKEeyCzT4F76D +GCVRS7keLiR/IkhBpZkYi+YYTG649rvfsOI5ab74lyv0PiBrLTekEO3xi6yLK3bm +4Eo3mNIFE7qzUBgizWZyE1ri119P5K3+TRUz/TIafjKfkxUw4w8I3CnZzFzm3YF7 +w47jvAws1rEO4UTT7paTp0g0AIP1JIuk4dN8wLL07fSxWZCNCQqTwwrT0J1xXdH5 +jsgAxXPQ5GwetoP6pFs6F/hwxR1RZs99EmICvBeLnkWLZG70do+EOfI2paya80Gd +lcir1YxqwEwt2J1Xm5hmxQ== +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/test_ecdsa_cert3.pem b/proxygen/httpserver/tests/certs/test_ecdsa_cert3.pem new file mode 100644 index 0000000000..1aabd6a37a --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_ecdsa_cert3.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHDCCAQQCAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwOTE5MjIxMzQ3WhcNNDIwMjA0MjIxMzQ3WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEMLrfx4XVO8FwVtZ372TciaxM+Wc/DFUfnZLmw97OHHks6SAR +B/bnpVu23QsD/muYSiB6vvzLYxCJuYguH6NWvTANBgkqhkiG9w0BAQUFAAOCAQEA +Ik+T3frtQsOIWv8xN/wMR3p+qZCvqOUadEC4jO3b04w4BuVSKEMerkRu6Y+n8ZQZ +p1LkRtwUc8+5B9Gf8HIQijsD6BXCtpJB7M0WnyyUBvRVseNn2VhiCsNYEVEbeC/T +YYMI5l+sr5j4VS3smj+2YsjGo3mHMGXfQ4RMmpdLsDo/GEHxZ21DD8bS+41aYrZ7 +JGTC1yn0iiM33brRvyD49sad4AJ1CIvD5DHmkiwC85AW1tBvpolTA9xpfjPQDgM+ +MGMte40zM/HVWDMRTUWNhuPSp7XVZu7glxK90AYfsHRqb/DXoR0Df8qUQygt1et0 +JqwbV1NafTD2CGt9SFhNNQ== +-----END CERTIFICATE----- diff --git a/proxygen/httpserver/tests/certs/test_ecdsa_key1.pem b/proxygen/httpserver/tests/certs/test_ecdsa_key1.pem new file mode 100644 index 0000000000..e3ca701fc0 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_ecdsa_key1.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIG6+DnEONCjOb4bmys60IISRCPxn26Iga3ie+tKLy6x+oAoGCCqGSM49 +AwEHoUQDQgAEz2PAexkUGpUHzTaRNL8OtwoYurZpsCRaxTEvw0GTUOv5bpe0ituq +BHvSfQEdSeN2jfdHOmeQ82KTSpqiNbW2lg== +-----END EC PRIVATE KEY----- diff --git a/proxygen/httpserver/tests/certs/test_ecdsa_key2.pem b/proxygen/httpserver/tests/certs/test_ecdsa_key2.pem new file mode 100644 index 0000000000..de6e215533 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_ecdsa_key2.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIDjTZCEXwIl57daCOp7/0kv9HBjJIM+DBMuYGQx7j/vzoAoGCCqGSM49 +AwEHoUQDQgAEhPeKFAokkp4KmIVE7WqKc8w2k0YOwudHXXZ2fAQNxZcGHjml5Mj9 +evH8Ezd9OoJ/u7cnuFKorBce4ypHZNA8zw== +-----END EC PRIVATE KEY----- diff --git a/proxygen/httpserver/tests/certs/test_ecdsa_key3.pem b/proxygen/httpserver/tests/certs/test_ecdsa_key3.pem new file mode 100644 index 0000000000..49071230f1 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_ecdsa_key3.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIFvllTFRRZbSME7guGQMnzS05hbmMAWTxSP4CwknRo6IoAoGCCqGSM49 +AwEHoUQDQgAEMLrfx4XVO8FwVtZ372TciaxM+Wc/DFUfnZLmw97OHHks6SARB/bn +pVu23QsD/muYSiB6vvzLYxCJuYguH6NWvQ== +-----END EC PRIVATE KEY----- diff --git a/proxygen/httpserver/tests/certs/test_key1.pem b/proxygen/httpserver/tests/certs/test_key1.pem new file mode 100644 index 0000000000..650f501d54 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_key1.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAprvPkBYr6LTFOldRjTQ9zKf86tBu9kG1a4CLu4c12twWTf04 +b8lfpG5qUMt13IeI5CH9ygjkLz6gZjsGDICVokG5P5fd9k+3eIv6m0K5rgNUXeSY +JJTbfIhEw99fdI4tpu5irtnWLGGsDApF1O2NDLYh6U0+eB1OWOGhqrSUAMPibief +2jtLsETaRZrYSFknPfgrNjzcIfhnAv3rMnkEc55knV8l7UZCLgUaRPfS4ZcTe1VJ +ghHPCbbfQ6AEcHZaXhOlX0voAXesB5RVuyPMuhQzBfasBstjITIpdbQIAlnFuF/v +o8JRhqJKjOWek6DJyH7yjw9ZtvXsMTJCun9M7wIDAQABAoIBAQCGJrJ4Yf5uO5Q8 +vqjVDeVzVu4+F/pPlMrddg33knCYaWBg246fEs0rRdOwsiNgjoRr2ZWTCthd0uvH +lVHmmUbLyEm+iviCB9281hOK/ILdKbyl1xk6xbJbXmDFoGHzK7o7h65KtOaHywZc +oZ9SFNfaFGjwh7/tcNbq2I/1A1nZynTko6iLVpgV0kkQCpaweFYQMXWv/ELkFHeL +7tFIA50XFXbNDqnAuaW6XrIrW33ZeOJfruF2OG+QVWyTBgczk0fodDZJFS5MbDXu +UB+W1nDhakZFbugtDSXMd3BMnLZFdsa12FYTMNG050w2OTHOF5ILX+IFwzbnlboX +fbralUKRAoGBANnjttZWcNtM4L8JkUJwXsDgTD/0d+iu5gCmSs0p9E4wGbWlu+In +cNE6CV4Dy+GMk5GzXFR+GV/IlPvVzSOmbFsFdBi8L3c/1IrPbQv8dEosKm6BvV9O +0zIBaPuzgU2yyBFxdpfsHynAZoLdY3rq6IJdNmJrmDcVgEfBVOExU7EVAoGBAMPl +hqNmGi3VwHPQ2iiuM48ijPbuS0hK3dUjx2A4otOYAro86Q4egcdtyBOONhBwD89h +I6BUo+vReV6ikI8LQfoplBaRos7qJ2e9SOxmRIJGAZPkGlFF0uljxKZ2Hdtmruae +mJOZqKCa38sTnqWyXV/xCXE5X94EXuJP17L2Bt7zAoGAOG7gFheBV2tL8m657qlI +AVCWryHURLG35IctbIHnQrD2l7N7PBHXCHmtn2oATkSom94GleOrEsHSxH8ViJw8 +CD8bWKS07n/bvrAGoEocnHFf9AsqTxsNXDA9TqOpY8RgSRRIEQUY9Sld45sPfvCE +k+8sfMU9QVcSSINsRn8OHBkCgYEAgzBGD01EQOfB/42hW9b1fmjEAGYrElnY33Eb +hyvGl29YfEJoTOVPQjAZ6ka1nCJ/5ACIrEmikT1yS1cQ+kquv4pyuv6DCpCzHP0d +Rfti699YFSOQIFdjXJtMybGWYyUMAjO5uDcSP6QYNVaJSyv87lBsY1/p/LPumx6f +NCEhDtMCgYBpcK4f2E+JjaGHABX5OS5Soegtgj7JjZv/M3N2OLvX2xrlkxAEPlJ5 +nvaVjikBcOsj3/+LDrBMDoEbG2JFaopiue8pW+tZWfbhJw0pf02f+hgHjCaR+1Ny +Qqd+ERH7vFjwzc3UuZay1NbU9/wVMNsL7jWKnvsKKCk9PxG2OzP2iQ== +-----END RSA PRIVATE KEY----- diff --git a/proxygen/httpserver/tests/certs/test_key2.pem b/proxygen/httpserver/tests/certs/test_key2.pem new file mode 100644 index 0000000000..2b1001c978 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_key2.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA8uWxaUPDq8Ydzmofy2hD8XQnfniY+VnkIe53aMITJV+iMU1G +8cguXxwIK6P+iyNDMMOG6tuObng/9yfS7Zt7Ov4F4tf/S0qHNK2yfZZjn0QISkwt +FboA0nso60y0QhlKeySsVnd8Bry53kNAb5CVO9u/RH/YHP7ykqUFGln5tX9gpCN0 +Iw1LUZvABH4rxDP/9mYrKRDgBkYfKJ2xLxG0IyIdM9P0F8RLs/lqENVQw1X4TNkF +L94VYEKl/5OiepY4cx2zGpqlSYcNXEqfR9wBdCpH0+mMf8FFxE8m7W269BefNn5z +g7U2dJXIm1YV0lnjRt92P1RiUvKOkm4Og7WpowIDAQABAoIBAQClLFkmcfRfnQur +0DLqyW5ahVhOAohUGDKweE8vJ7qJUa0jxZ8Wz7/o4VEVDCIOT95jDLN0hfUnXhJx +ad1fwrlb9l3eUm/CrV2gDvYvvNLl/Qd+LqsB+UiR2TqMN87km/owH1IqQnpOwovK +fwUUWMeCuv4oAJ/fp+cgnaMItmK+bhr12OvTS2X739op+6c6fNZmtXAyQSckTs71 +TTErNVu9lEs4epBZ8YRvb8dYIAEWgqnHOOz+RRmGyrQevIV+9TBK3L+bgx6OAfDq +TNdmdMeHHpXW1TI4JxYf2K+9esS+FL6Hf8zHsRNtRYu9pS6jkYw0REtW7N7E8roB +dZSsI8NRAoGBAP9ngY6TJAS6jV7n+CID6iyVF+7OCU8LiTsHr9nBa6sxL1VFED5o +Gxb7z4UyjVWaSaKyGMnuQx4k5t814nKmYnBRCuVPDhnn2ztNC6ixjV6s06Itc0t8 +8KQy/FdsYAnMvZ+je3+jIWR/jK3lH6K6gzThk/oeAlukC5bZjchF8l01AoGBAPN2 +uCbob2i3Hyn3mbAmoNXIqGSXIUrvyh5i9u0gwKMU/QNBWHtm1erUszZnPYIOaFhC +TsrA1E5xOiTqwKkd6Lg+fX7g6eCV+c+rJbl7wYtyKVeTw5m6wzjsoUqacLjnVCK6 +WLaZn/IWtujDmmMYwbFiP13frGiUFEvOVtASQ753AoGBAOZSlG5b2QZ+qZClxon+ +V8beqVeM7K4g7B+Uvgu0twEJ+PJ/trdgsNVYPnuS7Av/eFpFG7+2o0Zi5uTyNgVI +cMty+k1yrnfENFtVDqeRfribSLsfG7M+t9CLvi6kqDMONQ7qoiunlCyKLfaAAriA +VGRy7TyIpX25AU6HYKn0Ei3lAoGBAIFSwqcIOIWrIAau2xhSrIRivfAQx0KC9R4G ++5siFrGJ6IveHh4OlfrTWQ7A8E8xUAPx1OCmZR/1zSjm+cfbd07HAupulk0R3UyO +YM+SCVEFJyi5+OCj8CdAqrxyJQZS+sInsg/ssqVpo2co381bzSdoRLico3w5jD5o +MHz99rYpAoGAMBsrsbhhOvPMTF8NPu4oEksEUTSc8WzbMaOj1gppMKLhrXvzjCqP +ffpgEKcTADCiQ+I/L5amAUm314zcAie8dI/dBSe8dxgGHmtMnOgskMxbFO55C62y +pbYlw7w0FtUo0DKPjhW5w3cYRgji4QfbVZeqAaNyIafWMhuJP1H2v7E= +-----END RSA PRIVATE KEY----- diff --git a/proxygen/httpserver/tests/certs/test_key3.pem b/proxygen/httpserver/tests/certs/test_key3.pem new file mode 100644 index 0000000000..4d76e6aefe --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_key3.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAvGL/otOV3fqmBgLFBctGT25NID4silHADcZfRs4pvJDb8d9P +rKyKkz3HfWzaY/5MwNbkuyTfiqX1hm5fSGCFTp7wS569s1s3neixTtTQzDgW+m2o +MIOapY8OL9rgU5TCUXqy3lv4MsFbcj2z9Gbnp+kLbsLGlxdO1MGzEwKJGsfCjmbt +IFc4+6gtmm6SwY9WZ6y64OUmyBxbYKZmaYn8XM8l6NvXVeo3MzGkQS8mjQCWfP+Q +SiCwm/GjsG/uRGChp9rvKDbf3D2pB79el4q00AWi9pH/rheuO8NJpuuqSd4Qffzb +DU6qrcux3Nb2N5cDPi4pq4xawgbLOcaUaQj4CQIDAQABAoIBAEbdgsZwi2bGW9/U ++OJ2FgvZNUAUw1KA8Br+bWbINOEPKP2ygFk3SqWazsk6wmhtB9hevgo8E68nQNYB +/OvQCzWJCmGX5Eps3N4U42YBHk5EJzirOWKOjwUKbE7KKiqmwfY+jrygohwNqmWu +GlysvSXnv3o54NSDWw8jkkZrZMHij9OMvbWE6gzaAyk3JIRtnVVvuOEflbDwRyzb +0BPygswDHaZFHVSYwiQSj4v7yPNSpiLSV8qHwmhRqeJM4Vknjhspf/ojFDt4pkx3 +k/lyga0si8YTzeFYHsdzIRAiAqbTy5PH7KQ/oGOH1BZ7gynFd71GiR3sMa1VhCho +ukZcBIECgYEA48hE7fT04aqWeqBRKrvDVRGHFXlXnlIj2/KVSFXUxuLyyVGpkEro +d93pxV8nOI4wVkvmgHA/TVgJFDSMaz7kGvFm+/IJi0XYwpGapepoAM5+M0GTKmIO +OFor6EwGof7EAqUwA1txccFiLYNq1VszPkp6Vc9A5FC6z+oeMga665kCgYEA07ld +ZCxX6NT6ZaI1xTGLO0I4JbkK7wpMRpifBG5AWTkghCkYjL6JgyNYt7f+GLmWLYwh +b4Aqi2t99HMAmfWpefbEFTvbSrzTuNpp9UxxpFxDZmrI0Lyh0K89WDQz3vdZX1CW +F9Cj0pi8DdT8wHjx2aEip0GEHL0r8QgaFV0utfECgYAwhRBpv6wKBdRX+p5Pg/9v +t3QW1uobB1QsHdg+uEnCs8UTrRl5avtpzRNmv4YTPfZ9610GLuLNDtugOjFPBu5/ +7rwJHt15lT8+8tUGUpAk29d2A5ndhVWSG2MO8GLZTvNhvN4lWO0pVhAbscqn9+1K +b4LvlF42RBwG/c/AyD44wQKBgQCiaYJTkpSI/b9q8UiwEPZOKdjqTw0tG0gxq2fo +PS7Ngr4Hw4KajJrt5tJqdBDOJKcoH2OSpc19D6MaPpHKiKFN9taptzXcGgeLuLgp +PE+8li35xZZwURMMAzFalQjpU/LR4/6PHDRfG1y+e5C7kMttd2ceyGowpYisp9ti +Yg2v8QKBgENPVPXCNOBLjdBj0W8Jg4aEE08547wEZLGg+4yp/U4SNyEoFO3WYlHb +P0V2+LqYEYKf23VCxTOoG4r1Yt04G0cWHZAyII298IK5iLXGbEChx6XgWhcpHQdg +2OyiBd591Ymwx80CuQM8vRfHtlfL5+47fyUYLlPBFS+1QreMDaAa +-----END RSA PRIVATE KEY----- diff --git a/proxygen/httpserver/tests/certs/test_key4.pem b/proxygen/httpserver/tests/certs/test_key4.pem new file mode 100644 index 0000000000..a8ead75e68 --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_key4.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpgIBAAKCAQEAtMslNMNsNTMtnUgSJ5s1WDkUNUwU0uM0lTW9rdYWT0ZUmYBD +CG5IQotW7oQkJOcaBhXna6RDpXW9zMW0qae080whJwzXwVBUqJtt6UMrMDPtgITz +L8b8ZOVgPFWlPnMX1/hY48iL74J5dDziE/KlNvkGk+B9pthdge6syqaVkKoGAuhr +ORoYS3z4/5yXAJJKjvxxZSUmSdjamuD9Th43lJyqbATeFfRmQKbJGUezT9NUC4Md +CISKxMbd5e1zC0XBSSGSR1oxS+lqq+xslkY8OfJ+gs9FQnq7goCxhG7kJ46ys59d +ZtDit0t4k53InzoBN5pBoh3NUMXGPGzLgtjoaQIDAQABAoIBAQCDca77TZm+lBb1 +4xBP+2gtXFJ07Xu2V7pDQW9GlABkYp3ffIE//+5Jc0Ug5LHioqMpCYPXnvYPtiro +VlWjg2rr+M2htpSm1+C1NUwT6+HaieaDKiYk9mdKc/oj46DgZo1PWCBIe+ZBnHrx +J9K27hNwhF1j5jN0uppBfWXjcHoNUlYOl1+goszxiX0GIFn9xq0EDIDFLbKuw75l +qTDoMFTBGdTgSNjpL2D7WZonlpbd7rOjzOAvELC/Vd8ytt9bzG5IvsLeFU5JXFY2 +KY4hAv8tbH8vMx6UIr5VfNrO9kNqJkTYSruE1rmTP0QX12y/EEWEG37OO1Onk4pz +Ka6oaUABAoGBAOyU/KTD4J1SIuVuPqHt8/DsknPdXPAhKaPb77E2d4B0wG3Pdk6G +IQgEhVikK/pGnQk4lhug1fTNWUujVTgD1op0l9rHHwyus5aMj9cJDP2naUmBNLXI +jjw54i3gYv4v69D7p54sYV2PRMup6tjvUXTr4riD2ErDb/hBkMZu5GlBAoGBAMOh +8V9YJ97tMkhwBvYdRkPp+H4JgnPmXzJlq+Ojl/IsgStHSwekaZP9k//WrFJ0+w4I +vIAjKgfeSoGJe2vtlPvq0YUkor+zKO+0FIYFexd05tb8LPf4HImkHEcPEMkUz9id +PD1W4KgtDOlOpPxJ+K9CbWtSkanoawbaFccWO80pAoGBAMnDtkN+mg0Rg5lPkDLh +kO2jlLMj0qMoZS7N33mvDfOLnqEIEcmeMoV39ZFHUgo8NqkqjTo6zL3ec41CWudO +vqWOEgQPVbenNpnqfrHRkjaxk+WQP7fYtxU9+FtPxp0pmV+9f5IyH0d/bBiVbShd +0YZ5tf3O06PPUarTn/jbkgkBAoGBAKapmBA58f2g5W2Awu15ExtYgDIft9s1L3Sn +2UAdZp0R/Rj5q5nfH3LMXQFfyX6V2iuilbQ0QOJjJeYlUdgolvvmmIhtJZla8E0F +hVaH5M2e8enE/CpkXSuFe/GtjAdCi69mhKNdGBcuCgnYzgWAnzPvy3fa5+1v633y +3Qq2jkvBAoGBAOPJU8nE88jiu/nHnq591z8gQmsWWPBOuhjqrcvgBlsu62G02G17 +0j0iyp3ITvZIBgCl2kr/c+69DcE2TNRaFC9CePMWr233cyJ/Z26x866RK59WOcyT +/+upj6UI5/vfrGDEiDYy3r8VgniAVmADVfXQdmxiH+KFuFPJjHfQwoPg +-----END RSA PRIVATE KEY----- diff --git a/proxygen/httpserver/tests/certs/test_key5.pem b/proxygen/httpserver/tests/certs/test_key5.pem new file mode 100644 index 0000000000..99b497e83a --- /dev/null +++ b/proxygen/httpserver/tests/certs/test_key5.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA3+ThdcTL2XjqQ0KeOGiV9HyDHCXl7U63XjYzBzQeHA/ZEPlq +CTfd2iOWbVpIFgDOyxDwJ+tXv9QuJnpj8sQso/BVN1m4lrT/MpjB1O+XAUcRSlgj +wXBEEkGTqi2FgQIMIhyKtlIq7+uhY7pXbwQrzweEbhhWiYC32sIGVCN2qyXIYokL +ODXYFiHkMGx2BrLmvaUbFH7zOnjvGUeYLDDcS0/u9l/mpKeePsiVTZ/ddiLNEfzf +apD60h0e0o0f6Nf5DScllCQLoVEIgMc7IcySUI/CMuWgCA+IPumFFSEgHldFZhIO +NszJtYkUCAiDKOAA7ro9nY9HtPYoAxSbiv6xqQIDAQABAoIBADvLpgdM8uB28aHb +BeSs0ffE1UNMP9ccIc7wjLpMygnWlbN52T7zA7ZBsOgtp3jw6EnFkU4oBXx/D4BJ +Sa5JhF6YTjoFJU5ispFxyLq07lByCYUgexQrhpKVnvlhRPmKcXEezFEbKsp+1NCM +sNM+evDE4jKBH4ODtBahoQ0Rl5znb3UawODBsFLZabUNK/+qO/cS+veDI2y0bfg0 +hlTxzEJRCWlQcG0/IYEOIo/69qihtUYi622SLDOxGRf6j4HHEAIQvGmccWLhRNtE +5va8dxn2lilNn9/zc4pLJrR9EcPjP3mMsrFwrFMJU5KdRaJS31bIEdx/ZPLD/ty+ +Og+TxnECgYEA8c5qQ61zR+yNfcRiNRq3irYyJI1/QtTzxEY/M3P03R6TeXHpStkH +A4bh2CYHUUxqDDH+4lzPG1gTg8J0U4WHcilsB7n9Vs9m+KKW4mmtHoZxf0m6ZCNA +6NQHeDcJTa6Pnh3qkG0eqqtBmyovg944EpAx5Hb5J/8xHfEi/yYAdz0CgYEA7QlN +GbfqwR9joO8inrEfiP/unHPsw4tcJVwK4vrH81rvggv9mKfJDJCe+XEODv/BD/mZ +fAtwB2DrkSFyzNJgZjpKrdc2phY5GucZgwmoKhydIENmc8lC78be71fBhkeDn1BI +qOOIn5uFZ5RW3ZHgpC0iaOjJnZ+crUWDlPei6t0CgYA4ftG1HkFg/JPSXp/TaHqZ +nhf5ElY5dye9I+yASQdc4lfyd/rZ0fshh9IcfkYXaJMeZk/281gwO1PT5QrouLn8 +olDrPTlDnxGf5dz66VXZW/AInWE/JD12KJPWMoWp8K79cl/rfpM7rOeXKTQQy0qu +i6Icju+HuMfxBvX2Rxq0wQKBgF/Vz99jFb3pM/3AUa35jPd2CoIk+IgDE0ljkl88 +55TDomxg7rJGvCmhWzUc+YMk8mjEEMMvWKcTD4sJDAI12JXcYY8xoT27ZHE1GIJ7 +aAtGsFx4A7cymyaYsE/ymiLxTQ0fh5EJFZb6aRB80DYbIckfGndyDvn0q4L+xPl4 +udzdAoGBAIiihXYJA6HeLv2GYRpsTUxH+/HHNvla9YAC/Fkr7nSPWvLqtFWtU58N +ASPlRk3/+YTGnEZen6m8xKXm5LI1pMMQFXx/kGBOMKko5JcDbyTtkw50B/JzFCCV +akwQOl/AmIx2LAJJUDcVqJMPkTwoSQoMhTXZDqEjszVBv4gzBFwH +-----END RSA PRIVATE KEY----- diff --git a/proxygen/lib/Makefile.am b/proxygen/lib/Makefile.am new file mode 100644 index 0000000000..d432cb8b57 --- /dev/null +++ b/proxygen/lib/Makefile.am @@ -0,0 +1,13 @@ +SUBDIRS = test utils ssl services http + +lib_LTLIBRARIES = libproxygenlib.la + +libproxygenlib_la_SOURCES = +nodist_EXTRA_libproxygenlib_la_SOURCES = dummy.cpp +libproxygenlib_la_LIBADD = \ + http/libproxygenhttp.la \ + services/libproxygenservices.la \ + ssl/libproxygenssl.la \ + utils/libutils.la + +libproxygenlib_la_LDFLAGS=-static diff --git a/proxygen/lib/http/HTTPCommonHeaders.template.gperf b/proxygen/lib/http/HTTPCommonHeaders.template.gperf new file mode 100644 index 0000000000..213ccf834a --- /dev/null +++ b/proxygen/lib/http/HTTPCommonHeaders.template.gperf @@ -0,0 +1,61 @@ +%{ +// Copyright 2004-present Facebook. All rights reserved. + +#include "proxygen/lib/http/HTTPCommonHeaders.h" + +#include + +#include + +namespace proxygen { + +%} + +%language=C++ +%compare-lengths +%ignore-case +%struct-type +%readonly-tables +%global-table +%enum +%define class-name HTTPCommonHeadersInternal + +struct HTTPCommonHeaderName { const char* name; uint8_t code; }; + + + + + +/* the following is a placeholder for the build script to generate a list + * of enum values from the list in HTTPCommonHeaders.txt + */ +%%%%% + + + + +HTTPHeaderCode HTTPCommonHeaders::hash(const char* name, unsigned len) { + const HTTPCommonHeaderName* match = + HTTPCommonHeadersInternal::in_word_set(name, len); + return (match == nullptr) ? HTTP_HEADER_OTHER : HTTPHeaderCode(match->code); +} + +std::string* HTTPCommonHeaders::headerNames_; + +void HTTPCommonHeaders::initHeaderNames() { + DCHECK_LE(MAX_HASH_VALUE, 255); + headerNames_ = new std::string[256]; + for (int j = MIN_HASH_VALUE; j <= MAX_HASH_VALUE; ++j) { + uint8_t code = wordlist[j].code; + const uint8_t OFFSET = 2; // first 2 values are reserved for special cases + if (code >= OFFSET && code < TOTAL_KEYWORDS + OFFSET + && wordlist[j].name[0] != '\0') { + DCHECK_EQ(headerNames_[code], std::string()); + // this would mean a duplicate header code in the .gperf file + + headerNames_[code] = wordlist[j].name; + } + } +} + +} // proxygen diff --git a/proxygen/lib/http/HTTPCommonHeaders.template.h b/proxygen/lib/http/HTTPCommonHeaders.template.h new file mode 100644 index 0000000000..11049e56b1 --- /dev/null +++ b/proxygen/lib/http/HTTPCommonHeaders.template.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +/** + * Codes (hashes) of common HTTP header names + */ +enum HTTPHeaderCode : uint8_t { + // code reserved to indicate the absence of an HTTP header + HTTP_HEADER_NONE = 0, + // code for any HTTP header name not in the list of common headers + HTTP_HEADER_OTHER = 1, + + /* the following is a placeholder for the build script to generate a list + * of enum values from the list in HTTPCommonHeaders.txt + * + * enum name of Some-Header is HTTP_HEADER_SOME_HEADER, + * so an example fragment of the generated list could be: + * ... + * HTTP_HEADER_WARNING = 65, + * HTTP_HEADER_WWW_AUTHENTICATE = 66, + * HTTP_HEADER_X_BACKEND = 67, + * HTTP_HEADER_X_BLOCKID = 68, + * ... + */ +%%%%% + +}; + +class HTTPCommonHeaders { + public: + // Perfect hash function to match common HTTP header names + static HTTPHeaderCode hash(const char* name, unsigned len); + + inline static HTTPHeaderCode hash(const std::string& name) { + return hash(name.data(), name.length()); + } + + static void initHeaderNames(); + + inline static const std::string* getPointerToHeaderName(HTTPHeaderCode code) { + return headerNames_ + code; + } + + private: + static std::string* headerNames_; +}; + +} // proxygen diff --git a/proxygen/lib/http/HTTPCommonHeaders.txt b/proxygen/lib/http/HTTPCommonHeaders.txt new file mode 100644 index 0000000000..090c4ab70e --- /dev/null +++ b/proxygen/lib/http/HTTPCommonHeaders.txt @@ -0,0 +1,82 @@ +Accept +Accept-Charset +Accept-Datetime +Accept-Encoding +Accept-Language +Accept-Ranges +Access-Control-Allow-Credentials +Access-Control-Allow-Headers +Access-Control-Allow-Methods +Access-Control-Allow-Origin +Access-Control-Expose-Headers +Access-Control-Max-Age +Access-Control-Request-Headers +Access-Control-Request-Method +Age +Allow +Authorization +Cache-Control +Connection +Content-Disposition +Content-Encoding +Content-Language +Content-Length +Content-Location +Content-MD5 +Content-Range +Content-Type +Cookie +DNT +Date +ETag +Expect +Expires +From +Front-End-Https +Host +If-Match +If-Modified-Since +If-None-Match +If-Range +If-Unmodified-Since +Keep-Alive +Last-Modified +Link +Location +Max-Forwards +Origin +P3P +Pragma +Proxy-Authenticate +Proxy-Authorization +Proxy-Connection +Range +Referer +Refresh +Retry-After +Server +Set-Cookie +Strict-Transport-Security +TE +Timestamp +Trailer +Transfer-Encoding +Upgrade +User-Agent +VIP +Vary +Via +WWW-Authenticate +Warning +X-Accel-Redirect +X-Content-Security-Policy-Report-Only +X-Content-Type-Options +X-Forwarded-For +X-Forwarded-Proto +X-Frame-Options +X-Powered-By +X-Real-IP +X-Requested-With +X-UA-Compatible +X-Wap-Profile +X-XSS-Protection diff --git a/proxygen/lib/http/HTTPConnector.cpp b/proxygen/lib/http/HTTPConnector.cpp new file mode 100644 index 0000000000..c323e20d35 --- /dev/null +++ b/proxygen/lib/http/HTTPConnector.cpp @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPConnector.h" + +#include "proxygen/lib/http/codec/HTTP1xCodec.h" +#include "proxygen/lib/http/codec/SPDYCodec.h" +#include "proxygen/lib/http/session/HTTPTransaction.h" +#include "proxygen/lib/http/session/HTTPUpstreamSession.h" +#include "proxygen/lib/ssl/SSLUtil.h" + +#include + +using namespace apache::thrift::async; +using namespace apache::thrift::transport; +using namespace folly; +using namespace std; + +namespace proxygen { + +namespace { + +unique_ptr makeCodec(const string& chosenProto, + bool forceHTTP1xCodecTo1_1) { + auto spdyVersion = SPDYCodec::getVersion(chosenProto); + if (spdyVersion) { + return folly::make_unique(TransportDirection::UPSTREAM, + *spdyVersion); + } else { + if (!chosenProto.empty() && + !HTTP1xCodec::supportsNextProtocol(chosenProto)) { + LOG(ERROR) << "Chosen upstream protocol " << + "\"" << chosenProto << "\" is unimplemented. " << + "Attempting to use HTTP/1.1"; + } + + return folly::make_unique(TransportDirection::UPSTREAM, + forceHTTP1xCodecTo1_1); + } +} + +} + +HTTPConnector::HTTPConnector( + Callback* callback, + TAsyncTimeoutSet* timeoutSet, + const string& plaintextProtocol, + bool forceHTTP1xCodecTo1_1): + cb_(CHECK_NOTNULL(callback)), + timeoutSet_(timeoutSet), + plaintextProtocol_(plaintextProtocol), + forceHTTP1xCodecTo1_1_(forceHTTP1xCodecTo1_1) {} + +HTTPConnector::~HTTPConnector() { + reset(); +} + +void HTTPConnector::reset() { + if (socket_) { + auto cb = cb_; + cb_ = nullptr; + socket_.reset(); // This invokes connectError() but will be ignored + cb_ = cb; + } +} + +void HTTPConnector::connect( + EventBase* eventBase, + const folly::SocketAddress& connectAddr, + chrono::milliseconds timeoutMs, + const TAsyncSocket::OptionMap& socketOptions, + const folly::SocketAddress& bindAddr) { + + DCHECK(!isBusy()); + transportInfo_ = TransportInfo(); + transportInfo_.ssl = false; + socket_.reset(new TAsyncSocket(eventBase)); + connectStart_ = getCurrentTime(); + socket_->connect(this, connectAddr, timeoutMs.count(), + socketOptions, bindAddr); +} + +void HTTPConnector::connectSSL( + EventBase* eventBase, + const folly::SocketAddress& connectAddr, + const shared_ptr& context, + SSL_SESSION* session, + chrono::milliseconds timeoutMs, + const TAsyncSocket::OptionMap& socketOptions, + const folly::SocketAddress& bindAddr) { + + DCHECK(!isBusy()); + transportInfo_ = TransportInfo(); + transportInfo_.ssl = true; + auto sslSock = new TAsyncSSLSocket(context, eventBase); + if (session) { + sslSock->setSSLSession(session, true /* take ownership */); + } + socket_.reset(sslSock); + connectStart_ = getCurrentTime(); + socket_->connect(this, connectAddr, timeoutMs.count(), + socketOptions, bindAddr); +} + +std::chrono::milliseconds HTTPConnector::timeElapsed() { + if (timePointInitialized(connectStart_)) { + return millisecondsSince(connectStart_); + } + return std::chrono::milliseconds(0); +} + +// Callback interface + +void HTTPConnector::connectSuccess() noexcept { + if (!cb_) { + return; + } + + folly::SocketAddress localAddress; + folly::SocketAddress peerAddress; + socket_->getLocalAddress(&localAddress); + socket_->getPeerAddress(&peerAddress); + + std::unique_ptr codec; + + transportInfo_.acceptTime = getCurrentTime(); + if (transportInfo_.ssl) { + TAsyncSSLSocket* sslSocket = static_cast(socket_.get()); + + const char* npnProto; + unsigned npnProtoLen; + sslSocket->getSelectedNextProtocol( + reinterpret_cast(&npnProto), &npnProtoLen); + + transportInfo_.sslNextProtocol = string(npnProto, npnProtoLen); + transportInfo_.sslSetupTime = millisecondsSince(connectStart_); + transportInfo_.sslCipher = sslSocket->getNegotiatedCipherName(); + transportInfo_.sslVersion = sslSocket->getSSLVersion(); + transportInfo_.sslResume = SSLUtil::getResumeState(sslSocket); + + codec = makeCodec(transportInfo_.sslNextProtocol, forceHTTP1xCodecTo1_1_); + } else { + codec = makeCodec(plaintextProtocol_, forceHTTP1xCodecTo1_1_); + } + + HTTPUpstreamSession* session = new HTTPUpstreamSession( + timeoutSet_, + std::move(socket_), localAddress, peerAddress, + std::move(codec), transportInfo_, nullptr); + + cb_->connectSuccess(session); +} + +void HTTPConnector::connectError(const TTransportException& ex) noexcept { + socket_.reset(); + if (cb_) { + cb_->connectError(ex); + } +} + +} diff --git a/proxygen/lib/http/HTTPConnector.h b/proxygen/lib/http/HTTPConnector.h new file mode 100644 index 0000000000..9c6a81891d --- /dev/null +++ b/proxygen/lib/http/HTTPConnector.h @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/services/TransportInfo.h" +#include "proxygen/lib/utils/Time.h" + +#include +#include +#include + +namespace proxygen { + +class HTTPUpstreamSession; + +/** + * This class establishes new connections to HTTP or HTTPS servers. It + * can be reused, even to connect to different addresses, but it can only + * service setting up one connection at a time. + */ +class HTTPConnector: + private apache::thrift::async::TAsyncSocket::ConnectCallback { + public: + /** + * This class defines the pure virtual interface on which to receive the + * result on. + */ + class Callback { + public: + virtual ~Callback() {} + virtual void connectSuccess(HTTPUpstreamSession* session) = 0; + virtual void connectError( + const apache::thrift::transport::TTransportException& ex) = 0; + }; + + /** + * Construct a HTTPConnector. The constructor arguments are those + * parameters HTTPConnector needs to keep a copy of through the + * connection process. + * + * @param callback The interface on which to receive the result. + * Whatever object is passed here MUST outlive this + * connector and MUST NOT be null. + * @param timeoutSet The timeout set to be used for the transactions + * that are opened on the session. + * @param plaintextProto An optional protocol string to specify the + * next protocol to use for unsecure connections. + * If omitted, http/1.1 will be assumed. + * @param forceHTTP1xCodecTo11 If true and this connector creates + * a session using an HTTP1xCodec, that codec will + * only serialize messages as HTTP/1.1. + */ + HTTPConnector(Callback* callback, + apache::thrift::async::TAsyncTimeoutSet* timeoutSet, + const std::string& plaintextProto = "", + bool forceHTTP1xCodecTo11 = false); + + /** + * Clients may delete the connector at any time to cancel it. No + * callbacks will be received. + */ + ~HTTPConnector(); + + /** + * Reset the object so that it can begin a new connection. No callbacks + * will be invoked as a result of executing this function. After this + * function returns, isBusy() will return false. + */ + void reset(); + + /** + * Begin the process of getting a plaintext connection to the server + * specified by 'connectAddr'. This function immediately starts async + * work and may invoke functions on Callback immediately. + * + * @param eventBase The event base to put events on. + * @param connectAddr The address to connect to. + * @param timeoutMs Optional. If this value is greater than zero, then a + * connect error will be given if no connection is + * established within this amount of time. + * @param socketOptions Optional socket options to set on the connection. + * @param bindAddr Optional address to bind to locally. + */ + void connect( + folly::EventBase* eventBase, + const folly::SocketAddress& connectAddr, + std::chrono::milliseconds timeoutMs = std::chrono::milliseconds(0), + const apache::thrift::async::TAsyncSocket::OptionMap& socketOptions = + apache::thrift::async::TAsyncSocket::emptyOptionMap, + const folly::SocketAddress& bindAddr = + apache::thrift::async::TAsyncSocket::anyAddress); + + /** + * Begin the process of getting a secure connection to the server + * specified by 'connectAddr'. This function immediately starts async + * work and may invoke functions on Callback immediately. + * + * @param eventBase The event base to put events on. + * @param connectAddr The address to connect to. + * @param ctx SSL context to use. Must not be null. + * @param session Optional ssl session to use. + * @param timeoutMs Optional. If this value is greater than zero, then a + * connect error will be given if no connection is + * established within this amount of time. + * @param socketOptions Optional socket options to set on the connection. + * @param bindAddr Optional address to bind to locally. + */ + void connectSSL( + folly::EventBase* eventBase, + const folly::SocketAddress& connectAddr, + const std::shared_ptr& ctx, + SSL_SESSION* session = nullptr, + std::chrono::milliseconds timeoutMs = std::chrono::milliseconds(0), + const apache::thrift::async::TAsyncSocket::OptionMap& socketOptions = + apache::thrift::async::TAsyncSocket::emptyOptionMap, + const folly::SocketAddress& bindAddr = + apache::thrift::async::TAsyncSocket::anyAddress); + + /** + * @returns the number of milliseconds since connecting began, or + * zero if connecting hasn't started yet. + */ + std::chrono::milliseconds timeElapsed(); + + /** + * @returns true iff this connector is busy setting up a connection. If + * this is false, it is safe to call connect() or connectSSL() on it again. + */ + bool isBusy() const { return socket_.get(); } + + private: + void connectSuccess() noexcept override; + void connectError(const apache::thrift::transport::TTransportException& ex) + noexcept override; + + Callback* cb_; + apache::thrift::async::TAsyncTimeoutSet* timeoutSet_; + apache::thrift::async::TAsyncSocket::UniquePtr socket_; + TransportInfo transportInfo_; + std::string plaintextProtocol_; + TimePoint connectStart_; + bool forceHTTP1xCodecTo1_1_; +}; + +} diff --git a/proxygen/lib/http/HTTPConstants.cpp b/proxygen/lib/http/HTTPConstants.cpp new file mode 100644 index 0000000000..928bb1563f --- /dev/null +++ b/proxygen/lib/http/HTTPConstants.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPConstants.h" + +namespace proxygen { + +#define CONNECTION_CLOSE_REASON_STRING(e, r) r, +const char* connectionCloseStrings[] = { + CONNECTION_CLOSE_REASON_GEN(CONNECTION_CLOSE_REASON_STRING) +}; +#undef CONNECTION_CLOSE_REASON_STRING + +const char* getConnectionCloseReasonStringByIndex(unsigned int index) { + if (index >= (unsigned int)ConnectionCloseReason::kMAX_REASON) { + index = (unsigned int)ConnectionCloseReason::kMAX_REASON - 1; + } + + return connectionCloseStrings[index]; +} + +const char* getConnectionCloseReasonString(ConnectionCloseReason r) { + return connectionCloseStrings[(unsigned int)r]; +} + +} diff --git a/proxygen/lib/http/HTTPConstants.h b/proxygen/lib/http/HTTPConstants.h new file mode 100644 index 0000000000..1fa0c22719 --- /dev/null +++ b/proxygen/lib/http/HTTPConstants.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +//enum class ConnectionCloseReason : unsigned int { +// SHUTDOWN, // shutdown...probably due to the short shutdown +// // time we won't be able to see any of this +// READ_EOF, // received FIN from client so the connection is +// // not reusable any more +// GOAWAY, // session closed due to ingress goaway +// SESSION_PARSE_ERROR, // http/spdy parse error +// REMOTE_ERROR, // The various 5xx error +// TRANSACTION_ABORT, // transaction sendAbort() +// TIMEOUT, // read/write timeout excluding shutdown +// IO_READ_ERROR, // read error +// IO_WRITE_ERROR, // write error +// REQ_NOTREUSABLE, // client request is not reusable (http 1.0 or +// // Connection: close) +// ERR_RESP, // various 4xx error +// UNKNOWN, // this probably indicate some bug that the close +// // reason is not accounted for +// kMAX_REASON +//}; + +#define CONNECTION_CLOSE_REASON_GEN(x) \ + x(SHUTDOWN, "shutdown") \ + x(READ_EOF, "read_eof") \ + x(GOAWAY, "goaway") \ + x(SESSION_PARSE_ERROR, "session_parse_err") \ + x(REMOTE_ERROR, "remote_err") \ + x(TRANSACTION_ABORT, "transaction_abort") \ + x(TIMEOUT, "timeout") \ + x(IO_READ_ERROR, "io_read_err") \ + x(IO_WRITE_ERROR, "io_write_err") \ + x(REQ_NOTREUSABLE, "req_not_reusable") \ + x(ERR_RESP, "err_resp") \ + x(UNKNOWN, "unknown") \ + x(FLOW_CONTROL, "flow_control") \ + x(kMAX_REASON, "unset") + +#define CONNECTION_CLOSE_REASON_ENUM(e, r) e, +enum class ConnectionCloseReason { + CONNECTION_CLOSE_REASON_GEN(CONNECTION_CLOSE_REASON_ENUM) +}; +#undef CONNECTION_CLOSE_REASON_ENUM + +extern const char* getConnectionCloseReasonStringByIndex(unsigned int i); +extern const char* getConnectionCloseReasonString(ConnectionCloseReason r); + +/** + * Protocol to which the HTTPTransaction was upgraded + */ +enum class UpgradeProtocol: int { + // We only support changing to TCP after CONNECT requests + TCP +}; + +} diff --git a/proxygen/lib/http/HTTPException.cpp b/proxygen/lib/http/HTTPException.cpp new file mode 100644 index 0000000000..e0d6154e77 --- /dev/null +++ b/proxygen/lib/http/HTTPException.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPException.h" + +namespace proxygen { + +std::string HTTPException::describe() const { + std::stringstream ss; + ss << *this; + return ss.str(); +} + +std::ostream& operator<<(std::ostream& os, const HTTPException& ex) { + os << "what=\"" << ex.what() + << "\", direction=" << static_cast(ex.getDirection()) + << ", proxygenError=" << getErrorString(ex.getProxygenError()) + << ", codecStatusCode=" << (ex.hasCodecStatusCode() ? + getErrorCodeString(ex.getCodecStatusCode()) : + "-1") + << ", httpStatusCode=" << ex.getHttpStatusCode(); + return os; +} + +} diff --git a/proxygen/lib/http/HTTPException.h b/proxygen/lib/http/HTTPException.h new file mode 100644 index 0000000000..738a5c69ea --- /dev/null +++ b/proxygen/lib/http/HTTPException.h @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/http/ProxygenErrorEnum.h" +#include "proxygen/lib/http/codec/ErrorCode.h" +#include "proxygen/lib/utils/Exception.h" + +#include +#include + +namespace proxygen { + +/** + * This class encapsulates the various errors that can occur on an + * http session. Errors can occur at various levels: the connection can + * be closed for reads and/or writes, the message body may fail to parse, + * or various protocol constraints may be violated. + */ +class HTTPException : public proxygen::Exception { + public: + /** + * Indicates which direction of the data flow was affected by this + * exception. For instance, if a class receives HTTPException(INGRESS), + * then it should consider ingress callbacks finished (whether or not + * the underlying transport actually shut down). Likewise for + * HTTPException(EGRESS), the class should consider the write stream + * shut down. INGRESS_AND_EGRESS indicates both directions are finished. + */ + enum class Direction { + INGRESS = 0, + EGRESS, + INGRESS_AND_EGRESS, + }; + + explicit HTTPException(Direction dir, const std::string& msg) + : Exception(msg), + dir_(dir) {} + + template + explicit HTTPException(Direction dir, Args&&... args) + : Exception(std::forward(args)...), + dir_(dir) {} + + HTTPException(const HTTPException& ex) : + Exception(static_cast(ex)), + dir_(ex.dir_), + proxygenError_(ex.proxygenError_), + httpStatusCode_(ex.httpStatusCode_), + codecStatusCode_(ex.codecStatusCode_), + errno_(ex.errno_) { + if (ex.currentIngressBuf_) { + currentIngressBuf_ = std::move(ex.currentIngressBuf_->clone()); + } + if (ex.partialMsg_) { + partialMsg_ = folly::make_unique(*ex.partialMsg_.get()); + } + } + + /** + * Returns a string representation of this exception. This function is + * intended for debugging and logging only. For the true exception + * string, use what() + */ + std::string describe() const; + + Direction getDirection() const { + return dir_; + } + + bool isIngressException() const { + return dir_ == Direction::INGRESS || + dir_ == Direction::INGRESS_AND_EGRESS; + } + + bool isEgressException() const { + return dir_ == Direction::EGRESS || + dir_ == Direction::INGRESS_AND_EGRESS; + } + + // Accessors for ProxygenError + bool hasProxygenError() const { + return (proxygenError_ != kErrorNone); + } + + void setProxygenError(ProxygenError proxygenError) { + proxygenError_ = proxygenError; + } + + ProxygenError getProxygenError() const { + return proxygenError_; + } + + // Accessors for HTTP error codes + bool hasHttpStatusCode() const { + return (httpStatusCode_ != 0); + } + + void setHttpStatusCode(uint32_t statusCode) { + httpStatusCode_ = statusCode; + } + + uint32_t getHttpStatusCode() const { + return httpStatusCode_; + } + + // Accessors for Codec specific status codes + bool hasCodecStatusCode() const { + return codecStatusCode_.hasValue(); + } + void setCodecStatusCode(ErrorCode statusCode) { + codecStatusCode_ = statusCode; + } + ErrorCode getCodecStatusCode() const { + CHECK(hasCodecStatusCode()); + return *codecStatusCode_; + } + + // Accessors for errno + bool hasErrno() const { + return (errno_ != 0); + } + void setErrno(uint32_t errno) { + errno_ = errno; + } + uint32_t getErrno() const { + return errno_; + } + + void setCurrentIngressBuf(std::unique_ptr buf) { + currentIngressBuf_ = std::move(buf); + } + + std::unique_ptr moveCurrentIngressBuf() { + return std::move(currentIngressBuf_); + } + + void setPartialMsg(std::unique_ptr partialMsg) { + partialMsg_ = std::move(partialMsg); + } + + std::unique_ptr movePartialMsg() { + return std::move(partialMsg_); + } + + private: + + Direction dir_; + ProxygenError proxygenError_{kErrorNone}; + uint32_t httpStatusCode_{0}; + folly::Optional codecStatusCode_; + uint32_t errno_{0}; + // current ingress buffer, may be compressed + std::unique_ptr currentIngressBuf_; + // partial message that is being parsed + std::unique_ptr partialMsg_; +}; + +std::ostream& operator<<(std::ostream& os, const HTTPException& ex); + +} diff --git a/proxygen/lib/http/HTTPHeaderSize.h b/proxygen/lib/http/HTTPHeaderSize.h new file mode 100644 index 0000000000..4155966c9a --- /dev/null +++ b/proxygen/lib/http/HTTPHeaderSize.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +/** + * A structure that encapsulates byte counters related to the HTTP headers. + */ +struct HTTPHeaderSize { + /** + * The number of bytes used to represent the header after compression or + * before decompression. If header compression is not supported, the value + * is set to 0. + */ + uint32_t compressed{0}; + + /** + * The number of bytes used to represent the serialized header before + * compression or after decompression, in plain-text format. + */ + uint32_t uncompressed{0}; +}; + +} diff --git a/proxygen/lib/http/HTTPHeaders.cpp b/proxygen/lib/http/HTTPHeaders.cpp new file mode 100644 index 0000000000..91dd75a6df --- /dev/null +++ b/proxygen/lib/http/HTTPHeaders.cpp @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#define PROXYGEN_HTTPHEADERS_IMPL +#include "proxygen/lib/http/HTTPHeaders.h" + +#include +#include + +using std::bitset; +using std::string; +using std::vector; + +namespace proxygen { + +const string empty_string; + +bitset<256>& HTTPHeaders::perHopHeaderCodes() { + static bitset<256> perHopHeaderCodes; + return perHopHeaderCodes; +} + +void +HTTPHeaders::initGlobals() { + HTTPCommonHeaders::initHeaderNames(); + + auto& perHopHeaders = perHopHeaderCodes(); + perHopHeaders[HTTP_HEADER_CONNECTION] = true; + perHopHeaders[HTTP_HEADER_KEEP_ALIVE] = true; + perHopHeaders[HTTP_HEADER_PROXY_AUTHENTICATE] = true; + perHopHeaders[HTTP_HEADER_PROXY_AUTHORIZATION] = true; + perHopHeaders[HTTP_HEADER_PROXY_CONNECTION] = true; + perHopHeaders[HTTP_HEADER_TE] = true; + perHopHeaders[HTTP_HEADER_TRAILER] = true; + perHopHeaders[HTTP_HEADER_TRANSFER_ENCODING] = true; + perHopHeaders[HTTP_HEADER_UPGRADE] = true; +} + +HTTPHeaders::HTTPHeaders() : + deletedCount_(0) { + codes_.reserve(kInitialVectorReserve); + headerNames_.reserve(kInitialVectorReserve); + headerValues_.reserve(kInitialVectorReserve); +} + +void HTTPHeaders::add(folly::StringPiece name, folly::StringPiece value) { + CHECK(name.size()); + const HTTPHeaderCode code = HTTPCommonHeaders::hash(name.data(), name.size()); + codes_.push_back(code); + headerNames_.push_back((code == HTTP_HEADER_OTHER) + ? new std::string(name.data(), name.size()) + : HTTPCommonHeaders::getPointerToHeaderName(code)); + headerValues_.emplace_back(value.data(), value.size()); +} + +void HTTPHeaders::rawAdd(const std::string& name, const std::string& value) { + add(name, value); +} + +void HTTPHeaders::addFromCodec(const char* str, size_t len, string&& value) { + const HTTPHeaderCode code = HTTPCommonHeaders::hash(str, len); + codes_.push_back(code); + headerNames_.push_back((code == HTTP_HEADER_OTHER) + ? new string(str, len) + : HTTPCommonHeaders::getPointerToHeaderName(code)); + headerValues_.emplace_back(std::move(value)); +} + +bool HTTPHeaders::exists(folly::StringPiece name) const { + const HTTPHeaderCode code = HTTPCommonHeaders::hash(name.data(), + name.size()); + if (code != HTTP_HEADER_OTHER) { + return exists(code); + } else { + ITERATE_OVER_STRINGS(name, { return true; }); + return false; + } +} + +bool HTTPHeaders::exists(HTTPHeaderCode code) const { + return memchr((void*)codes_.data(), code, codes_.size()) != nullptr; +} + +size_t HTTPHeaders::getNumberOfValues(HTTPHeaderCode code) const { + size_t count = 0; + ITERATE_OVER_CODES(code, { + ++count; + }); + return count; +} + +size_t HTTPHeaders::getNumberOfValues(folly::StringPiece name) const { + size_t count = 0; + forEachValueOfHeader(name, [&] (folly::StringPiece value) -> bool { + ++count; + return false; + }); + return count; +} + +bool HTTPHeaders::remove(folly::StringPiece name) { + const HTTPHeaderCode code = HTTPCommonHeaders::hash(name.data(), + name.size()); + if (code != HTTP_HEADER_OTHER) { + return remove(code); + } else { + bool removed = false; + ITERATE_OVER_STRINGS(name, { + delete headerNames_[pos]; + codes_[pos] = HTTP_HEADER_NONE; + removed = true; + ++deletedCount_; + }); + return removed; + } +} + +bool HTTPHeaders::remove(HTTPHeaderCode code) { + bool removed = false; + ITERATE_OVER_CODES(code, { + codes_[pos] = HTTP_HEADER_NONE; + removed = true; + ++deletedCount_; + }); + return removed; +} + +void HTTPHeaders::disposeOfHeaderNames() { + for (size_t i = 0; i < codes_.size(); ++i) { + if (codes_[i] == HTTP_HEADER_OTHER) { + delete headerNames_[i]; + } + } +} + +HTTPHeaders::~HTTPHeaders () { + disposeOfHeaderNames(); +} + +HTTPHeaders::HTTPHeaders(const HTTPHeaders& hdrs) : + codes_(hdrs.codes_), + headerNames_(hdrs.headerNames_), + headerValues_(hdrs.headerValues_), + deletedCount_(hdrs.deletedCount_) { + for (size_t i = 0; i < codes_.size(); ++i) { + if (codes_[i] == HTTP_HEADER_OTHER) { + headerNames_[i] = new string(*hdrs.headerNames_[i]); + } + } +} + +HTTPHeaders::HTTPHeaders(HTTPHeaders&& hdrs) noexcept : + codes_(std::move(hdrs.codes_)), + headerNames_(std::move(hdrs.headerNames_)), + headerValues_(std::move(hdrs.headerValues_)), + deletedCount_(hdrs.deletedCount_) { + hdrs.removeAll(); +} + +HTTPHeaders& HTTPHeaders::operator= (const HTTPHeaders& hdrs) { + if (this != &hdrs) { + disposeOfHeaderNames(); + codes_ = hdrs.codes_; + headerNames_ = hdrs.headerNames_; + headerValues_ = hdrs.headerValues_; + deletedCount_ = hdrs.deletedCount_; + for (size_t i = 0; i < codes_.size(); ++i) { + if (codes_[i] == HTTP_HEADER_OTHER) { + headerNames_[i] = new string(*hdrs.headerNames_[i]); + } + } + } + return *this; +} + +HTTPHeaders& HTTPHeaders::operator= (HTTPHeaders&& hdrs) { + if (this != &hdrs) { + codes_ = std::move(hdrs.codes_); + headerNames_ = std::move(hdrs.headerNames_); + headerValues_ = std::move(hdrs.headerValues_); + deletedCount_ = hdrs.deletedCount_; + + hdrs.removeAll(); + } + + return *this; +} + +void HTTPHeaders::removeAll() { + disposeOfHeaderNames(); + + codes_.clear(); + headerNames_.clear(); + headerValues_.clear(); + deletedCount_ = 0; +} + +size_t HTTPHeaders::size() const { + return codes_.size() - deletedCount_; +} + +bool +HTTPHeaders::transferHeaderIfPresent(folly::StringPiece name, + HTTPHeaders& strippedHeaders) { + bool transferred = false; + const HTTPHeaderCode code = HTTPCommonHeaders::hash(name.data(), + name.size()); + if (code == HTTP_HEADER_OTHER) { + ITERATE_OVER_STRINGS(name, { + strippedHeaders.codes_.push_back(HTTP_HEADER_OTHER); + // in the next line, ownership of pointer goes to strippedHeaders + strippedHeaders.headerNames_.push_back(headerNames_[pos]); + strippedHeaders.headerValues_.push_back(headerValues_[pos]); + codes_[pos] = HTTP_HEADER_NONE; + transferred = true; + ++deletedCount_; + }); + } else { // code != HTTP_HEADER_OTHER + ITERATE_OVER_CODES(code, { + strippedHeaders.codes_.push_back(code); + strippedHeaders.headerNames_.push_back(headerNames_[pos]); + strippedHeaders.headerValues_.push_back(headerValues_[pos]); + codes_[pos] = HTTP_HEADER_NONE; + transferred = true; + ++deletedCount_; + }); + } + return transferred; +} + +void +HTTPHeaders::stripPerHopHeaders(HTTPHeaders& strippedHeaders) { + int len; + forEachValueOfHeader(HTTP_HEADER_CONNECTION, [&] + (const string& stdStr) -> bool { + // Remove all headers specified in Connection header + // look for multiple values separated by commas + char const* str = stdStr.c_str(); + + // skip leading whitespace + while (isLWS(*str)) str++; + + while (*str != 0) { + char const* pos = strchr(str, ','); + if (pos == nullptr) { + // last (or only) token, done + + // count chars in the token + len = 0; + while (str[len] != 0 && !isLWS(str[len])) len++; + if (len > 0) { + string hdr(str, len); + if (transferHeaderIfPresent(hdr, strippedHeaders)) { + VLOG(3) << "Stripped connection-named hop-by-hop header " << hdr; + } + } + break; + } + len = pos - str; + // strip trailing whitespace + while (len > 0 && isLWS(str[len - 1])) len--; + if (len > 0) { + // non-empty token + string hdr(str, len); + if (transferHeaderIfPresent(hdr, strippedHeaders)) { + VLOG(3) << "Stripped connection-named hop-by-hop header " << hdr; + } + } // else empty token, no-op + str = pos + 1; + + // skip whitespace + while (isLWS(*str)) str++; + } + return false; // continue processing "connection" headers + }); + + // Strip hop-by-hop headers + auto& perHopHeaders = perHopHeaderCodes(); + for (size_t i = 0; i < codes_.size(); ++i) { + if (perHopHeaders[codes_[i]]) { + strippedHeaders.codes_.push_back(codes_[i]); + strippedHeaders.headerNames_.push_back(headerNames_[i]); + strippedHeaders.headerValues_.push_back(headerValues_[i]); + codes_[i] = HTTP_HEADER_NONE; + ++deletedCount_; + VLOG(3) << "Stripped hop-by-hop header " << *headerNames_[i]; + } + } +} + +void HTTPHeaders::copyTo(HTTPHeaders& hdrs) const { + for (size_t i = 0; i < codes_.size(); ++i) { + if (codes_[i] != HTTP_HEADER_NONE) { + hdrs.codes_.push_back(codes_[i]); + hdrs.headerNames_.push_back((codes_[i] == HTTP_HEADER_OTHER) ? + new string(*headerNames_[i]) : headerNames_[i]); + hdrs.headerValues_.push_back(headerValues_[i]); + } + } +} + +} diff --git a/proxygen/lib/http/HTTPHeaders.h b/proxygen/lib/http/HTTPHeaders.h new file mode 100644 index 0000000000..2594d1cc3d --- /dev/null +++ b/proxygen/lib/http/HTTPHeaders.h @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include "proxygen/lib/http/HTTPCommonHeaders.h" +#include "proxygen/lib/utils/UtilInl.h" + +#include +#include +#include + +namespace proxygen { + +extern const std::string empty_string; + +/** + * Return true if the character is linear whitespace, as defined by the LWS + * definition in RFC 2616, and false otherwise + */ +inline bool isLWS(char c) { + // Technically \r and \n are only allowed in LWS if they appear together. + if (c == ' ' || c == '\t' || c == '\r' || c == '\n') { + return true; + } + return false; +} + +/** + * A collection of HTTP headers. + * + * This is broken out from HTTPMessage, as it's convenient for other things to + * be able to use collections of HTTP headers that are easy to work with. The + * structure is optimized for real-life header collection sizes. + * + * Headers are stored as Name/Value pairs, in the order they are received on + * the wire. We hash the names of all common HTTP headers (using a static + * perfect hash function generated using gperf from HTTPCommonHeaders.gperf) + * into 1-byte hashes (we call them "codes") and only store these. We search + * them using memchr, which has an x86_64 assembly implementation with + * complexity O(n/16) ;) + * + * Instead of creating strings with header names, we point to a static array + * of strings in HTTPCommonHeaders. If the header name is not in our set of + * common header names (this is considered unlikely, because we intend this set + * to be very complete), then we create a new string with its name (we own that + * pointer then). For such headers, we store the code HTTP_HEADER_OTHER. + * + * The code HTTP_HEADER_NONE signifies a header that has been removed. + * + * Most methods which take a header name have two versions: one accepting + * a string, and one accepting a code. It is recommended to use the latter + * if possible, as in: + * headers.add(HTTP_HEADER_LOCATION, location); + * rather than: + * headers.add("Location", location); + */ +class HTTPHeaders { + public: + /* + * separator used to concatenate multiple values of the same header + * check out sections 4.2 and 14.45 from rfc2616 + */ + const std::string COMBINE_SEPARATOR = ", "; + + HTTPHeaders(); + ~HTTPHeaders(); + HTTPHeaders (const HTTPHeaders&); + HTTPHeaders& operator= (const HTTPHeaders&); + HTTPHeaders (HTTPHeaders&&) noexcept; + HTTPHeaders& operator= (HTTPHeaders&&); + + /** + * Add the header 'name' with value 'value'; if other instances of this + * header name exist, they will be retained. + */ + void add(folly::StringPiece name, folly::StringPiece value); + template // T = string + void add(folly::StringPiece name, T&& value); + template // T = string + void add(HTTPHeaderCode code, T&& value); + void rawAdd(const std::string& name, const std::string& value); + + void addFromCodec(const char* str, size_t len, std::string&& value); + + /** + * For the header 'name', set its value to the single header 'value', + * removing any other instances of this header. + */ + void set(folly::StringPiece name, const std::string& value) { + // this could be somewhat optimized but probably not an issue yet + remove(name); + add(name, value); + } + void set(HTTPHeaderCode code, const std::string& value) { + remove(code); + add(code, value); + } + void rawSet(const std::string& name, const std::string& value) { + set(name, value); + } + + /** + * Do we have an instance of the given header? + */ + bool exists(folly::StringPiece name) const; + bool exists(HTTPHeaderCode code) const; + bool rawExists(std::string& name) const { + return exists(name); + } + + /** + * combine all the value for this header into a string + */ + template + std::string combine(const T& header) const; + + /** + * Process the list of all headers, in the order that they were seen: + * for each header:value pair, the function/functor/lambda-expression + * given as the second parameter will be executed. It should take two + * const string & parameters and return void. Example use: + * hdrs.forEach([&] (const string& header, const string& val) { + * std::cout << header << ": " << val; + * }); + */ + template // (const string &, const string &) -> void + inline void forEach(LAMBDA func) const; + + /** + * Process the list of all headers, in the order that they were seen: + * for each header:value pair, the function/functor/lambda-expression + * given as the second parameter will be executed. It should take one + * HTTPHeaderCode (code) parameter, two const string & parameters and + * return void. Example use: + * hdrs.forEachWithCode([&] (HTTPHeaderCode code, + * const string& header, + * const string& val) { + * std::cout << header << "(" << code << "): " << val; + * }); + */ + template + inline void forEachWithCode(LAMBDA func) const; + + /** + * Process the list of all headers, in the order that they were seen: + * for each header:value pair, the function/functor/lambda-expression + * given as the parameter will be executed to determine whether the + * header should be removed. Example use: + * + * hdrs.removeByPredicate([&] (HTTPHeaderCode code, + * const string& header, + * const string& val) { + * return boost::regex_match(header, "^X-Fb-.*"); + * }); + * + * return true only if one or more headers are removed. + */ + template // (const string &, const string &) -> bool + inline bool removeByPredicate(LAMBDA func); + + /** + * Returns the value of the header if it's found in the message and is the + * only value under the given name. If either of these is violated, returns + * empty_string. + */ + template // either uint8_t or string + const std::string & getSingleOrEmpty(const T& nameOrCode) const; + const std::string rawGet(const std::string& header) const { + return getSingleOrEmpty(header); + } + + /** + * Get the number of values corresponding to a given header name. + */ + size_t getNumberOfValues(HTTPHeaderCode code) const; + size_t getNumberOfValues(folly::StringPiece name) const; + + /** + * Process the ordered list of values for the given header name: + * for each value, the function/functor/lambda-expression given as the second + * parameter will be executed. It should take one const string & parameter + * and return bool (false to keep processing, true to stop it). Example use: + * hdrs.forEachValueOfHeader("someheader", [&] (const string& val) { + * std::cout << val; + * return false; + * }); + * This method returns true if processing was stopped (by func returning + * true), and false otherwise. + */ + template // const string & -> bool + inline bool forEachValueOfHeader(folly::StringPiece name, LAMBDA func) const; + template // const string & -> bool + inline bool forEachValueOfHeader(HTTPHeaderCode code, LAMBDA func) const; + + /** + * Remove all instances of the given header, returning true if anything was + * removed and false if this header didn't exist in our set. + */ + bool remove(folly::StringPiece name); + bool remove(HTTPHeaderCode code); + void rawRemove(const std::string& name) { + remove(name); + } + + /** + * Remove all headers. + */ + void removeAll(); + + /** + * Remove per-hop-headers and headers named in the Connection header + * and place the value in strippedHeaders + */ + void stripPerHopHeaders(HTTPHeaders& strippedHeaders); + + /** + * Get the total number of headers. + */ + size_t size() const; + + /** + * Copy all headers from this to hdrs. + */ + void copyTo(HTTPHeaders& hdrs) const; + + /** + * Determines whether header with a given code is a per-hop header, + * which should be stripped by stripPerHopHeaders(). + */ + static std::bitset<256>& perHopHeaderCodes(); + + private: + // vector storing the 1-byte hashes of header names + folly::fbvector codes_; + + /** + * Vector storing pointers to header names; we own those pointers which + * correspond to HTTP_HEADER_OTHER codes. + */ + folly::fbvector headerNames_; + + folly::fbvector headerValues_; + + size_t deletedCount_; + + /** + * The initial capacity of the three vectors, reserved right after + * construction. + */ + static const size_t kInitialVectorReserve = 16; + + /** + * Moves the named header and values from this group to the destination + * group. No-op if the header doesn't exist. Returns true if header(s) were + * moved. + */ + bool transferHeaderIfPresent(folly::StringPiece name, HTTPHeaders& dest); + + static void initGlobals() __attribute__ ((__constructor__)); + + // deletes the strings in headerNames_ that we own + void disposeOfHeaderNames(); +}; + +// Implementation follows - it has to be in the .h because of the templates + +template // T = string +void HTTPHeaders::add(folly::StringPiece name, T&& value) { + assert(name.size()); + const HTTPHeaderCode code = HTTPCommonHeaders::hash(name.data(), name.size()); + codes_.push_back(code); + headerNames_.push_back((code == HTTP_HEADER_OTHER) + ? new std::string(name.data(), name.size()) + : HTTPCommonHeaders::getPointerToHeaderName(code)); + headerValues_.emplace_back(std::forward(value)); +} + +template // T = string +void HTTPHeaders::add(HTTPHeaderCode code, T&& value) { + codes_.push_back(code); + headerNames_.push_back(HTTPCommonHeaders::getPointerToHeaderName(code)); + headerValues_.emplace_back(std::forward(value)); +} + +// iterate over the positions (in vector) of all headers with given code +#define ITERATE_OVER_CODES(Code, Block) { \ + const HTTPHeaderCode* ptr = codes_.data(); \ + while(true) { \ + ptr = (HTTPHeaderCode*) memchr((void*)ptr, (Code), \ + codes_.size() - (ptr - codes_.data())); \ + if (ptr == nullptr) break; \ + const int pos = ptr - codes_.data(); \ + {Block} \ + ptr++; \ + } \ +} + +// iterate over the positions of all headers with given name +#define ITERATE_OVER_STRINGS(String, Block) \ + ITERATE_OVER_CODES(HTTP_HEADER_OTHER, { \ + if (caseInsensitiveEqual((String), *headerNames_[pos])) { \ + {Block} \ + } \ +}) + +template // (const string &, const string &) -> void +void HTTPHeaders::forEach(LAMBDA func) const { + for (size_t i = 0; i < codes_.size(); ++i) { + if (codes_[i] != HTTP_HEADER_NONE) { + func(*headerNames_[i], headerValues_[i]); + } + } +} + +template +void HTTPHeaders::forEachWithCode(LAMBDA func) const { + for (size_t i = 0; i < codes_.size(); ++i) { + if (codes_[i] != HTTP_HEADER_NONE) { + func(codes_[i], *headerNames_[i], headerValues_[i]); + } + } +} + +template // const string & -> bool +bool HTTPHeaders::forEachValueOfHeader(folly::StringPiece name, + LAMBDA func) const { + const HTTPHeaderCode code = HTTPCommonHeaders::hash(name.data(), name.size()); + if (code != HTTP_HEADER_OTHER) { + return forEachValueOfHeader(code, func); + } else { + ITERATE_OVER_STRINGS(name, { + if (func(headerValues_[pos])) { + return true; + } + }); + return false; + } +} + +template // const string & -> bool +bool HTTPHeaders::forEachValueOfHeader(HTTPHeaderCode code, + LAMBDA func) const { + ITERATE_OVER_CODES(code, { + if (func(headerValues_[pos])) { + return true; + } + }); + return false; +} + +template +std::string HTTPHeaders::combine(const T& header) const { + std::string combined = ""; + forEachValueOfHeader(header, [&] (const std::string& value) -> bool { + if (combined.empty()) { + combined.append(value); + } else { + combined.append(COMBINE_SEPARATOR).append(value); + } + return false; + }); + return combined; +} + +// LAMBDA: (HTTPHeaderCode, const string&, const string&) -> bool +template +bool HTTPHeaders::removeByPredicate(LAMBDA func) { + bool removed = false; + for (size_t i = 0; i < codes_.size(); ++i) { + if (codes_[i] == HTTP_HEADER_NONE || + !func(codes_[i], *headerNames_[i], headerValues_[i])) { + continue; + } + + if (codes_[i] == HTTP_HEADER_OTHER) { + delete headerNames_[i]; + headerNames_[i] = nullptr; + } + + codes_[i] = HTTP_HEADER_NONE; + ++deletedCount_; + removed = true; + } + + return removed; +} + +template // either uint8_t or string +const std::string & HTTPHeaders::getSingleOrEmpty(const T& nameOrCode) const { + const std::string* res = nullptr; + forEachValueOfHeader(nameOrCode, [&] (const std::string& value) -> bool { + if (res != nullptr) { + // a second value is found + res = nullptr; + return true; // stop processing + } else { + // the first value is found + res = &value; + return false; + } + }); + if (res == nullptr) { + return empty_string; + } else { + return *res; + } +} + +#ifndef PROXYGEN_HTTPHEADERS_IMPL +#undef ITERATE_OVER_CODES +#undef ITERATE_OVER_STRINGS +#endif // PROXYGEN_HTTPHEADERS_IMPL + +} diff --git a/proxygen/lib/http/HTTPMessage.cpp b/proxygen/lib/http/HTTPMessage.cpp new file mode 100644 index 0000000000..e251939e16 --- /dev/null +++ b/proxygen/lib/http/HTTPMessage.cpp @@ -0,0 +1,831 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPMessage.h" + +#include +#include +#include +#include +#include +#include +#include + +using folly::IOBuf; +using folly::Optional; +using folly::StringPiece; +using std::pair; +using std::string; +using std::unique_ptr; + +namespace { +const string kHeaderStr_ = "header."; +const string kQueryStr_ = "query."; +const string kCookieStr = "cookie."; + +/** + * Create a C locale once and pass it to all the boost string methods + * that would otherwise create and destruct a temporary locale object + * per call. (Performance profiling showed that we were spending + * approximately 1% of our total CPU time on temporary locale objects.) + */ +std::locale defaultLocale; +} + +namespace proxygen { + +std::mutex HTTPMessage::mutexDump_; + +const pair HTTPMessage::kHTTPVersion10(1, 0); +const pair HTTPMessage::kHTTPVersion11(1, 1); + +void HTTPMessage::stripPerHopHeaders() { + // Some code paths end up recyling a single HTTPMessage instance for multiple + // requests, and adding their own per-hop headers each time. In that case, we + // don't want to accumulate these headers. + strippedPerHopHeaders_.removeAll(); + + if (!trailersAllowed_) { + // Because stripPerHopHeaders can be called multiple times, don't + // let subsequent instances clear this flag + trailersAllowed_ = checkForHeaderToken(HTTP_HEADER_TE, "trailers", false); + } + + headers_.stripPerHopHeaders(strippedPerHopHeaders_); +} + +HTTPMessage::HTTPMessage() : + startTime_(getCurrentTime()), + seqNo_(-1), + localIP_(), + versionStr_("1.0"), + fields_(), + version_(1,0), + sslVersion_(0), sslCipher_(nullptr), spdy_(0), pri_(0), + parsedCookies_(false), parsedQueryParams_(false), + chunked_(false), upgraded_(false), wantsKeepalive_(true), + trailersAllowed_(false), secure_(false) { +} + +HTTPMessage::~HTTPMessage() { +} + +HTTPMessage::HTTPMessage(const HTTPMessage& message) : + startTime_(message.startTime_), + seqNo_(message.seqNo_), + dstAddress_(message.dstAddress_), + dstIP_(message.dstIP_), + dstPort_(message.dstPort_), + localIP_(message.localIP_), + versionStr_(message.versionStr_), + fields_(message.fields_), + cookies_(message.cookies_), + queryParams_(message.queryParams_), + version_(message.version_), + headers_(message.headers_), + strippedPerHopHeaders_(message.headers_), + sslVersion_(message.sslVersion_), + sslCipher_(message.sslCipher_), + spdy_(message.spdy_), + parsedCookies_(message.parsedCookies_), + parsedQueryParams_(message.parsedQueryParams_), + chunked_(message.chunked_), + upgraded_(message.upgraded_), + wantsKeepalive_(message.wantsKeepalive_), + trailersAllowed_(message.trailersAllowed_), + secure_(message.secure_) { + if (message.trailers_) { + trailers_.reset(new HTTPHeaders(*message.trailers_.get())); + } +} + +HTTPMessage& HTTPMessage::operator=(const HTTPMessage& message) { + if (&message == this) { + return *this; + } + startTime_ = message.startTime_; + seqNo_ = message.seqNo_; + dstAddress_ = message.dstAddress_; + dstIP_ = message.dstIP_; + dstPort_ = message.dstPort_; + localIP_ = message.localIP_; + versionStr_ = message.versionStr_; + fields_ = message.fields_; + cookies_ = message.cookies_; + queryParams_ = message.queryParams_; + version_ = message.version_; + headers_ = message.headers_; + strippedPerHopHeaders_ = message.headers_; + sslVersion_ = message.sslVersion_; + sslCipher_ = message.sslCipher_; + spdy_ = message.spdy_; + parsedCookies_ = message.parsedCookies_; + parsedQueryParams_ = message.parsedQueryParams_; + chunked_ = message.chunked_; + upgraded_ = message.upgraded_; + wantsKeepalive_ = message.wantsKeepalive_; + trailersAllowed_ = message.trailersAllowed_; + secure_ = message.secure_; + + if (message.trailers_) { + trailers_.reset(new HTTPHeaders(*message.trailers_.get())); + } else { + trailers_.reset(); + } + return *this; +} + +void HTTPMessage::setMethod(HTTPMethod method) { + Request& req = request(); + req.method_ = method; +} + +void HTTPMessage::setMethod(folly::StringPiece method) { + VLOG(9) << "setMethod: " << method; + Request& req = request(); + boost::optional result = stringToMethod(method); + if (result) { + req.method_ = *result; + } else { + req.method_ = method.str(); + auto& storedMethod = boost::get(req.method_); + std::transform(storedMethod.begin(), storedMethod.end(), + storedMethod.begin(), ::toupper); + } +} + +boost::optional HTTPMessage::getMethod() const { + const auto& req = request(); + if (req.method_.which() == 2) { + return boost::get(req.method_); + } + return boost::none; +} + +/** + * @Returns a string representation of the request method (fpreq) + */ +const std::string& HTTPMessage::getMethodString() const { + const auto& req = request(); + if (req.method_.which() == 1) { + return boost::get(req.method_); + } else if (req.method_.which() == 2) { + return methodToString(boost::get(req.method_)); + } + return empty_string; +} + +void HTTPMessage::setHTTPVersion(uint8_t maj, uint8_t min) { + version_.first = maj; + version_.second = min; + versionStr_ = folly::to(maj, ".", min); +} + +const pair& HTTPMessage::getHTTPVersion() const { + return version_; +} + +int HTTPMessage::processMaxForwards() { + if (getMethod() == HTTPMethod::TRACE || getMethod() == HTTPMethod::OPTIONS) { + const string& value = headers_.getSingleOrEmpty(HTTP_HEADER_MAX_FORWARDS); + if (value.length() > 0) { + int64_t max_forwards = 0; + try { + max_forwards = folly::to(value); + } catch (const std::range_error& ex) { + return 400; + } + + if (max_forwards < 0) { + return 400; + } else if (max_forwards == 0) { + return 501; + } else { + headers_.set(HTTP_HEADER_MAX_FORWARDS, + folly::to(max_forwards - 1)); + } + } + } + return 0; +} + +bool HTTPMessage::isHTTP1_0() const { + return version_ == kHTTPVersion10; +} + +bool HTTPMessage::isHTTP1_1() const { + return version_ == kHTTPVersion11; +} + +string HTTPMessage::formatDateHeader() { + const auto now = std::chrono::system_clock::to_time_t( + std::chrono::system_clock::now()); + + char buff[1024]; + tm timeTupple; + gmtime_r(&now, &timeTupple); + + strftime(buff, 1024, "%a, %d %b %Y %H:%M:%S %Z", &timeTupple); + return std::string(buff); +} + +void HTTPMessage::ensureHostHeader() { + if (!headers_.exists(HTTP_HEADER_HOST)) { + headers_.add(HTTP_HEADER_HOST, + getDstAddress().getFamily() == AF_INET6 + ? '[' + getDstIP() + ']' : getDstIP()); + } +} + +void HTTPMessage::setStatusCode(uint16_t status) { + response().status_ = status; + response().statusStr_ = folly::to(status); +} + +uint16_t HTTPMessage::getStatusCode() const { + return response().status_; +} + +void +HTTPMessage::constructDirectResponse(const pair& version, + const int statusCode, + const string& statusMsg, + int contentLength) { + setStatusCode(statusCode); + setStatusMessage(statusMsg); + constructDirectResponse(version, contentLength); +} + +void +HTTPMessage::constructDirectResponse(const pair& version, + int contentLength) { + setHTTPVersion(version.first, version.second); + + headers_.set(HTTP_HEADER_CONTENT_LENGTH, folly::to(contentLength)); + + if (!headers_.exists(HTTP_HEADER_CONTENT_TYPE)) { + headers_.add(HTTP_HEADER_CONTENT_TYPE, "text/plain"); + } + setIsChunked(false); + setIsUpgraded(false); +} + +void HTTPMessage::parseCookies() const { + DCHECK(!parsedCookies_); + parsedCookies_ = true; + + headers_.forEachValueOfHeader(HTTP_HEADER_COOKIE, + [&](const string& headerval) { + splitNameValuePieces(headerval, ';', '=', + [this](StringPiece cookieName, StringPiece cookieValue) { + cookies_.emplace(cookieName, cookieValue); + }); + + return false; // continue processing "cookie" headers + }); +} + +void HTTPMessage::unparseCookies() { + cookies_.clear(); + parsedCookies_ = false; +} + +const StringPiece HTTPMessage::getCookie(const string& name) const { + // Parse the cookies if we haven't done so yet + if (!parsedCookies_) { + parseCookies(); + } + + auto it = cookies_.find(name); + if (it == cookies_.end()) { + return StringPiece(); + } else { + return it->second; + } +} + +void HTTPMessage::parseQueryParams() const { + DCHECK(!parsedQueryParams_); + const Request& req = request(); + + parsedQueryParams_ = true; + if (req.query_.empty()) { + return; + } + + splitNameValue(req.query_, '&', '=', + [this] (string&& paramName, string&& paramValue) { + + auto it = queryParams_.find(paramName); + if (it == queryParams_.end()) { + queryParams_.emplace(std::move(paramName), std::move(paramValue)); + } else { + // We have some unit tests that make sure we always return the last + // value when there are duplicate parameters. I don't think this really + // matters, but for now we might as well maintain the same behavior. + it->second = std::move(paramValue); + } + }); +} + +void HTTPMessage::unparseQueryParams() { + queryParams_.clear(); + parsedQueryParams_ = false; +} + +const string* HTTPMessage::getQueryParamPtr(const string& name) const { + // Parse the query parameters if we haven't done so yet + if (!parsedQueryParams_) { + parseQueryParams(); + } + + auto it = queryParams_.find(name); + if (it == queryParams_.end()) { + return nullptr; + } + return &it->second; +} + +bool HTTPMessage::hasQueryParam(const string& name) const { + return getQueryParamPtr(name) != nullptr; +} + +const string& HTTPMessage::getQueryParam(const string& name) const { + const string* ret = getQueryParamPtr(name); + return ret ? *ret : empty_string; +} + +int HTTPMessage::getIntQueryParam(const std::string& name) const { + return folly::to(getQueryParam(name)); +} + +int HTTPMessage::getIntQueryParam(const std::string& name, int defval) const { + try { + return getIntQueryParam(name); + } catch (const std::exception& ex) { + } + + return defval; +} + +std::string HTTPMessage::getDecodedQueryParam(const std::string& name) const { + auto val = getQueryParam(name); + + std::string result; + try { + folly::uriUnescape(val, result, folly::UriEscapeMode::QUERY); + } catch (const std::exception& ex) { + LOG(WARNING) << "Invalid escaped query param: " << folly::exceptionStr(ex); + } + return result; +} + +const std::map& HTTPMessage::getQueryParams() const { + // Parse the query parameters if we haven't done so yet + if (!parsedQueryParams_) { + parseQueryParams(); + } + return queryParams_; +} + +bool HTTPMessage::setQueryString(const std::string& query) { + ParseURL u(request().url_); + + if (u.valid()) { + // Recreate the URL by just changing the query string + request().url_ = createUrl(u.scheme(), + u.authority(), + u.path(), + query, // new query string + u.fragment()); + request().query_ = query; + return true; + } + + VLOG(4) << "Error parsing URL during setQueryString: " << request().url_; + return false; +} + +bool HTTPMessage::removeQueryParam(const std::string& name) { + // Parse the query parameters if we haven't done so yet + if (!parsedQueryParams_) { + parseQueryParams(); + } + + if (!queryParams_.erase(name)) { + // Query param was not found. + return false; + } + + auto query = createQueryString(queryParams_, request().query_.length()); + return setQueryString(query); +} + +bool HTTPMessage::setQueryParam(const std::string& name, + const std::string& value) { + // Parse the query parameters if we haven't done so yet + if (!parsedQueryParams_) { + parseQueryParams(); + } + + queryParams_[name] = value; + auto query = createQueryString(queryParams_, request().query_.length()); + return setQueryString(query); +} + +std::string HTTPMessage::createQueryString( + const std::map& params, uint32_t maxLength) { + std::string query; + query.reserve(maxLength); + for (auto it = params.begin(); it != params.end(); it++) { + if (it != params.begin()) { + query.append("&"); + } + query.append(it->first + "=" + it->second); + } + query.shrink_to_fit(); + return query; +} + +std::string HTTPMessage::createUrl(const folly::StringPiece scheme, + const folly::StringPiece authority, + const folly::StringPiece path, + const folly::StringPiece query, + const folly::StringPiece fragment) { + std::string url; + url.reserve(scheme.size() + authority.size() + path.size() + query.size() + + fragment.size() + 5); // 5 chars for ://,? and # + if (!scheme.empty()) { + folly::toAppend(scheme.str(), "://", &url); + } + folly::toAppend(authority, path, &url); + if (!query.empty()) { + folly::toAppend('?', query, &url); + } + if (!fragment.empty()) { + folly::toAppend('#', fragment, &url); + } + url.shrink_to_fit(); + return url; +} + +void HTTPMessage::splitNameValuePieces( + const string& input, + char pairDelim, + char valueDelim, + std::function callback) { + + StringPiece sp(input); + while (!sp.empty()) { + size_t pairDelimPos = sp.find(pairDelim); + StringPiece keyValue; + + if (pairDelimPos == string::npos) { + keyValue = sp; + sp.advance(sp.size()); + } else { + keyValue = sp.subpiece(0, pairDelimPos); + // Skip '&' char + sp.advance(pairDelimPos + 1); + } + + if (keyValue.empty()) { + continue; + } + + size_t valueDelimPos = keyValue.find(valueDelim); + if (valueDelimPos == string::npos) { + // Key only query param + callback(trim(keyValue), StringPiece()); + } else { + auto name = keyValue.subpiece(0, valueDelimPos); + auto value = keyValue.subpiece(valueDelimPos + 1); + callback(trim(name), trim(value)); + } + } +} + +StringPiece HTTPMessage::trim(StringPiece sp) { + // TODO: use a library function from boost? + for (; !sp.empty() && sp.front() == ' '; sp.pop_front()) { + } + for (; !sp.empty() && sp.back() == ' '; sp.pop_back()) { + } + return sp; +} + +void HTTPMessage::splitNameValue( + const string& input, + char pairDelim, + char valueDelim, + std::function callback) { + + folly::StringPiece sp(input); + while (!sp.empty()) { + size_t pairDelimPos = sp.find(pairDelim); + folly::StringPiece keyValue; + + if (pairDelimPos == string::npos) { + keyValue = sp; + sp.advance(sp.size()); + } else { + keyValue = sp.subpiece(0, pairDelimPos); + // Skip '&' char + sp.advance(pairDelimPos + 1); + } + + if (keyValue.empty()) { + continue; + } + + size_t valueDelimPos = keyValue.find(valueDelim); + if (valueDelimPos == string::npos) { + // Key only query param + string name = keyValue.str(); + string value; + + boost::trim(name, defaultLocale); + callback(std::move(name), std::move(value)); + } else { + string name = keyValue.subpiece(0, valueDelimPos).str(); + string value = keyValue.subpiece(valueDelimPos + 1).str(); + + boost::trim(name, defaultLocale); + boost::trim(value, defaultLocale); + callback(std::move(name), std::move(value)); + } + } +} + +void HTTPMessage::dumpMessage(int vlogLevel) const { + VLOG(vlogLevel) << "Version: " << versionStr_ + << ", chunked: " << chunked_ + << ", upgraded: " << upgraded_; + + // Common fields to both requests and responses. + std::vector> fields {{ + {"local_ip", &localIP_}, + {"version", &versionStr_}, + {"dst_ip", &dstIP_}, + {"dst_port", &dstPort_}, + }}; + + if (fields_.type() == typeid(Request)) { + // Request fields. + const Request& req = request(); + fields.push_back(make_pair("client_ip", &req.clientIP_)); + fields.push_back(make_pair("client_port", &req.clientPort_)); + fields.push_back(make_pair("method", &getMethodString())); + fields.push_back(make_pair("path", &req.path_)); + fields.push_back(make_pair("query", &req.query_)); + fields.push_back(make_pair("url", &req.url_)); + } else if (fields_.type() == typeid(Response)) { + // Response fields. + const Response& resp = response(); + fields.push_back(make_pair("status", &resp.statusStr_)); + fields.push_back(make_pair("status_msg", &resp.statusMsg_)); + } + + VLOG(vlogLevel) << "Fields for message: "; + for (auto field : fields) { + if (!field.second->empty()) { + VLOG(vlogLevel) << " " << field.first + << ":" << stripCntrlChars(*field.second); + } + } + + VLOG(vlogLevel) << "Headers for message: "; + headers_.forEach([&] (const string& h, const string& v) { + VLOG(vlogLevel) << " " << stripCntrlChars(h) << ": " << stripCntrlChars(v); + }); +} + +void +HTTPMessage::atomicDumpMessage(int vlogLevel) const { + std::lock_guard g(mutexDump_); + dumpMessage(vlogLevel); +} + +void HTTPMessage::dumpMessageToSink(google::LogSink* logSink) const { + LOG_TO_SINK(logSink, INFO) << "Version: " << versionStr_ + << ", chunked: " << chunked_ + << ", upgraded: " << upgraded_; + + // Common fields to both requests and responses. + std::vector> fields {{ + {"local_ip", &localIP_}, + {"version", &versionStr_}, + {"dst_ip", &dstIP_}, + {"dst_port", &dstPort_}, + }}; + + if (fields_.type() == typeid(Request)) { + // Request fields. + const Request& req = request(); + fields.push_back(make_pair("client_ip", &req.clientIP_)); + fields.push_back(make_pair("client_port", &req.clientPort_)); + fields.push_back(make_pair("method", &getMethodString())); + fields.push_back(make_pair("path", &req.path_)); + fields.push_back(make_pair("query", &req.query_)); + fields.push_back(make_pair("url", &req.url_)); + } else if (fields_.type() == typeid(Response)) { + // Response fields. + const Response& resp = response(); + fields.push_back(make_pair("status", &resp.statusStr_)); + fields.push_back(make_pair("status_msg", &resp.statusMsg_)); + } + + LOG_TO_SINK(logSink, INFO) << "Fields for message: "; + for (auto field : fields) { + if (!field.second->empty()) { + LOG_TO_SINK(logSink, INFO) << " " << field.first + << ":" << folly::backslashify(*field.second); + } + } + + LOG_TO_SINK(logSink, INFO) << "Headers for message: "; + headers_.forEach([&logSink] (const string& h, const string& v) { + LOG_TO_SINK(logSink, INFO) << " " << folly::backslashify(h) + << ": " << folly::backslashify(v); + }); +} + +bool HTTPMessage::computeKeepalive() const { + if (version_.first < 1) { + return false; + } + + // RFC 2616 isn't explicitly clear about whether "close" is case-sensitive. + // Section 2.1 states that literal tokens in the BNF are case-insensitive + // unless stated otherwise. The "close" token isn't explicitly mentioned + // in the BNF, but other header fields such as the character set and + // content coding are explicitly called out as being case insensitive. + // + // We'll treat the "close" token case-insensitively. This is the most + // conservative approach, since disabling keepalive when it was requested + // is better than enabling keepalive for a client that didn't expect it. + // + // Note that we only perform ASCII lowering here. This is good enough, + // since the token we are looking for is ASCII. + if (checkForHeaderToken(HTTP_HEADER_CONNECTION, "close", false)) { + // The Connection header contained a "close" token, so keepalive + // is disabled. + return false; + } + + if (version_ == kHTTPVersion10) { + // HTTP 1.0 persistent connections require a Connection: Keep-Alive + // header to be present for the connection to be persistent. + if (checkForHeaderToken(HTTP_HEADER_CONNECTION, "keep-alive", false)) { + return true; + } + return false; + } + + // It's a keepalive connection. + return true; +} + +bool HTTPMessage::checkForHeaderToken(const HTTPHeaderCode headerCode, + char const* token, + bool caseSensitive) const { + StringPiece tokenPiece(token); + // Search through all of the headers with this name. + // forEachValueOfHeader will return true iff it was "broken" prematurely + // with "return true" in the lambda-function + return headers_.forEachValueOfHeader(headerCode, [&] (const string& value) { + string lower; + // Use StringPiece, since it implements a faster find() than std::string + StringPiece headerValue; + if (caseSensitive) { + headerValue.reset(value); + } else { + // TODO: We only perform ASCII lowering right now. Technically the + // headers could contain data in other encodings, if encoded according + // to RFC 2047 (encoded strings will start with "=?"). + lower = value; + boost::to_lower(lower, defaultLocale); + headerValue.reset(lower); + } + + // Look for the specified token + size_t idx = 0; + size_t end = headerValue.size(); + while (idx < end) { + idx = headerValue.find(tokenPiece, idx); + if (idx == string::npos) { + break; + } + + // Search backwards to make sure we found the value at the beginning + // of a token. + bool at_token_start = false; + size_t prev = idx; + while (true) { + if (prev == 0) { + at_token_start = true; + break; + } + --prev; + char c = headerValue[prev]; + if (c == ',') { + at_token_start = true; + break; + } + if (!isLWS(c)) { + // not at a token start + break; + } + } + if (!at_token_start) { + idx += 1; + continue; + } + + // Search forwards to see if we found the value at the end of a token + bool at_token_end = false; + size_t next = idx + tokenPiece.size(); + while (true) { + if (next >= end) { + at_token_end = true; + break; + } + char c = headerValue[next]; + if (c == ',') { + at_token_end = true; + break; + } + if (!isLWS(c)) { + // not at a token end + break; + } + ++next; + } + if (at_token_end) { + // We found the token we're looking for + return true; + } + + idx += 1; + } + return false; // keep processing + }); +} + +const char* HTTPMessage::getDefaultReason(uint16_t status) { + switch (status) { + case 100: return "Continue"; + case 101: return "Switching Protocols"; + case 200: return "OK"; + case 201: return "Created"; + case 202: return "Accepted"; + case 203: return "Non-Authoritative Information"; + case 204: return "No Content"; + case 205: return "Reset Content"; + case 206: return "Partial Content"; + case 300: return "Multiple Choices"; + case 301: return "Moved Permanently"; + case 302: return "Found"; + case 303: return "See Other"; + case 304: return "Not Modified"; + case 305: return "Use Proxy"; + case 307: return "Temporary Redirect"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 402: return "Payment Required"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 406: return "Not Acceptable"; + case 407: return "Proxy Authentication Required"; + case 408: return "Request Timeout"; + case 409: return "Conflict"; + case 410: return "Gone"; + case 411: return "Length Required"; + case 412: return "Precondition Failed"; + case 413: return "Request Entity Too Large"; + case 414: return "Request-URI Too Long"; + case 415: return "Unsupported Media Type"; + case 416: return "Requested Range Not Satisfiable"; + case 417: return "Expectation Failed"; + case 418: return "I'm a teapot"; + case 500: return "Internal Server Error"; + case 501: return "Not Implemented"; + case 502: return "Bad Gateway"; + case 503: return "Service Unavailable"; + case 504: return "Gateway Timeout"; + case 505: return "HTTP Version Not Supported"; + } + + // Note: Some Microsoft clients behave badly if the reason string + // is left empty. Therefore return a non-empty string here. + return "-"; +} + +} // proxygen diff --git a/proxygen/lib/http/HTTPMessage.h b/proxygen/lib/http/HTTPMessage.h new file mode 100644 index 0000000000..587fe63436 --- /dev/null +++ b/proxygen/lib/http/HTTPMessage.h @@ -0,0 +1,724 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/HTTPHeaders.h" +#include "proxygen/lib/http/HTTPMethod.h" +#include "proxygen/lib/utils/ParseURL.h" +#include "proxygen/lib/utils/Time.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace proxygen { + +/** + * An HTTP request or response minus the body. + * + * Some of the methods on this class will assert if called from the wrong + * context since they only make sense for a request or response. Make sure + * you know what type of HTTPMessage this is before calling such methods. + * + * All header names stored in this class are case-insensitive. + */ +class HTTPMessage { + public: + + HTTPMessage(); + ~HTTPMessage(); + HTTPMessage(const HTTPMessage& message); + HTTPMessage& operator=(const HTTPMessage& message); + + /** + * Is this a chunked message? (fpreq, fpresp) + */ + void setIsChunked(bool chunked) { chunked_ = chunked; } + bool getIsChunked() const { return chunked_; } + + /** + * Is this an upgraded message? (fpreq, fpresp) + */ + void setIsUpgraded(bool upgraded) { upgraded_ = upgraded; } + bool getIsUpgraded() const { return upgraded_; } + + /** + * Set/Get client address + */ + void setClientAddress(const folly::SocketAddress& addr) { + request().clientAddress_ = addr; + request().clientIP_ = addr.getAddressStr(); + request().clientPort_ = folly::to(addr.getPort()); + } + + const folly::SocketAddress& getClientAddress() const { + return request().clientAddress_; + } + + const std::string& getClientIP() const { + return request().clientIP_; + } + + const std::string& getClientPort() const { + return request().clientPort_; + } + + /** + * Set/Get destination (vip) address + */ + void setDstAddress(const folly::SocketAddress& addr) { + dstAddress_ = addr; + dstIP_ = addr.getAddressStr(); + dstPort_ = folly::to(addr.getPort()); + } + + const folly::SocketAddress& getDstAddress() const { + return dstAddress_; + } + + const std::string& getDstIP() const { + return dstIP_; + } + + const std::string& getDstPort() const { + return dstPort_; + } + + /** + * Set/Get the local IP address + */ + template // T = string + void setLocalIp(T&& ip) { + localIP_ = std::forward(ip); + } + const std::string& getLocalIp() const { + return localIP_; + } + + /** + * Access the method (fpreq) + */ + void setMethod(HTTPMethod method); + void setMethod(folly::StringPiece method); + void rawSetMethod(const std::string& method) { + setMethod(method); + } + + /** + * @Returns an HTTPMethod enum value representing the method if it is a + * standard request method, or else "none" if it is an extension method + * (fpreq) + */ + boost::optional getMethod() const; + + /** + * @Returns a string representation of the request method (fpreq) + */ + const std::string& getMethodString() const; + + /** + * Access the URL component (fpreq) + * + * The component from the initial "METHOD HTTP/..." line. When + * valid, this is a full URL, not just a path. + */ + template // T = string + ParseURL setURL(T&& url) { + VLOG(9) << "setURL: " << url; + + // Set the URL, path, and query string parameters + ParseURL u(url); + if (u.valid()) { + VLOG(9) << "set path: " << u.path() << " query:" << u.query(); + request().path_ = u.path().str(); + request().query_ = u.query().str(); + unparseQueryParams(); + } else { + VLOG(4) << "Error in parsing URL: " << url; + } + + request().url_ = std::forward(url); + return u; + } + // The template function above doesn't work with char*, + // so explicitly convert to a string first. + void setURL(const char* url) { + setURL(std::string(url)); + } + const std::string& getURL() const { + return request().url_; + } + void rawSetURL(const std::string& url) { + setURL(url); + } + + /** + * Access the path component (fpreq) + */ + const std::string& getPath() const { + return request().path_; + } + + /** + * Access the query component (fpreq) + */ + const std::string& getQueryString() const { + return request().query_; + } + + /** + * Version constants + */ + static const std::pair kHTTPVersion10; + static const std::pair kHTTPVersion11; + + /** + * Access the HTTP version number (fpreq, fpres) + */ + void setHTTPVersion(uint8_t major, uint8_t minor); + const std::pair& getHTTPVersion() const; + + /** + * Access the HTTP status message string (res) + */ + template // T = string + void setStatusMessage(T&& msg) { + response().statusMsg_ = std::forward(msg); + } + const std::string& getStatusMessage() const { + return response().statusMsg_; + } + void rawSetStatusMessage(std::string msg) { + setStatusMessage(msg); + } + + /** + * Get/Set the HTTP version string (like "1.1"). + * XXX: Note we only support X.Y format while setting version. + */ + const std::string& getVersionString() const { + return versionStr_; + } + void setVersionString(const std::string& ver) { + if (ver.size() != 3 || + ver[1] != '.' || + !isdigit(ver[0]) || + !isdigit(ver[2])) { + return; + } + + setHTTPVersion(ver[0] - '0', ver[2] - '0'); + } + + /** + * Access the headers (fpreq, fpres) + */ + HTTPHeaders& getHeaders() { return headers_; } + const HTTPHeaders& getHeaders() const { return headers_; } + + /** + * Access the trailers + */ + HTTPHeaders* getTrailers() { return trailers_.get(); } + const HTTPHeaders* getTrailers() const { return trailers_.get(); } + + /** + * Set the trailers, replacing any that might already be present + */ + void setTrailers(std::unique_ptr&& trailers) { + trailers_ = std::move(trailers); + } + + /** + * Decrements Max-Forwards header, when present on OPTIONS or TRACE methods. + * + * Returns HTTP status code. + */ + int processMaxForwards(); + + /** + * Returns true if the version of this message is HTTP/1.0 + */ + bool isHTTP1_0() const; + + /** + * Returns true if the version of this message is HTTP/1.1 + */ + bool isHTTP1_1() const; + + /** + * Returns true if this is a 1xx response. + */ + bool is1xxResponse() const { return (getStatusCode() / 100) == 1; } + + /** + * Formats the current time appropriately for a Date header + */ + static std::string formatDateHeader(); + + /** + * Ensures this HTTPMessage contains a host header, adding a default one + * with the destination address if necessary. + */ + void ensureHostHeader(); + + /** + * Indicates if this request wants the connection to be kept-alive + * (default true). Not all codecs respect this option. + */ + void setWantsKeepalive(bool wantsKeepaliveVal) { + wantsKeepalive_ = wantsKeepaliveVal; + } + bool wantsKeepalive() const { + return wantsKeepalive_; + } + + /** + * Returns true if trailers are allowed on this message. Trailers + * are not allowed on responses unless the client is expecting them. + */ + bool trailersAllowed() const { return trailersAllowed_; } + /** + * Accessor to set whether trailers are allowed in the response + */ + void setTrailersAllowed(bool trailersAllowedVal) { + trailersAllowed_ = trailersAllowedVal; + } + + /** + * Returns true if this message has trailers that need to be serialized + */ + bool hasTrailers() const { + return trailersAllowed_ && trailers_ && trailers_->size() > 0; + } + + /** + * Access the status code (fpres) + */ + void setStatusCode(uint16_t status); + uint16_t getStatusCode() const; + + /** + * Fill in the fields for a response message header that the server will + * send directly to the client. + * + * @param version HTTP version (major, minor) + * @param statusCode HTTP status code to respond with + * @param msg textual message to embed in "message" status field + * @param contentLength the length of the data to be written out through + * this message + */ + void constructDirectResponse(const std::pair& version, + const int statusCode, + const std::string& statusMsg, + int contentLength = 0); + + /** + * Fill in the fields for a response message header that the server will + * send directly to the client. This function assumes the status code and + * status message have already been set on this HTTPMessage object + * + * @param version HTTP version (major, minor) + * @param contentLength the length of the data to be written out through + * this message + */ + void constructDirectResponse(const std::pair& version, + int contentLength = 0); + + /** + * Check if query parameter with the specified name exists. + */ + bool hasQueryParam(const std::string& name) const; + + /** + * Get the query parameter with the specified name. + * + * Returns a pointer to the query parameter value, or nullptr if there is no + * parameter with the specified name. The returned value is only valid as + * long as this HTTPMessage object. + */ + const std::string* getQueryParamPtr(const std::string& name) const; + + /** + * Get the query parameter with the specified name. + * + * Returns a reference to the query parameter value, or + * proxygen::empty_string if there is no parameter with the + * specified name. The returned value is only valid as long as this + * HTTPMessage object. + */ + const std::string& getQueryParam(const std::string& name) const; + + /** + * Get the query parameter with the specified name as int. + * + * If the conversion fails, throws exception. + */ + int getIntQueryParam(const std::string& name) const; + + /** + * Get the query parameter with the specified name as int. + * + * Returns the query parameter if it can be parsed as int otherwise the + * default value. + */ + int getIntQueryParam(const std::string& name, int defval) const; + + /** + * Get the query parameter with the specified name after percent decoding. + * + * Returns empty string if parameter is missing or folly::uriUnescape + * query param + */ + std::string getDecodedQueryParam(const std::string& name) const; + + /** + * Get all the query parameters. + * + * Returns a reference to the query parameters map. The returned + * value is only valid as long as this + * HTTPMessage object. + */ + const std::map& getQueryParams() const; + + /** + * Set the query string to the specified value, and recreate the url_. + * + * Returns true if the query string was changed successfully. + */ + bool setQueryString(const std::string& query); + + /** + * Remove the query parameter with the specified name. + * + * Returns true if the query parameter was present and deleted. + */ + bool removeQueryParam(const std::string& name); + + /** + * Sets the query parameter with the specified name to the specified value. + * + * Returns true if the query parameter was successfully set. + */ + bool setQueryParam(const std::string& name, const std::string& value); + + /** + * Get the cookie with the specified name. + * + * Returns a StringPiece to the cookie value, or an empty StringPiece if + * there is no cookie with the specified name. The returned cookie is + * only valid as long as the Cookie Header in HTTPMessage object exists. + * Applications should make sure they call unparseCookies() when editing + * the Cookie Header, so that the StringPiece references are cleared. + */ + const folly::StringPiece getCookie(const std::string& name) const; + + /** + * Print the message out. + */ + void dumpMessage(int verbosity) const; + + /** + * Print the message out, serializes through mutex + * Used in shutdown path + */ + void atomicDumpMessage(int verbosity) const; + + /** + * Print the message out to logSink. + */ + void dumpMessageToSink(google::LogSink* logSink) const; + + /** + * Interact with headers that are defined to be per-hop. + * + * It is expected that during request processing, stripPerHopHeaders() will + * be called before the message is proxied to the other connection. + */ + void stripPerHopHeaders(); + + const HTTPHeaders& getStrippedPerHopHeaders() const { + return strippedPerHopHeaders_; + } + + void setSecure(bool secure) { secure_ = secure; } + bool isSecure() const { return secure_; } + int getSecureVersion() const { return sslVersion_; } + const char* getSecureCipher() const { return sslCipher_; } + void setSecureInfo(int ver, const char* cipher) { + // cipher is a static const char* provided and managed by openssl lib + sslVersion_ = ver; sslCipher_ = cipher; + } + void setSPDY(uint16_t spdyVersion) { spdy_ = spdyVersion; } + bool isSPDY() const { return spdy_; } + uint16_t getSPDYVersion() const { return spdy_; } + + /* Setter and getter for the SPDY priority value. + * + * Negative values of pri are interpreted much like negative array + * indexes in python, so -1 will be the largest numerical priority + * value for this SPDY version (i.e. 3 for SPDY/2 or 7 for SPDY/3), + * -2 the second largest (i.e. 2 for SPDY/2 or 6 for SPDY/3). + */ + void setPriority(int8_t pri) { pri_ = pri; } + uint8_t getPriority() const { + int8_t pri = pri_; + + if (pri < 0) { + pri += (getSPDYVersion() > 2) ? 8 : 4; + } + + return pri; + } + + /** + * get and setter for transaction sequence number + */ + void setSeqNo(int32_t seqNo) { seqNo_ = seqNo; } + int32_t getSeqNo() const { return seqNo_; } + + /** + * getter and setter for size in serialized form + */ + void setIngressHeaderSize(const HTTPHeaderSize& size) { + size_ = size; + } + const HTTPHeaderSize& getIngressHeaderSize() const { + return size_; + } + + /** + * Get the time when the first byte of the message arrived + */ + TimePoint getStartTime() const { return startTime_; } + + /** + * Check if a particular token value is present in a header that consists of + * a list of comma separated tokens. (e.g., a header with a #rule + * body as specified in the RFC 2616 BNF notation.) + */ + bool checkForHeaderToken(const HTTPHeaderCode headerCode, + char const* token, + bool caseSensitive) const; + + /** + * Forget about the parsed cookies. + * + * Ideally HTTPMessage should automatically forget about the current parsed + * cookie state whenever a Cookie header is changed. However, at the moment + * callers have to explicitly call unparseCookies() after modifying the + * cookie headers. + */ + void unparseCookies(); + + /** + * Get the default reason string for a status code. + * + * This returns the reason string for the specified status code as specified + * in RFC 2616. For unknown status codes, the string "-" is returned. + */ + static const char* getDefaultReason(uint16_t status); + + /** + * Computes whether the state of this message is compatible with keepalive. + * Changing the headers, version, etc can change the result. + */ + bool computeKeepalive() const; + + /** + * @returns true if this HTTPMessage represents an HTTP request + */ + bool isRequest() const { + return fields_.which() == 1; + } + + /** + * @returns true if this HTTPMessage represents an HTTP response + */ + bool isResponse() const { + return fields_.which() == 2; + } + + /** + * Assuming input contains + * (), + * invoke callback once with each name,value pair. + */ + static void splitNameValuePieces( + const std::string& input, + char pairDelim, + char valueDelim, + std::function callback); + + static void splitNameValue( + const std::string& input, + char pairDelim, + char valueDelim, + std::function callback); + + /** + * Form the URL from the individual components. + * url -> {scheme}://{authority}{path}?{query}#{fragment} + */ + static std::string createUrl(const folly::StringPiece scheme, + const folly::StringPiece authority, + const folly::StringPiece path, + const folly::StringPiece query, + const folly::StringPiece fragment); + + /** + * Create a query string from the query parameters map + * containing the name-value pairs. + */ + static std::string createQueryString( + const std::map& params, uint32_t maxSize); + + protected: + // Message start time, in msec since the epoch. + TimePoint startTime_; + int32_t seqNo_; + + private: + + void parseCookies() const; + + void parseQueryParams() const; + void unparseQueryParams(); + + /** + * Trims whitespace from the beggining and end of the StringPiece. + */ + static folly::StringPiece trim(folly::StringPiece sp); + + /** The 12 standard fields for HTTP messages. Use accessors. + * An HTTPMessage is either a Request or Response. + * Once an accessor for either is used, that fixes the type of HTTPMessage. + * If an access is then used for the other type, a DCHECK will fail. + */ + struct Request { + folly::SocketAddress clientAddress_; + std::string clientIP_; + std::string clientPort_; + mutable boost::variant method_; + std::string path_; + std::string query_; + std::string url_; + }; + + struct Response { + uint16_t status_; + std::string statusStr_; + std::string statusMsg_; + }; + + folly::SocketAddress dstAddress_; + std::string dstIP_; + std::string dstPort_; + + std::string localIP_; + std::string versionStr_; + + mutable boost::variant fields_; + + Request& request() { + DCHECK(fields_.which() == 0 || fields_.which() == 1) << fields_.which(); + if (fields_.which() == 0) { + fields_ = Request(); + } + + return boost::get(fields_); + } + + const Request& request() const { + DCHECK(fields_.which() == 0 || fields_.which() == 1) << fields_.which(); + if (fields_.which() == 0) { + fields_ = Request(); + } + + return boost::get(fields_); + } + + Response& response() { + DCHECK(fields_.which() == 0 || fields_.which() == 2) << fields_.which(); + if (fields_.which() == 0) { + fields_ = Response(); + } + + return boost::get(fields_); + } + + const Response& response() const { + DCHECK(fields_.which() == 0 || fields_.which() == 2) << fields_.which(); + if (fields_.which() == 0) { + fields_ = Response(); + } + + return boost::get(fields_); + } + + /* + * Cookies and query parameters + * These are mutable since we parse them lazily in getCookie() and + * getQueryParam() + */ + mutable std::map cookies_; + // TODO: use StringPiece for queryParams_ and delete splitNameValue() + mutable std::map queryParams_; + + std::pair version_; + HTTPHeaders headers_; + HTTPHeaders strippedPerHopHeaders_; + HTTPHeaderSize size_; + std::unique_ptr trailers_; + + int sslVersion_; + const char* sslCipher_; + uint16_t spdy_; // == 0 - no SPDY; > 0 - SPDY Version + uint8_t pri_; + + mutable bool parsedCookies_:1; + mutable bool parsedQueryParams_:1; + bool chunked_:1; + bool upgraded_:1; + bool wantsKeepalive_:1; + bool trailersAllowed_:1; + + // Whether the message is received in HTTPS. + bool secure_:1; + + // used by atomicDumpMessage + static std::mutex mutexDump_; +}; + +/** + * Returns a std::string that has the control characters removed from the + * input string. + */ +template +std::string stripCntrlChars(const Str& str) { + std::string res; + res.reserve(str.length()); + for (size_t i = 0; i < str.size(); ++i) { + if (!(str[i] <= 0x1F || str[i] == 0x7F)) { + res += str[i]; + } + } + return res; +} + +} // proxygen diff --git a/proxygen/lib/http/HTTPMessageFilters.h b/proxygen/lib/http/HTTPMessageFilters.h new file mode 100644 index 0000000000..48bebb411f --- /dev/null +++ b/proxygen/lib/http/HTTPMessageFilters.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPTransaction.h" +#include "proxygen/lib/utils/DestructorCheck.h" + +#include + +namespace proxygen { + +class HTTPMessageFilter: public HTTPTransaction::Handler, + public DestructorCheck { + public: + void setNextTransactionHandler(HTTPTransaction::Handler* next) { + CHECK(next); + nextTransactionHandler_ = next; + } + HTTPTransaction::Handler* getNextTransactionHandler() { + return nextTransactionHandler_; + } + + // These HTTPTransaction::Handler callbacks may be overwritten + // The default behavior is to pass the call through. + void onHeadersComplete(std::unique_ptr msg) noexcept override { + nextTransactionHandler_->onHeadersComplete(std::move(msg)); + } + void onBody(std::unique_ptr chain) noexcept override { + nextTransactionHandler_->onBody(std::move(chain)); + } + void onChunkHeader(size_t length) noexcept override { + nextTransactionHandler_->onChunkHeader(length); + } + void onChunkComplete() noexcept override { + nextTransactionHandler_->onChunkComplete(); + } + void onTrailers(std::unique_ptr trailers) noexcept override { + nextTransactionHandler_->onTrailers(std::move(trailers)); + } + void onEOM() noexcept override { + nextTransactionHandler_->onEOM(); + } + void onUpgrade(UpgradeProtocol protocol) noexcept override { + nextTransactionHandler_->onUpgrade(protocol); + } + void onError(const HTTPException& error) noexcept override { + nextTransactionHandler_->onError(error); + } + + // These HTTPTransaction::Handler callbacks cannot be overrwritten + void setTransaction(HTTPTransaction* txn) noexcept override final { + nextTransactionHandler_->setTransaction(txn); + } + void detachTransaction() noexcept override final { + nextTransactionHandler_->detachTransaction(); + } + void onEgressPaused() noexcept override final { + nextTransactionHandler_->onEgressPaused(); + } + void onEgressResumed() noexcept override final { + nextTransactionHandler_->onEgressResumed(); + } + void onPushedTransaction(HTTPTransaction* txn) noexcept override final { + nextTransactionHandler_->onPushedTransaction(txn); + } + protected: + void nextOnHeadersComplete(std::unique_ptr msg) { + nextTransactionHandler_->onHeadersComplete(std::move(msg)); + } + void nextOnBody(std::unique_ptr chain) { + nextTransactionHandler_->onBody(std::move(chain)); + } + void nextOnChunkHeader(size_t length) { + nextTransactionHandler_->onChunkHeader(length); + } + void nextOnChunkComplete() { + nextTransactionHandler_->onChunkComplete(); + } + void nextOnTrailers(std::unique_ptr trailers) { + nextTransactionHandler_->onTrailers(std::move(trailers)); + } + void nextOnEOM() { + nextTransactionHandler_->onEOM(); + } + void nextOnError(const HTTPException& ex) { + nextTransactionHandler_->onError(ex); + } + HTTPTransaction::Handler* nextTransactionHandler_{nullptr}; +}; + +} // proxygen diff --git a/proxygen/lib/http/HTTPMethod.cpp b/proxygen/lib/http/HTTPMethod.cpp new file mode 100644 index 0000000000..62cabf275c --- /dev/null +++ b/proxygen/lib/http/HTTPMethod.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPMethod.h" + +#include "proxygen/lib/http/HTTPHeaders.h" + +#include +#include + +#define HTTP_METHOD_STR(method) #method + +namespace { +static const std::vector kMethodStrings = { + HTTP_METHOD_GEN(HTTP_METHOD_STR) +}; +} + +namespace proxygen { + +boost::optional stringToMethod(folly::StringPiece method) { + FOR_EACH_ENUMERATE(index, cur, kMethodStrings) { + if (caseInsensitiveEqual(*cur, method)) { + return HTTPMethod(index); + } + } + return boost::none; +} + +const std::string& methodToString(HTTPMethod method) { + return kMethodStrings[static_cast(method)]; +} + +std::ostream& operator <<(std::ostream& out, HTTPMethod method) { + out << methodToString(method); + return out; +} + +} diff --git a/proxygen/lib/http/HTTPMethod.h b/proxygen/lib/http/HTTPMethod.h new file mode 100644 index 0000000000..8745944a1c --- /dev/null +++ b/proxygen/lib/http/HTTPMethod.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +namespace proxygen { + +// Ordered by frequency to minimize time spent in iteration +#define HTTP_METHOD_GEN(x) \ + x(GET), \ + x(POST), \ + x(OPTIONS), \ + x(DELETE), \ + x(HEAD), \ + x(CONNECT), \ + x(PUT), \ + x(TRACE) + +#define HTTP_METHOD_ENUM(method) method + +/** + * See the definitions in RFC2616 5.1.1 for the source of this + * list. Today, proxygen only understands the methods defined in 5.1.1 and + * is not aware of any extension methods. If you wish to support extension + * methods, you must handle those separately from this enum. + */ +enum class HTTPMethod { + HTTP_METHOD_GEN(HTTP_METHOD_ENUM) +}; + +#undef HTTP_METHOD_ENUM + +/** + * Returns the HTTPMethod that matches the method. Although RFC2616 5.1.1 + * says methods are case-sensitive, we ignore case here since most + * programmers probably really meant "GET" not "get". If the method is not + * recognized, the return value will be None + */ +extern boost::optional stringToMethod(folly::StringPiece method); + +/** + * Returns a string representation of the method. If EXTENSION_METHOD is + * passed, then an empty string is returned + */ +extern const std::string& methodToString(HTTPMethod method); + +std::ostream& operator<<(std::ostream& os, HTTPMethod method); + +} diff --git a/proxygen/lib/http/Makefile.am b/proxygen/lib/http/Makefile.am new file mode 100644 index 0000000000..8f6a31e9a1 --- /dev/null +++ b/proxygen/lib/http/Makefile.am @@ -0,0 +1,133 @@ +SUBDIRS = . codec session test + +BUILT_SOURCES = HTTPCommonHeaders.h HTTPCommonHeaders.cpp + +CommonHeadersGen: HTTPCommonHeaders.template.h HTTPCommonHeaders.template.gperf HTTPCommonHeaders.txt + INSTALLED_GPERF=1 FBCODE_DIR=$(top_srcdir)/.. INSTALL_DIR=$(srcdir) ruby generate_http_gperfs.rb + touch CommonHeadersGen + +HTTPCommonHeaders.h: CommonHeadersGen + +HTTPCommonHeaders.cpp: CommonHeadersGen + +noinst_LTLIBRARIES = libproxygenhttp.la + +libproxygenhttpdir = $(includedir)/proxygen/lib/http +nobase_libproxygenhttp_HEADERS = \ + HTTPCommonHeaders.h \ + HTTPConnector.h \ + HTTPConstants.h \ + HTTPException.h \ + HTTPHeaderSize.h \ + HTTPHeaders.h \ + HTTPMessage.h \ + HTTPMessageFilters.h \ + HTTPMethod.h \ + ProxygenErrorEnum.h \ + RFC2616.h \ + Window.h \ + codec/CodecDictionaries.h \ + codec/CodecProtocol.h \ + codec/ErrorCode.h \ + codec/FlowControlFilter.h \ + codec/HTTP1xCodec.h \ + codec/HTTPChecks.h \ + codec/HTTPCodec.h \ + codec/HTTPCodecFilter.h \ + codec/HTTPSettings.h \ + codec/SPDYCodec.h \ + codec/SPDYConstants.h \ + codec/SPDYUtil.h \ + codec/SPDYVersion.h \ + codec/SPDYVersionSettings.h \ + codec/SettingsId.h \ + codec/TransportDirection.h \ + codec/compress/GzipHeaderCodec.h \ + codec/compress/HPACKCodec.h \ + codec/compress/HPACKConstants.h \ + codec/compress/HPACKContext.h \ + codec/compress/HPACKDecodeBuffer.h \ + codec/compress/HPACKDecoder.h \ + codec/compress/HPACKEncodeBuffer.h \ + codec/compress/HPACKEncoder.h \ + codec/compress/HPACKHeader.h \ + codec/compress/Header.h \ + codec/compress/HeaderCodec.h \ + codec/compress/HeaderPiece.h \ + codec/compress/HeaderTable.h \ + codec/compress/Huffman.h \ + codec/compress/Logging.h \ + codec/compress/StaticHeaderTable.h \ + session/ByteEventTracker.h \ + session/ByteEvents.h \ + session/CodecErrorResponseHandler.h \ + session/HTTPDirectResponseHandler.h \ + session/HTTPDownstreamSession.h \ + session/HTTPErrorPage.h \ + session/HTTPEvent.h \ + session/HTTPSession.h \ + session/HTTPSessionAcceptor.h \ + session/HTTPSessionController.h \ + session/HTTPSessionStats.h \ + session/HTTPTransaction.h \ + session/HTTPTransactionEgressSM.h \ + session/HTTPTransactionIngressSM.h \ + session/HTTPUpstreamSession.h \ + session/SimpleController.h \ + session/TransportFilter.h + +libproxygenhttp_la_SOURCES = \ + HTTPCommonHeaders.cpp \ + codec/CodecProtocol.cpp \ + codec/compress/GzipHeaderCodec.cpp \ + codec/compress/HeaderTable.cpp \ + codec/compress/HPACKCodec.cpp \ + codec/compress/HPACKContext.cpp \ + codec/compress/HPACKDecodeBuffer.cpp \ + codec/compress/HPACKDecoder.cpp \ + codec/compress/HPACKEncodeBuffer.cpp \ + codec/compress/HPACKEncoder.cpp \ + codec/compress/HPACKHeader.cpp \ + codec/compress/Huffman.cpp \ + codec/compress/Logging.cpp \ + codec/compress/StaticHeaderTable.cpp \ + codec/ErrorCode.cpp \ + codec/FlowControlFilter.cpp \ + codec/HTTP1xCodec.cpp \ + codec/HTTPChecks.cpp \ + codec/HTTPCodecFilter.cpp \ + codec/HTTPSettings.cpp \ + codec/SPDYCodec.cpp \ + codec/SPDYConstants.cpp \ + codec/SPDYUtil.cpp \ + codec/SettingsId.cpp \ + codec/TransportDirection.cpp \ + HTTPConnector.cpp \ + HTTPConstants.cpp \ + HTTPException.cpp \ + HTTPHeaders.cpp \ + HTTPMessage.cpp \ + HTTPMethod.cpp \ + ProxygenErrorEnum.cpp \ + RFC2616.cpp \ + session/ByteEvents.cpp \ + session/CodecErrorResponseHandler.cpp \ + session/HTTPDirectResponseHandler.cpp \ + session/HTTPDownstreamSession.cpp \ + session/HTTPErrorPage.cpp \ + session/HTTPEvent.cpp \ + session/HTTPSessionAcceptor.cpp \ + session/HTTPSession.cpp \ + session/HTTPTransaction.cpp \ + session/HTTPTransactionEgressSM.cpp \ + session/HTTPTransactionIngressSM.cpp \ + session/HTTPUpstreamSession.cpp \ + session/ByteEventTracker.cpp \ + session/SimpleController.cpp \ + session/TransportFilter.cpp \ + Window.cpp + +libproxygenhttp_la_LIBADD = \ + ../services/libproxygenservices.la \ + ../ssl/libproxygenssl.la \ + ../utils/libutils.la diff --git a/proxygen/lib/http/ProxygenErrorEnum.cpp b/proxygen/lib/http/ProxygenErrorEnum.cpp new file mode 100644 index 0000000000..05e46f37b3 --- /dev/null +++ b/proxygen/lib/http/ProxygenErrorEnum.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/ProxygenErrorEnum.h" + +#define PROXYGEN_ERROR_STR(error) #error + +namespace { + static const char* errorStrings[] = { + PROXYGEN_ERROR_GEN(PROXYGEN_ERROR_STR) + }; +} + +namespace proxygen { + +const char* getErrorString(ProxygenError error) { + if (error < kErrorNone || error >= kErrorMax) { + return errorStrings[kErrorMax]; + } else { + return errorStrings[error]; + } +} + +const char* getErrorStringByIndex(int i) { + return errorStrings[i]; +} + +} diff --git a/proxygen/lib/http/ProxygenErrorEnum.h b/proxygen/lib/http/ProxygenErrorEnum.h new file mode 100644 index 0000000000..5370793a9f --- /dev/null +++ b/proxygen/lib/http/ProxygenErrorEnum.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +// TODO #5101452 remove all references to "read" and "write" since that +// should be indicated by the direction field of HTTPException + +// Max must be the last one. +#define PROXYGEN_ERROR_GEN(x) \ + x(None), \ + x(Message), \ + x(Connect), \ + x(ConnectTimeout), \ + x(Read), \ + x(Write), \ + x(Timeout), \ + x(Handshake), \ + x(NoServer), \ + x(MaxRedirects), \ + x(InvalidRedirect), \ + x(ResponseAction), \ + x(MaxConnects), \ + x(Dropped), \ + x(Connection), \ + x(ConnectionReset), \ + x(ParseHeader), \ + x(ParseBody), \ + x(EOF), \ + x(ClientRenegotiation), \ + x(Unknown), \ + x(BadDecompress), \ + x(SSL), \ + x(StreamAbort), \ + x(StreamUnacknowledged), \ + x(WriteTimeout), \ + x(AddressPrivate), \ + x(AddressFamilyNotSupported), \ + x(DNSNoResults), \ + x(MalformedInput), \ + x(UnsupportedExpectation), \ + x(MethodNotSupported), \ + x(UnsupportedScheme), \ + x(Shutdown), \ + x(IngressStateTransition), \ + x(ClientSilent), \ + x(Max) + +// Increase this if you add more error types and Max exceeds 63 +#define PROXYGEN_ERROR_BITSIZE 6 + +#define PROXYGEN_ERROR_ENUM(error) kError##error + +enum ProxygenError { + PROXYGEN_ERROR_GEN(PROXYGEN_ERROR_ENUM) +}; + +#undef PROXYGEN_ERROR_ENUM + +extern const char* getErrorString(ProxygenError error); + +extern const char* getErrorStringByIndex(int i); + +} diff --git a/proxygen/lib/http/RFC2616.cpp b/proxygen/lib/http/RFC2616.cpp new file mode 100644 index 0000000000..edf9d4c6da --- /dev/null +++ b/proxygen/lib/http/RFC2616.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/RFC2616.h" + +#include "proxygen/lib/http/HTTPHeaders.h" + +#include +#include + +namespace proxygen { namespace RFC2616 { + +BodyAllowed isRequestBodyAllowed(boost::optional method) { + if (method == HTTPMethod::TRACE) { + return BodyAllowed::NOT_ALLOWED; + } + if (method == HTTPMethod::OPTIONS || method == HTTPMethod::POST || + method == HTTPMethod::PUT || method == HTTPMethod::CONNECT) { + return BodyAllowed::DEFINED; + } + return BodyAllowed::NOT_DEFINED; +} + +bool responseBodyMustBeEmpty(unsigned status) { + return (status == 304 || status == 204 || + (100 <= status && status < 200)); +} + +bool bodyImplied(const HTTPHeaders& headers) { + return headers.exists(HTTP_HEADER_TRANSFER_ENCODING) || + headers.exists(HTTP_HEADER_CONTENT_LENGTH); +} + +bool parseQvalues(folly::StringPiece value, std::vector &output) { + bool result = true; + static folly::ThreadLocal> tokens; + tokens->clear(); + folly::split(",", value, *tokens, true /*ignore empty*/); + for (auto& token: *tokens) { + auto pos = token.find(';'); + double qvalue = 1.0; + if (pos != std::string::npos) { + auto qpos = token.find("q=", pos); + if (qpos != std::string::npos) { + folly::StringPiece qvalueStr(token.data() + qpos + 2, + token.size() - (qpos + 2)); + try { + qvalue = folly::to(&qvalueStr); + } catch (const std::range_error&) { + // q= + result = false; + } + // we could validate that the remainder of qvalueStr was all whitespace, + // for now we just discard it + } else { + // ; but no q= + result = false; + } + token.reset(token.start(), pos); + } + // strip leading whitespace + while (token.size() > 0 && isspace(token[0])) { + token.reset(token.start() + 1, token.size() - 1); + } + if (token.size() == 0) { + // empty token + result = false; + } else { + output.emplace_back(token, qvalue); + } + } + return result && output.size() > 0; +} + +}} diff --git a/proxygen/lib/http/RFC2616.h b/proxygen/lib/http/RFC2616.h new file mode 100644 index 0000000000..5a22da308e --- /dev/null +++ b/proxygen/lib/http/RFC2616.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPMethod.h" + +#include +#include + +namespace proxygen { + +class HTTPHeaders; + +namespace RFC2616 { + +/** + * This file contains functions for determining when certain tricky parts of RFC + * 2616 arise. + */ + +/** + * The HTTP request as defined in RFC 2616 may or may not have a body. In some + * cases they MUST NOT have a body. In other cases, the body has no semantic + * meaning and so is not defined. Finally, for some methods, the body is well + * defined. Please see Section 9 and 4.3 for details on this. + */ +enum class BodyAllowed { + DEFINED, + NOT_DEFINED, + NOT_ALLOWED, +}; +BodyAllowed isRequestBodyAllowed(boost::optional method); + +/** + * Some status codes imply that there MUST NOT be a response body. See section + * 4.3: "All 1xx (informational), 204 (no content), and 304 (not modified) + * responses MUST NOT include a message-body." + * @param status The code to test (100 <= status <= 999) + */ +bool responseBodyMustBeEmpty(unsigned status); + +/** + * Returns true if the headers imply that a body will follow. Note that in some + * situations a body may come even if this function returns false (e.g. a 1.0 + * response body's length can be given implicitly by closing the connection). + */ +bool bodyImplied(const HTTPHeaders& headers); + +/** + * Parse a string containing tokens and qvalues, such as the RFC strings for + * Accept-Charset, Accept-Encoding and Accept-Language. It won't work for + * complex Accept: headers because it doesn't return parameters or + * accept-extension. + * + * See RFC sections 14.2, 14.3, 14.4 for definitions of these header values + * + * TODO: optionally sort by qvalue descending + * + * Return true if the string was well formed according to the RFC. Note it can + * return false but still populate output with best-effort parsing. + */ +typedef std::pair TokenQPair; + +bool parseQvalues(folly::StringPiece value, std::vector &output); + +}} diff --git a/proxygen/lib/http/Window.cpp b/proxygen/lib/http/Window.cpp new file mode 100644 index 0000000000..bdfb63b81d --- /dev/null +++ b/proxygen/lib/http/Window.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/Window.h" + +#include +#include + +namespace proxygen { + +Window::Window(uint32_t capacity) { + CHECK(setCapacity(capacity)); +} + +int32_t Window::getSize() const { + return capacity_ - outstanding_; +} + +uint32_t Window::getNonNegativeSize() const { + auto size = getSize(); + return size > 0 ? size : 0; +} + +uint32_t Window::getCapacity() const { + // This conversion is safe since we always ensure capacity_ > 0 + return static_cast(capacity_); +} + +uint32_t Window::getOutstanding() const { + return outstanding_ < 0 ? 0 : outstanding_; +} + +bool Window::reserve(const uint32_t amount, bool strict) { + if (amount > std::numeric_limits::max()) { + VLOG(3) << "Cannot shrink window by more than 2^31 - 1. " + << "Attempted decrement of " << amount; + return false; + } + const uint32_t limit = std::numeric_limits::max() - outstanding_; + if (outstanding_ > 0 && amount > limit) { + VLOG(3) << "Overflow detected. Window change failed."; + return false; + } + const int32_t newOutstanding = outstanding_ + amount; + if (strict && newOutstanding > capacity_) { + VLOG(3) << "Outstanding bytes (" << newOutstanding << ") exceeded " + << "window capacity (" << capacity_ << ")"; + return false; + } + outstanding_ = newOutstanding; + return true; +} + +bool Window::free(const uint32_t amount) { + if (amount > std::numeric_limits::max()) { + VLOG(3) << "Cannot expand window by more than 2^31 - 1. " + << "Attempted increment of " << amount; + return false; + } + const uint32_t limit = outstanding_ - std::numeric_limits::min(); + if (outstanding_ < 0 && amount > limit) { + VLOG(3) << "Underflow detected. Window change failed."; + return false; + } + const int32_t newOutstanding = outstanding_ - amount; + if (newOutstanding < capacity_ - std::numeric_limits::max()) { + VLOG(3) << "Window exceeded 2^31 - 1. Window change failed."; + return false; + } + outstanding_ = newOutstanding; + return true; +} + +bool Window::setCapacity(const uint32_t capacity) { + if (capacity > std::numeric_limits::max()) { + VLOG(3) << "Cannot set initial window > 2^31 -1."; + return false; + } + + if (capacity > uint32_t(capacity_) && + (capacity - capacity_ > + uint32_t(std::numeric_limits::max() - getSize()))) { + VLOG(3) << "Increasing the capacity overflowed the window"; + return false; + } + capacity_ = static_cast(capacity); + return true; +} + +} diff --git a/proxygen/lib/http/Window.h b/proxygen/lib/http/Window.h new file mode 100644 index 0000000000..983dc9c77e --- /dev/null +++ b/proxygen/lib/http/Window.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +/** + * A class that implements SPDY & HTTP/2 window management. This class + * should be used for both connection and stream level flow control. + */ +class Window { + public: + /** + * Constructs a new Window. + * @param capacity The initial capacity of this Window. The initial size will + * also be equal to this. This parameter must not be > 2^31 -1 + */ + explicit Window(uint32_t capacity); + + /** + * Returns the number of bytes available to be consumed in this + * window. This could become a negative number if the initial window is + * set to a smaller number. + */ + int32_t getSize() const; + + /** + * Returns the number of bytes available to be consumed in this + * window. If that number went negative somehow, this function clamps + * the return value to zero. + */ + uint32_t getNonNegativeSize() const; + + /** + * Returns the size of the initial window. That is, the total number of + * bytes allowed to be outstanding on this window. + */ + uint32_t getCapacity() const; + + /** + * Returns the number of bytes reserved in this window. If multiple + * calls to free() caused this number to go negative, this function + * returns 0. + */ + uint32_t getOutstanding() const; + + /** + * @param amount The amount to reduce the window size by. Increases + * bytes outstanding by amount. + * @param strict If true, reserve() will return false if there is + * no space remaining in the window. If false, + * reserve() will return true until the integer + * overflows. + */ + bool reserve(uint32_t amount, bool strict = true); + + /** + * Increment the window size by amount. Decrease bytes outstanding by + * amount. The window size could become greater than the capacity here. + */ + bool free(uint32_t amount); + + /** + * Sets a new initial window size. This will also apply the delta + * between the current window size and the new window size. Returns false + * iff applying the new initial window fails. + */ + bool setCapacity(uint32_t capacity); + + private: + int32_t outstanding_{0}; + int32_t capacity_{0}; // always positive +}; + +} diff --git a/proxygen/lib/http/codec/CodecDictionaries.h b/proxygen/lib/http/codec/CodecDictionaries.h new file mode 100644 index 0000000000..155383f7d7 --- /dev/null +++ b/proxygen/lib/http/codec/CodecDictionaries.h @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +/** + * Compression dictionary for SPDYv2; note that the trailing null is included. + * From http://www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft2 + */ +const char kSPDYv2Dictionary[] = + "optionsgetheadpostputdeletetraceacceptaccept-charsetaccept-encodingaccept-" + "languageauthorizationexpectfromhostif-modified-sinceif-matchif-none-matchi" + "f-rangeif-unmodifiedsincemax-forwardsproxy-authorizationrangerefererteuser" + "-agent10010120020120220320420520630030130230330430530630740040140240340440" + "5406407408409410411412413414415416417500501502503504505accept-rangesageeta" + "glocationproxy-authenticatepublicretry-afterservervarywarningwww-authentic" + "ateallowcontent-basecontent-encodingcache-controlconnectiondatetrailertran" + "sfer-encodingupgradeviawarningcontent-languagecontent-lengthcontent-locati" + "oncontent-md5content-rangecontent-typeetagexpireslast-modifiedset-cookieMo" + "ndayTuesdayWednesdayThursdayFridaySaturdaySundayJanFebMarAprMayJunJulAugSe" + "pOctNovDecchunkedtext/htmlimage/pngimage/jpgimage/gifapplication/xmlapplic" + "ation/xhtmltext/plainpublicmax-agecharset=iso-8859-1utf-8gzipdeflateHTTP/1" + ".1statusversionurl"; + +/** + * Compression dictionary for SPDYv3. Copied from: + * http://www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3 + */ +const unsigned char kSPDYv3Dictionary[] = { + 0x00, 0x00, 0x00, 0x07, 0x6f, 0x70, 0x74, 0x69, // - - - - o p t i + 0x6f, 0x6e, 0x73, 0x00, 0x00, 0x00, 0x04, 0x68, // o n s - - - - h + 0x65, 0x61, 0x64, 0x00, 0x00, 0x00, 0x04, 0x70, // e a d - - - - p + 0x6f, 0x73, 0x74, 0x00, 0x00, 0x00, 0x03, 0x70, // o s t - - - - p + 0x75, 0x74, 0x00, 0x00, 0x00, 0x06, 0x64, 0x65, // u t - - - - d e + 0x6c, 0x65, 0x74, 0x65, 0x00, 0x00, 0x00, 0x05, // l e t e - - - - + 0x74, 0x72, 0x61, 0x63, 0x65, 0x00, 0x00, 0x00, // t r a c e - - - + 0x06, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x00, // - a c c e p t - + 0x00, 0x00, 0x0e, 0x61, 0x63, 0x63, 0x65, 0x70, // - - - a c c e p + 0x74, 0x2d, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, // t - c h a r s e + 0x74, 0x00, 0x00, 0x00, 0x0f, 0x61, 0x63, 0x63, // t - - - - a c c + 0x65, 0x70, 0x74, 0x2d, 0x65, 0x6e, 0x63, 0x6f, // e p t - e n c o + 0x64, 0x69, 0x6e, 0x67, 0x00, 0x00, 0x00, 0x0f, // d i n g - - - - + 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d, 0x6c, // a c c e p t - l + 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x00, // a n g u a g e - + 0x00, 0x00, 0x0d, 0x61, 0x63, 0x63, 0x65, 0x70, // - - - a c c e p + 0x74, 0x2d, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, // t - r a n g e s + 0x00, 0x00, 0x00, 0x03, 0x61, 0x67, 0x65, 0x00, // - - - - a g e - + 0x00, 0x00, 0x05, 0x61, 0x6c, 0x6c, 0x6f, 0x77, // - - - a l l o w + 0x00, 0x00, 0x00, 0x0d, 0x61, 0x75, 0x74, 0x68, // - - - - a u t h + 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, // o r i z a t i o + 0x6e, 0x00, 0x00, 0x00, 0x0d, 0x63, 0x61, 0x63, // n - - - - c a c + 0x68, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, // h e - c o n t r + 0x6f, 0x6c, 0x00, 0x00, 0x00, 0x0a, 0x63, 0x6f, // o l - - - - c o + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, // n n e c t i o n + 0x00, 0x00, 0x00, 0x0c, 0x63, 0x6f, 0x6e, 0x74, // - - - - c o n t + 0x65, 0x6e, 0x74, 0x2d, 0x62, 0x61, 0x73, 0x65, // e n t - b a s e + 0x00, 0x00, 0x00, 0x10, 0x63, 0x6f, 0x6e, 0x74, // - - - - c o n t + 0x65, 0x6e, 0x74, 0x2d, 0x65, 0x6e, 0x63, 0x6f, // e n t - e n c o + 0x64, 0x69, 0x6e, 0x67, 0x00, 0x00, 0x00, 0x10, // d i n g - - - - + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, // c o n t e n t - + 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, // l a n g u a g e + 0x00, 0x00, 0x00, 0x0e, 0x63, 0x6f, 0x6e, 0x74, // - - - - c o n t + 0x65, 0x6e, 0x74, 0x2d, 0x6c, 0x65, 0x6e, 0x67, // e n t - l e n g + 0x74, 0x68, 0x00, 0x00, 0x00, 0x10, 0x63, 0x6f, // t h - - - - c o + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x6c, 0x6f, // n t e n t - l o + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, // c a t i o n - - + 0x00, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, // - - c o n t e n + 0x74, 0x2d, 0x6d, 0x64, 0x35, 0x00, 0x00, 0x00, // t - m d 5 - - - + 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, // - c o n t e n t + 0x2d, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x00, 0x00, // - r a n g e - - + 0x00, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, // - - c o n t e n + 0x74, 0x2d, 0x74, 0x79, 0x70, 0x65, 0x00, 0x00, // t - t y p e - - + 0x00, 0x04, 0x64, 0x61, 0x74, 0x65, 0x00, 0x00, // - - d a t e - - + 0x00, 0x04, 0x65, 0x74, 0x61, 0x67, 0x00, 0x00, // - - e t a g - - + 0x00, 0x06, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, // - - e x p e c t + 0x00, 0x00, 0x00, 0x07, 0x65, 0x78, 0x70, 0x69, // - - - - e x p i + 0x72, 0x65, 0x73, 0x00, 0x00, 0x00, 0x04, 0x66, // r e s - - - - f + 0x72, 0x6f, 0x6d, 0x00, 0x00, 0x00, 0x04, 0x68, // r o m - - - - h + 0x6f, 0x73, 0x74, 0x00, 0x00, 0x00, 0x08, 0x69, // o s t - - - - i + 0x66, 0x2d, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x00, // f - m a t c h - + 0x00, 0x00, 0x11, 0x69, 0x66, 0x2d, 0x6d, 0x6f, // - - - i f - m o + 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x2d, 0x73, // d i f i e d - s + 0x69, 0x6e, 0x63, 0x65, 0x00, 0x00, 0x00, 0x0d, // i n c e - - - - + 0x69, 0x66, 0x2d, 0x6e, 0x6f, 0x6e, 0x65, 0x2d, // i f - n o n e - + 0x6d, 0x61, 0x74, 0x63, 0x68, 0x00, 0x00, 0x00, // m a t c h - - - + 0x08, 0x69, 0x66, 0x2d, 0x72, 0x61, 0x6e, 0x67, // - i f - r a n g + 0x65, 0x00, 0x00, 0x00, 0x13, 0x69, 0x66, 0x2d, // e - - - - i f - + 0x75, 0x6e, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, // u n m o d i f i + 0x65, 0x64, 0x2d, 0x73, 0x69, 0x6e, 0x63, 0x65, // e d - s i n c e + 0x00, 0x00, 0x00, 0x0d, 0x6c, 0x61, 0x73, 0x74, // - - - - l a s t + 0x2d, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, // - m o d i f i e + 0x64, 0x00, 0x00, 0x00, 0x08, 0x6c, 0x6f, 0x63, // d - - - - l o c + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, 0x00, // a t i o n - - - + 0x0c, 0x6d, 0x61, 0x78, 0x2d, 0x66, 0x6f, 0x72, // - m a x - f o r + 0x77, 0x61, 0x72, 0x64, 0x73, 0x00, 0x00, 0x00, // w a r d s - - - + 0x06, 0x70, 0x72, 0x61, 0x67, 0x6d, 0x61, 0x00, // - p r a g m a - + 0x00, 0x00, 0x12, 0x70, 0x72, 0x6f, 0x78, 0x79, // - - - p r o x y + 0x2d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, // - a u t h e n t + 0x69, 0x63, 0x61, 0x74, 0x65, 0x00, 0x00, 0x00, // i c a t e - - - + 0x13, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2d, 0x61, // - p r o x y - a + 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, // u t h o r i z a + 0x74, 0x69, 0x6f, 0x6e, 0x00, 0x00, 0x00, 0x05, // t i o n - - - - + 0x72, 0x61, 0x6e, 0x67, 0x65, 0x00, 0x00, 0x00, // r a n g e - - - + 0x07, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x72, // - r e f e r e r + 0x00, 0x00, 0x00, 0x0b, 0x72, 0x65, 0x74, 0x72, // - - - - r e t r + 0x79, 0x2d, 0x61, 0x66, 0x74, 0x65, 0x72, 0x00, // y - a f t e r - + 0x00, 0x00, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, // - - - s e r v e + 0x72, 0x00, 0x00, 0x00, 0x02, 0x74, 0x65, 0x00, // r - - - - t e - + 0x00, 0x00, 0x07, 0x74, 0x72, 0x61, 0x69, 0x6c, // - - - t r a i l + 0x65, 0x72, 0x00, 0x00, 0x00, 0x11, 0x74, 0x72, // e r - - - - t r + 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2d, 0x65, // a n s f e r - e + 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x00, // n c o d i n g - + 0x00, 0x00, 0x07, 0x75, 0x70, 0x67, 0x72, 0x61, // - - - u p g r a + 0x64, 0x65, 0x00, 0x00, 0x00, 0x0a, 0x75, 0x73, // d e - - - - u s + 0x65, 0x72, 0x2d, 0x61, 0x67, 0x65, 0x6e, 0x74, // e r - a g e n t + 0x00, 0x00, 0x00, 0x04, 0x76, 0x61, 0x72, 0x79, // - - - - v a r y + 0x00, 0x00, 0x00, 0x03, 0x76, 0x69, 0x61, 0x00, // - - - - v i a - + 0x00, 0x00, 0x07, 0x77, 0x61, 0x72, 0x6e, 0x69, // - - - w a r n i + 0x6e, 0x67, 0x00, 0x00, 0x00, 0x10, 0x77, 0x77, // n g - - - - w w + 0x77, 0x2d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, // w - a u t h e n + 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x00, 0x00, // t i c a t e - - + 0x00, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, // - - m e t h o d + 0x00, 0x00, 0x00, 0x03, 0x67, 0x65, 0x74, 0x00, // - - - - g e t - + 0x00, 0x00, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, // - - - s t a t u + 0x73, 0x00, 0x00, 0x00, 0x06, 0x32, 0x30, 0x30, // s - - - - 2 0 0 + 0x20, 0x4f, 0x4b, 0x00, 0x00, 0x00, 0x07, 0x76, // - O K - - - - v + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x00, // e r s i o n - - + 0x00, 0x08, 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, // - - H T T P - 1 + 0x2e, 0x31, 0x00, 0x00, 0x00, 0x03, 0x75, 0x72, // - 1 - - - - u r + 0x6c, 0x00, 0x00, 0x00, 0x06, 0x70, 0x75, 0x62, // l - - - - p u b + 0x6c, 0x69, 0x63, 0x00, 0x00, 0x00, 0x0a, 0x73, // l i c - - - - s + 0x65, 0x74, 0x2d, 0x63, 0x6f, 0x6f, 0x6b, 0x69, // e t - c o o k i + 0x65, 0x00, 0x00, 0x00, 0x0a, 0x6b, 0x65, 0x65, // e - - - - k e e + 0x70, 0x2d, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x00, // p - a l i v e - + 0x00, 0x00, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, // - - - o r i g i + 0x6e, 0x31, 0x30, 0x30, 0x31, 0x30, 0x31, 0x32, // n 1 0 0 1 0 1 2 + 0x30, 0x31, 0x32, 0x30, 0x32, 0x32, 0x30, 0x35, // 0 1 2 0 2 2 0 5 + 0x32, 0x30, 0x36, 0x33, 0x30, 0x30, 0x33, 0x30, // 2 0 6 3 0 0 3 0 + 0x32, 0x33, 0x30, 0x33, 0x33, 0x30, 0x34, 0x33, // 2 3 0 3 3 0 4 3 + 0x30, 0x35, 0x33, 0x30, 0x36, 0x33, 0x30, 0x37, // 0 5 3 0 6 3 0 7 + 0x34, 0x30, 0x32, 0x34, 0x30, 0x35, 0x34, 0x30, // 4 0 2 4 0 5 4 0 + 0x36, 0x34, 0x30, 0x37, 0x34, 0x30, 0x38, 0x34, // 6 4 0 7 4 0 8 4 + 0x30, 0x39, 0x34, 0x31, 0x30, 0x34, 0x31, 0x31, // 0 9 4 1 0 4 1 1 + 0x34, 0x31, 0x32, 0x34, 0x31, 0x33, 0x34, 0x31, // 4 1 2 4 1 3 4 1 + 0x34, 0x34, 0x31, 0x35, 0x34, 0x31, 0x36, 0x34, // 4 4 1 5 4 1 6 4 + 0x31, 0x37, 0x35, 0x30, 0x32, 0x35, 0x30, 0x34, // 1 7 5 0 2 5 0 4 + 0x35, 0x30, 0x35, 0x32, 0x30, 0x33, 0x20, 0x4e, // 5 0 5 2 0 3 - N + 0x6f, 0x6e, 0x2d, 0x41, 0x75, 0x74, 0x68, 0x6f, // o n - A u t h o + 0x72, 0x69, 0x74, 0x61, 0x74, 0x69, 0x76, 0x65, // r i t a t i v e + 0x20, 0x49, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, // - I n f o r m a + 0x74, 0x69, 0x6f, 0x6e, 0x32, 0x30, 0x34, 0x20, // t i o n 2 0 4 - + 0x4e, 0x6f, 0x20, 0x43, 0x6f, 0x6e, 0x74, 0x65, // N o - C o n t e + 0x6e, 0x74, 0x33, 0x30, 0x31, 0x20, 0x4d, 0x6f, // n t 3 0 1 - M o + 0x76, 0x65, 0x64, 0x20, 0x50, 0x65, 0x72, 0x6d, // v e d - P e r m + 0x61, 0x6e, 0x65, 0x6e, 0x74, 0x6c, 0x79, 0x34, // a n e n t l y 4 + 0x30, 0x30, 0x20, 0x42, 0x61, 0x64, 0x20, 0x52, // 0 0 - B a d - R + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x34, 0x30, // e q u e s t 4 0 + 0x31, 0x20, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, // 1 - U n a u t h + 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x34, 0x30, // o r i z e d 4 0 + 0x33, 0x20, 0x46, 0x6f, 0x72, 0x62, 0x69, 0x64, // 3 - F o r b i d + 0x64, 0x65, 0x6e, 0x34, 0x30, 0x34, 0x20, 0x4e, // d e n 4 0 4 - N + 0x6f, 0x74, 0x20, 0x46, 0x6f, 0x75, 0x6e, 0x64, // o t - F o u n d + 0x35, 0x30, 0x30, 0x20, 0x49, 0x6e, 0x74, 0x65, // 5 0 0 - I n t e + 0x72, 0x6e, 0x61, 0x6c, 0x20, 0x53, 0x65, 0x72, // r n a l - S e r + 0x76, 0x65, 0x72, 0x20, 0x45, 0x72, 0x72, 0x6f, // v e r - E r r o + 0x72, 0x35, 0x30, 0x31, 0x20, 0x4e, 0x6f, 0x74, // r 5 0 1 - N o t + 0x20, 0x49, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, // - I m p l e m e + 0x6e, 0x74, 0x65, 0x64, 0x35, 0x30, 0x33, 0x20, // n t e d 5 0 3 - + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, // S e r v i c e - + 0x55, 0x6e, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, // U n a v a i l a + 0x62, 0x6c, 0x65, 0x4a, 0x61, 0x6e, 0x20, 0x46, // b l e J a n - F + 0x65, 0x62, 0x20, 0x4d, 0x61, 0x72, 0x20, 0x41, // e b - M a r - A + 0x70, 0x72, 0x20, 0x4d, 0x61, 0x79, 0x20, 0x4a, // p r - M a y - J + 0x75, 0x6e, 0x20, 0x4a, 0x75, 0x6c, 0x20, 0x41, // u n - J u l - A + 0x75, 0x67, 0x20, 0x53, 0x65, 0x70, 0x74, 0x20, // u g - S e p t - + 0x4f, 0x63, 0x74, 0x20, 0x4e, 0x6f, 0x76, 0x20, // O c t - N o v - + 0x44, 0x65, 0x63, 0x20, 0x30, 0x30, 0x3a, 0x30, // D e c - 0 0 - 0 + 0x30, 0x3a, 0x30, 0x30, 0x20, 0x4d, 0x6f, 0x6e, // 0 - 0 0 - M o n + 0x2c, 0x20, 0x54, 0x75, 0x65, 0x2c, 0x20, 0x57, // - - T u e - - W + 0x65, 0x64, 0x2c, 0x20, 0x54, 0x68, 0x75, 0x2c, // e d - - T h u - + 0x20, 0x46, 0x72, 0x69, 0x2c, 0x20, 0x53, 0x61, // - F r i - - S a + 0x74, 0x2c, 0x20, 0x53, 0x75, 0x6e, 0x2c, 0x20, // t - - S u n - - + 0x47, 0x4d, 0x54, 0x63, 0x68, 0x75, 0x6e, 0x6b, // G M T c h u n k + 0x65, 0x64, 0x2c, 0x74, 0x65, 0x78, 0x74, 0x2f, // e d - t e x t - + 0x68, 0x74, 0x6d, 0x6c, 0x2c, 0x69, 0x6d, 0x61, // h t m l - i m a + 0x67, 0x65, 0x2f, 0x70, 0x6e, 0x67, 0x2c, 0x69, // g e - p n g - i + 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x6a, 0x70, 0x67, // m a g e - j p g + 0x2c, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x2f, 0x67, // - i m a g e - g + 0x69, 0x66, 0x2c, 0x61, 0x70, 0x70, 0x6c, 0x69, // i f - a p p l i + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x78, // c a t i o n - x + 0x6d, 0x6c, 0x2c, 0x61, 0x70, 0x70, 0x6c, 0x69, // m l - a p p l i + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x78, // c a t i o n - x + 0x68, 0x74, 0x6d, 0x6c, 0x2b, 0x78, 0x6d, 0x6c, // h t m l - x m l + 0x2c, 0x74, 0x65, 0x78, 0x74, 0x2f, 0x70, 0x6c, // - t e x t - p l + 0x61, 0x69, 0x6e, 0x2c, 0x74, 0x65, 0x78, 0x74, // a i n - t e x t + 0x2f, 0x6a, 0x61, 0x76, 0x61, 0x73, 0x63, 0x72, // - j a v a s c r + 0x69, 0x70, 0x74, 0x2c, 0x70, 0x75, 0x62, 0x6c, // i p t - p u b l + 0x69, 0x63, 0x70, 0x72, 0x69, 0x76, 0x61, 0x74, // i c p r i v a t + 0x65, 0x6d, 0x61, 0x78, 0x2d, 0x61, 0x67, 0x65, // e m a x - a g e + 0x3d, 0x67, 0x7a, 0x69, 0x70, 0x2c, 0x64, 0x65, // - g z i p - d e + 0x66, 0x6c, 0x61, 0x74, 0x65, 0x2c, 0x73, 0x64, // f l a t e - s d + 0x63, 0x68, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, // c h c h a r s e + 0x74, 0x3d, 0x75, 0x74, 0x66, 0x2d, 0x38, 0x63, // t - u t f - 8 c + 0x68, 0x61, 0x72, 0x73, 0x65, 0x74, 0x3d, 0x69, // h a r s e t - i + 0x73, 0x6f, 0x2d, 0x38, 0x38, 0x35, 0x39, 0x2d, // s o - 8 8 5 9 - + 0x31, 0x2c, 0x75, 0x74, 0x66, 0x2d, 0x2c, 0x2a, // 1 - u t f - - - + 0x2c, 0x65, 0x6e, 0x71, 0x3d, 0x30, 0x2e // - e n q - 0 - +}; + +} //proxygen diff --git a/proxygen/lib/http/codec/CodecProtocol.cpp b/proxygen/lib/http/codec/CodecProtocol.cpp new file mode 100644 index 0000000000..3ae4e2d111 --- /dev/null +++ b/proxygen/lib/http/codec/CodecProtocol.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/CodecProtocol.h" + +#include + +namespace proxygen { + +namespace { +static const std::string http_1_1 = "http/1.1"; +static const std::string spdy_2 = "spdy/2"; +static const std::string spdy_3 = "spdy/3"; +static const std::string spdy_3_1 = "spdy/3.1"; +static const std::string spdy_3_1_hpack = "spdy/3.1-hpack"; +static const std::string http_2 = "http/2"; +static const std::string empty = ""; +} + +extern const std::string& getCodecProtocolString(CodecProtocol proto) { + switch (proto) { + case CodecProtocol::HTTP_1_1: return http_1_1; + case CodecProtocol::SPDY_2: return spdy_2; + case CodecProtocol::SPDY_3: return spdy_3; + case CodecProtocol::SPDY_3_1: return spdy_3_1; + case CodecProtocol::SPDY_3_1_HPACK: return spdy_3_1_hpack; + case CodecProtocol::HTTP_2: return http_2; + } + LOG(FATAL) << "Unreachable"; + return empty; +} + +extern bool isValidCodecProtocolStr(const std::string& protocolStr) { + return protocolStr == http_1_1 || + protocolStr == spdy_2 || + protocolStr == spdy_3 || + protocolStr == spdy_3_1 || + protocolStr == spdy_3_1_hpack || + protocolStr == http_2; +} + +extern CodecProtocol getCodecProtocolFromStr(const std::string& protocolStr) { + if (protocolStr == http_1_1) { + return CodecProtocol::HTTP_1_1; + } else if (protocolStr == spdy_2) { + return CodecProtocol::SPDY_2; + } else if (protocolStr == spdy_3) { + return CodecProtocol::SPDY_3; + } else if (protocolStr == spdy_3_1) { + return CodecProtocol::SPDY_3_1; + } else if (protocolStr == spdy_3_1_hpack) { + return CodecProtocol::SPDY_3_1_HPACK; + } else if (protocolStr == http_2) { + return CodecProtocol::HTTP_2; + } else { + // return default protocol + return CodecProtocol::HTTP_1_1; + } +} + +extern bool isSpdyCodecProtocol(CodecProtocol protocol) { + return protocol == CodecProtocol::SPDY_2 || + protocol == CodecProtocol::SPDY_3 || + protocol == CodecProtocol::SPDY_3_1 || + protocol == CodecProtocol::SPDY_3_1_HPACK; +} + +extern uint8_t maxProtocolPriority(CodecProtocol protocol) { + // Priorities per protocol + switch (protocol) { + case CodecProtocol::SPDY_2: + // SPDY 2 Max Priority + return 3; + case CodecProtocol::SPDY_3: + case CodecProtocol::SPDY_3_1: + case CodecProtocol::SPDY_3_1_HPACK: + // SPDY 3 Max Priority + return 7; + case CodecProtocol::HTTP_1_1: + case CodecProtocol::HTTP_2: + //HTTP doesn't support priorities + return 0; + } + return 0; +} +} diff --git a/proxygen/lib/http/codec/CodecProtocol.h b/proxygen/lib/http/codec/CodecProtocol.h new file mode 100644 index 0000000000..178630da0c --- /dev/null +++ b/proxygen/lib/http/codec/CodecProtocol.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +enum class CodecProtocol : uint8_t { + HTTP_1_1, + SPDY_2, + SPDY_3, + SPDY_3_1, + SPDY_3_1_HPACK, + HTTP_2, +}; + +/** + * Returns a debugging name to refer to the given protocol. + */ +extern const std::string& getCodecProtocolString(CodecProtocol); + +/** + * Check if given debugging name refers to a valid protocol. + */ +extern bool isValidCodecProtocolStr(const std::string& protocolStr); + +/** + * Get the protocol from the given debugging name. + * If it's an invalid string, return the default protocol. + */ +extern CodecProtocol getCodecProtocolFromStr(const std::string& protocolStr); + +/** + * Check if the given protocol is SPDY. + */ +extern bool isSpdyCodecProtocol(CodecProtocol protocol); + +/** + * Returns the maximum priority for the given protocol. + */ +extern uint8_t maxProtocolPriority(CodecProtocol); +} diff --git a/proxygen/lib/http/codec/ErrorCode.cpp b/proxygen/lib/http/codec/ErrorCode.cpp new file mode 100644 index 0000000000..8499000986 --- /dev/null +++ b/proxygen/lib/http/codec/ErrorCode.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/ErrorCode.h" + +#include + +namespace proxygen { + +const uint8_t kMaxErrorCode = 12; + +const char* getErrorCodeString(ErrorCode error) { + switch (error) { + case ErrorCode::NO_ERROR: return "NO_ERROR"; + case ErrorCode::PROTOCOL_ERROR: return "PROTOCOL_ERROR"; + case ErrorCode::INTERNAL_ERROR: return "INTERNAL_ERROR"; + case ErrorCode::FLOW_CONTROL_ERROR: return "FLOW_CONTROL_ERROR"; + case ErrorCode::SETTINGS_TIMEOUT: return "SETTINGS_TIMEOUT"; + case ErrorCode::STREAM_CLOSED: return "STREAM_CLOSED"; + case ErrorCode::FRAME_SIZE_ERROR: return "FRAME_SIZE_ERROR"; + case ErrorCode::REFUSED_STREAM: return "REFUSED_STREAM"; + case ErrorCode::CANCEL: return "CANCEL"; + case ErrorCode::COMPRESSION_ERROR: return "COMPRESSION_ERROR"; + case ErrorCode::CONNECT_ERROR: return "CONNECT_ERROR"; + case ErrorCode::ENHANCE_YOUR_CALM: return "ENHANCE_YOUR_CALM"; + case ErrorCode::INADEQUATE_SECURITY: return "INADEQUATE_SECURITY"; + case ErrorCode::_SPDY_INVALID_STREAM: return "_SPDY_INVALID_STREAM"; + } + LOG(FATAL) << "Unreachable"; + return ""; +} + +} diff --git a/proxygen/lib/http/codec/ErrorCode.h b/proxygen/lib/http/codec/ErrorCode.h new file mode 100644 index 0000000000..4b415ebeb3 --- /dev/null +++ b/proxygen/lib/http/codec/ErrorCode.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +#define RETURN_IF_ERROR(err) \ + if (err != ErrorCode::NO_ERROR) { \ + VLOG(4) << "Returning with error=" << getErrorCodeString(err); \ + return err; \ + } + +namespace proxygen { + +// Error codes are 32-bit fields that are used in RST_STREAM and GOAWAY +// frames to convey the reasons for the stream or connection error. + +// We only need <1 byte to represent it in memory +enum class ErrorCode: uint8_t { + NO_ERROR = 0, + PROTOCOL_ERROR = 1, + INTERNAL_ERROR = 2, + FLOW_CONTROL_ERROR = 3, + SETTINGS_TIMEOUT = 4, + STREAM_CLOSED = 5, + FRAME_SIZE_ERROR = 6, + REFUSED_STREAM = 7, + CANCEL = 8, + COMPRESSION_ERROR = 9, + CONNECT_ERROR = 10, + ENHANCE_YOUR_CALM = 11, + INADEQUATE_SECURITY = 12, + // This code is *NOT* to be used outside of SPDYCodec. Delete this + // when we deprecate SPDY. + _SPDY_INVALID_STREAM = 100, +}; + +extern const uint8_t kMaxErrorCode; + +/** + * Returns a string representation of the error code. + */ +extern const char* getErrorCodeString(ErrorCode error); + +} diff --git a/proxygen/lib/http/codec/FlowControlFilter.cpp b/proxygen/lib/http/codec/FlowControlFilter.cpp new file mode 100644 index 0000000000..c54a868e57 --- /dev/null +++ b/proxygen/lib/http/codec/FlowControlFilter.cpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/FlowControlFilter.h" + +#include "proxygen/lib/http/codec/SPDYConstants.h" + +namespace proxygen { + +namespace { +HTTPException getException() { + HTTPException ex(HTTPException::Direction::INGRESS_AND_EGRESS); + ex.setCodecStatusCode(ErrorCode::FLOW_CONTROL_ERROR); + return ex; +} + +} + +FlowControlFilter::FlowControlFilter(Callback& callback, + folly::IOBufQueue& writeBuf, + HTTPCodec* codec, + uint32_t recvCapacity): + notify_(callback), + recvWindow_(spdy::kInitialWindow), + sendWindow_(spdy::kInitialWindow), + error_(false), + sendsBlocked_(false) { + if (recvCapacity < spdy::kInitialWindow) { + VLOG(4) << "Ignoring low conn-level recv window size of " << recvCapacity; + } else if (recvCapacity > spdy::kInitialWindow) { + auto delta = recvCapacity - spdy::kInitialWindow; + VLOG(4) << "Incrementing default conn-level recv window by " << delta; + CHECK(recvWindow_.setCapacity(recvCapacity)); + codec->generateWindowUpdate(writeBuf, 0, delta); + } +} + +bool FlowControlFilter::ingressBytesProcessed(folly::IOBufQueue& writeBuf, + uint32_t delta) { + toAck_ += delta; + if (toAck_ > recvWindow_.getCapacity() / 2) { + CHECK(recvWindow_.free(toAck_)); + call_->generateWindowUpdate(writeBuf, 0, toAck_); + toAck_ = 0; + return true; + } + return false; +} + +uint32_t FlowControlFilter::getAvailableSend() const { + return sendWindow_.getNonNegativeSize(); +} + +bool FlowControlFilter::isReusable() const { + if (error_) { + return false; + } + return call_->isReusable(); +} + +void FlowControlFilter::onBody(StreamID stream, + std::unique_ptr chain) { + if (!recvWindow_.reserve(chain->computeChainDataLength())) { + error_ = true; + HTTPException ex = getException(); + callback_->onError(0, ex, false); + } else { + callback_->onBody(stream, std::move(chain)); + } +} + +void FlowControlFilter::onWindowUpdate(StreamID stream, uint32_t amount) { + if (!stream) { + bool success = sendWindow_.free(amount); + if (!success) { + LOG(WARNING) << "Remote side sent connection-level WINDOW_UPDATE " + << "that could not be applied. Aborting session."; + // If something went wrong applying the flow control change, abort + // the entire session. + error_ = true; + HTTPException ex = getException(); + callback_->onError(stream, ex, false); + } + if (sendsBlocked_ && sendWindow_.getNonNegativeSize()) { + sendsBlocked_ = false; + notify_.onConnectionSendWindowOpen(); + } + // Don't forward. + } else { + callback_->onWindowUpdate(stream, amount); + } +} + +size_t FlowControlFilter::generateBody(folly::IOBufQueue& writeBuf, + StreamID stream, + std::unique_ptr chain, + bool eom) { + bool success = sendWindow_.reserve(chain->computeChainDataLength()); + // In the future, maybe make this DCHECK + CHECK(success) << "Session-level send window underflowed! " + << "Too much data sent without WINDOW_UPDATES!"; + + if (sendWindow_.getNonNegativeSize() == 0) { + // Need to inform when the send window is no longer full + sendsBlocked_ = true; + } + + return call_->generateBody(writeBuf, stream, std::move(chain), eom); +} + +size_t FlowControlFilter::generateWindowUpdate(folly::IOBufQueue& writeBuf, + StreamID stream, + uint32_t delta) { + CHECK(stream) << " someone tried to manually manipulate a conn-level window"; + return call_->generateWindowUpdate(writeBuf, stream, delta); +} + +} diff --git a/proxygen/lib/http/codec/FlowControlFilter.h b/proxygen/lib/http/codec/FlowControlFilter.h new file mode 100644 index 0000000000..e6b70a91cd --- /dev/null +++ b/proxygen/lib/http/codec/FlowControlFilter.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/Window.h" +#include "proxygen/lib/http/codec/HTTPCodecFilter.h" + +namespace folly { +class IOBufQueue; +} + +namespace proxygen { + +/** + * This class implements the logic for managing per-session flow + * control. Not every codec is interested in per-session flow control, so + * this filter can only be added in that case or else it is an error. + */ +class FlowControlFilter: + public PassThroughHTTPCodecFilter { + public: + class Callback { + public: + virtual ~Callback() {} + /** + * Notification channel to alert when the send window is no longer full. + */ + virtual void onConnectionSendWindowOpen() = 0; + }; + + /** + * Construct a flow control filter. + * @param callback A channel to be notified when the window is not + * full anymore. + * @param writeBuf The buffer to write egress on. This constructor + * may generate a window update frame on this buffer. + * @param codec The codec implementation. + * @param recvCapacity The initial size of the conn-level recv window. + * It must be >= 65536. + */ + explicit + FlowControlFilter(Callback& callback, + folly::IOBufQueue& writeBuf, + HTTPCodec* codec, + uint32_t recvCapacity); + + /** + * Notify the flow control filter that some ingress bytes were + * processed. If the number of bytes to acknowledge exceeds half the + * receive window's capacity, a WINDOW_UPDATE frame will be written. + * @param writeBuf The buffer to write egress on. This function may + * generate a WINDOW_UPDATE on this buffer. + * @param delta The number of bytes that were processed. + * @returns true iff we wrote a WINDOW_UPDATE frame to the write buf. + */ + bool ingressBytesProcessed(folly::IOBufQueue& writeBuf, uint32_t delta); + + /** + * @returns the number of bytes available in the connection-level send window + */ + uint32_t getAvailableSend() const; + + // Filter functions + + bool isReusable() const override; + + void onBody(StreamID stream, std::unique_ptr chain) override; + + void onWindowUpdate(StreamID stream, uint32_t amount) override; + + size_t generateBody(folly::IOBufQueue& writeBuf, + StreamID stream, + std::unique_ptr chain, + bool eom) override; + + size_t generateWindowUpdate(folly::IOBufQueue& writeBuf, + StreamID stream, + uint32_t delta) override; + + private: + + Callback& notify_; + Window recvWindow_; + Window sendWindow_; + uint32_t toAck_{0}; + bool error_:1; + bool sendsBlocked_:1; +}; + +} diff --git a/proxygen/lib/http/codec/HTTP1xCodec.cpp b/proxygen/lib/http/codec/HTTP1xCodec.cpp new file mode 100644 index 0000000000..4239abacd5 --- /dev/null +++ b/proxygen/lib/http/codec/HTTP1xCodec.cpp @@ -0,0 +1,1071 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/HTTP1xCodec.h" + +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/RFC2616.h" + +#include + +using folly::IOBuf; +using folly::IOBufQueue; +using std::string; +using std::unique_ptr; + +namespace { + +const char CRLF[] = "\r\n"; + +/** + * Write an ASCII decimal representation of an integer value + * @note This function does -not- append a trailing null byte. + * @param value Integer value to write. + * @param dst Location to which the value will be written. + * @return Number of bytes written. + */ +unsigned u64toa(uint64_t value, void* dst) { + // Write backwards. + char* next = (char*)dst; + char* start = next; + do { + *next++ = '0' + (value % 10); + value /= 10; + } while (value != 0); + unsigned length = next - start; + + // Reverse in-place. + next--; + while (next > start) { + char swap = *next; + *next = *start; + *start = swap; + next--; + start++; + } + return length; +} + +void +appendUint(IOBufQueue& queue, size_t& len, uint64_t value) { + char buf[32]; + size_t encodedLen = u64toa(value, buf); + queue.append(buf, encodedLen); + len += encodedLen; +} + +#define appendLiteral(queue, len, str) (len) += (sizeof(str) - 1); \ + (queue).append(str, sizeof(str) - 1) + +void +appendString(IOBufQueue& queue, size_t& len, const string& str) { + queue.append(str.data(), str.length()); + len += str.length(); +} + +const std::pair kHTTPVersion10(1, 0); + +} // anonymous namespace + +namespace proxygen { + +http_parser_settings HTTP1xCodec::kParserSettings; + +HTTP1xCodec::HTTP1xCodec(TransportDirection direction, bool forceUpstream1_1) + : callback_(nullptr), + ingressTxnID_(0), + egressTxnID_(0), + currentIngressBuf_(nullptr), + headerParseState_(HeaderParseState::kParsingHeaderIdle), + transportDirection_(direction), + keepaliveRequested_(KeepaliveRequested::UNSET), + forceUpstream1_1_(forceUpstream1_1), + parserActive_(false), + pendingEOF_(false), + parserPaused_(false), + parserError_(false), + requestPending_(false), + responsePending_(false), + egressChunked_(false), + inChunk_(false), + lastChunkWritten_(false), + keepalive_(true), + disableKeepalivePending_(false), + connectRequest_(false), + headRequest_(false), + expectNoResponseBody_(false), + mayChunkEgress_(false), + is1xxResponse_(false), + inRecvLastChunk_(false), + ingressUpgrade_(false), + ingressUpgradeComplete_(false), + egressUpgrade_(false), + headersComplete_(false) { + switch (direction) { + case TransportDirection::DOWNSTREAM: + http_parser_init(&parser_, HTTP_REQUEST); + break; + case TransportDirection::UPSTREAM: + http_parser_init(&parser_, HTTP_RESPONSE); + break; + } + parser_.data = this; +} + +HTTP1xCodec::~HTTP1xCodec() { + // This code used to throw a parse error there were unterminated headers + // being parsed. None of the cases where this can happen relied on the parse + // error. +} + +HTTPCodec::StreamID +HTTP1xCodec::createStream() { + if (transportDirection_ == TransportDirection::DOWNSTREAM) { + return ++ingressTxnID_; + } else { + return ++egressTxnID_; + } +} + +void +HTTP1xCodec::setParserPaused(bool paused) { + if ((paused == parserPaused_) || parserError_) { + // If we're bailing early, we better be paused already + DCHECK(parserError_ || + (HTTP_PARSER_ERRNO(&parser_) == HPE_PAUSED) == paused); + return; + } + if (paused) { + if (HTTP_PARSER_ERRNO(&parser_) == HPE_OK) { + http_parser_pause(&parser_, 1); + } + } else { + http_parser_pause(&parser_, 0); + } + parserPaused_ = paused; +} + +size_t +HTTP1xCodec::onIngress(const IOBuf& buf) { + if (parserError_) { + return 0; + } else if (ingressUpgradeComplete_) { + callback_->onBody(ingressTxnID_, buf.clone()); + return buf.computeChainDataLength(); + } else { + // Callers responsibility to prevent calling onIngress from a callback + CHECK(!parserActive_); + parserActive_ = true; + currentIngressBuf_ = &buf; + size_t bytesParsed = http_parser_execute(&parser_, + &kParserSettings, + (const char*)buf.data(), + buf.length()); + // in case we parsed a section of the headers but we're not done parsing + // the headers we need to keep accounting of it for total header size + if (!headersComplete_) { + headerSize_.uncompressed += bytesParsed; + } + parserActive_ = false; + parserError_ = (HTTP_PARSER_ERRNO(&parser_) != HPE_OK) && + (HTTP_PARSER_ERRNO(&parser_) != HPE_PAUSED); + if (parserError_) { + onParserError(); + } + if (currentHeaderName_.empty() && !currentHeaderNameStringPiece_.empty()) { + // we currently are storing a chunk of header name via pointers in + // currentHeaderNameStringPiece_, but the currentIngressBuf_ is about to + // vanish and so we need to copy over that data to currentHeaderName_ + currentHeaderName_.assign(currentHeaderNameStringPiece_.begin(), + currentHeaderNameStringPiece_.size()); + } + currentIngressBuf_ = nullptr; + if (pendingEOF_) { + onIngressEOF(); + pendingEOF_ = false; + } + return bytesParsed; + } +} + +void +HTTP1xCodec::onIngressEOF() { + if (parserError_) { + return; + } + if (parserActive_) { + pendingEOF_ = true; + return; + } + if (ingressUpgradeComplete_) { + callback_->onMessageComplete(ingressTxnID_, false); + return; + } + parserActive_ = true; + if (http_parser_execute(&parser_, &kParserSettings, nullptr, 0) != 0) { + parserError_ = true; + } else { + parserError_ = (HTTP_PARSER_ERRNO(&parser_) != HPE_OK) && + (HTTP_PARSER_ERRNO(&parser_) != HPE_PAUSED); + } + parserActive_ = false; + if (parserError_) { + onParserError(); + } +} + +void +HTTP1xCodec::onParserError(const char* what) { + inRecvLastChunk_ = false; + http_errno parser_errno = HTTP_PARSER_ERRNO(&parser_); + HTTPException error(HTTPException::Direction::INGRESS, + what ? what : folly::to( + "Error parsing message: ", + http_errno_description(parser_errno) + )); + // generate a string of parsed headers so that we can pass it to callback + if (msg_) { + error.setPartialMsg(std::move(msg_)); + } + // store the ingress buffer + if (currentIngressBuf_) { + error.setCurrentIngressBuf(std::move(currentIngressBuf_->clone())); + } + if (transportDirection_ == TransportDirection::DOWNSTREAM && + egressTxnID_ < ingressTxnID_) { + error.setHttpStatusCode(400); + } // else we've already egressed a response for this txn, don't attempt a 400 + // See http_parser.h for what these error codes mean + if (parser_errno == HPE_INVALID_EOF_STATE) { + error.setProxygenError(kErrorEOF); + } else if (parser_errno == HPE_HEADER_OVERFLOW || + parser_errno == HPE_INVALID_CONSTANT || + (parser_errno >= HPE_INVALID_VERSION && + parser_errno <= HPE_HUGE_CONTENT_LENGTH)) { + error.setProxygenError(kErrorParseHeader); + } else if (parser_errno == HPE_INVALID_CHUNK_SIZE || + parser_errno == HPE_HUGE_CHUNK_SIZE) { + error.setProxygenError(kErrorParseBody); + } else { + error.setProxygenError(kErrorUnknown); + } + callback_->onError(ingressTxnID_, error); +} + +bool +HTTP1xCodec::isReusable() const { + return keepalive_ && !egressUpgrade_ && !ingressUpgrade_; +} + +bool +HTTP1xCodec::isBusy() const { + return requestPending_ || responsePending_; +} + +void +HTTP1xCodec::addDateHeader(IOBufQueue& writeBuf, size_t& len) { + appendLiteral(writeBuf, len, "Date: "); + appendString(writeBuf, len, HTTPMessage::formatDateHeader()); + appendLiteral(writeBuf, len, CRLF); +} + +void +HTTP1xCodec::generateHeader(IOBufQueue& writeBuf, + StreamID txn, + const HTTPMessage& msg, + StreamID assocStream, + HTTPHeaderSize* size) { + CHECK(assocStream == 0) << "HTTP does not support pushed transactions, " + "assocStream=" << assocStream; + if (keepalive_ && disableKeepalivePending_) { + keepalive_ = false; + } + const bool upstream = (transportDirection_ == TransportDirection::UPSTREAM); + const bool downstream = !upstream; + if (upstream) { + DCHECK(txn == egressTxnID_); + requestPending_ = true; + responsePending_ = true; + connectRequest_ = (msg.getMethod() == HTTPMethod::CONNECT); + headRequest_ = (msg.getMethod() == HTTPMethod::HEAD); + expectNoResponseBody_ = connectRequest_ || headRequest_; + } else { + // In HTTP, transactions must be egressed sequentially -- no out of order + // responses. So txn must be egressTxnID_ + 1. Furthermore, we shouldn't + // ever egress a response before we see a request, so txn can't + // be > ingressTxnID_ + if ((txn != egressTxnID_ + 1 && + !(txn == egressTxnID_ && is1xxResponse_)) || + (txn > ingressTxnID_)) { + LOG(DFATAL) << "Out of order, duplicate or premature HTTP response"; + } + if (!is1xxResponse_) { + ++egressTxnID_; + } + is1xxResponse_ = msg.is1xxResponse(); + + expectNoResponseBody_ = + connectRequest_ || headRequest_ || + RFC2616::responseBodyMustBeEmpty(msg.getStatusCode()); + } + + if (downstream) { + auto statusCode = msg.getStatusCode(); + if (connectRequest_ && (statusCode >= 200 && statusCode < 300)) { + // Set egress upgrade flag if we are sending a 200 response + // to a CONNECT request we received earlier. + egressUpgrade_ = true; + } else if (statusCode == 101) { + // Set the upgrade flags if we upgraded after the request from client. + ingressUpgrade_ = true; + egressUpgrade_ = true; + } + else if (connectRequest_ && ingressUpgrade_) { + // Disable upgrade when rejecting CONNECT request + ingressUpgrade_ = false; + + // This codec/session is no longer useful as we might have + // forwarded some data before receiving the 200. + keepalive_ = false; + } + } else { + if (connectRequest_) { + // Sending a CONNECT request to an upstream server + egressUpgrade_ = true; + } + } + + egressChunked_ = msg.getIsChunked() && !egressUpgrade_; + lastChunkWritten_ = false; + std::pair version = msg.getHTTPVersion(); + if (version > HTTPMessage::kHTTPVersion11) { + version = HTTPMessage::kHTTPVersion11; + } + + size_t len = 0; + switch (transportDirection_) { + case TransportDirection::DOWNSTREAM: + appendLiteral(writeBuf, len, "HTTP/"); + appendUint(writeBuf, len, version.first); + appendLiteral(writeBuf, len, "."); + appendUint(writeBuf, len, version.second); + appendLiteral(writeBuf, len, " "); + appendUint(writeBuf, len, msg.getStatusCode()); + appendLiteral(writeBuf, len, " "); + appendString(writeBuf, len, msg.getStatusMessage()); + break; + case TransportDirection::UPSTREAM: + if (forceUpstream1_1_ && version < HTTPMessage::kHTTPVersion11) { + version = HTTPMessage::kHTTPVersion11; + } + appendString(writeBuf, len, msg.getMethodString()); + appendLiteral(writeBuf, len, " "); + appendString(writeBuf, len, msg.getURL()); + appendLiteral(writeBuf, len, " HTTP/"); + appendUint(writeBuf, len, version.first); + appendLiteral(writeBuf, len, "."); + appendUint(writeBuf, len, version.second); + mayChunkEgress_ = (version.first == 1) && (version.second >= 1); + break; + } + if (keepalive_ && + (!msg.wantsKeepalive() || + version.first < 1 || + (downstream && version == HTTPMessage::kHTTPVersion10 && + keepaliveRequested_ != KeepaliveRequested::YES))) { + // Disable keepalive if + // - the message asked to turn it off + // - it's HTTP/0.9 + // - this is a response to a 1.0 request that didn't say keep-alive + keepalive_ = false; + } + egressChunked_ &= mayChunkEgress_; + appendLiteral(writeBuf, len, CRLF); + const string* deferredContentLength = nullptr; + bool hasTransferEncodingChunked = false; + bool hasUpgradeHeader = false; + bool hasDateHeader = false; + msg.getHeaders().forEachWithCode([&] (HTTPHeaderCode code, + const string& header, + const string& value) { + if (code == HTTP_HEADER_CONTENT_LENGTH) { + // Write the Content-Length last (t1071703) + deferredContentLength = &value; + return; // continue + } else if (code == HTTP_HEADER_CONNECTION) { + // TODO: add support for the case where "close" is part of + // a comma-separated list of values + static const string kClose = "close"; + if (caseInsensitiveEqual(value, kClose)) { + keepalive_ = false; + } + // We'll generate a new Connection header based on the keepalive_ state + return; + } else if (!hasTransferEncodingChunked && + code == HTTP_HEADER_TRANSFER_ENCODING) { + static const string kChunked = "chunked"; + if (!caseInsensitiveEqual(value, kChunked)) { + return; + } + hasTransferEncodingChunked = true; + if (!mayChunkEgress_) { + return; + } + } else if (code == HTTP_HEADER_UPGRADE) { + hasUpgradeHeader = true; + } else if (!hasDateHeader && code == HTTP_HEADER_DATE) { + hasDateHeader = true; + } + size_t lineLen = header.length() + value.length() + 4; // 4 for ": " + CRLF + auto writable = writeBuf.preallocate(lineLen, + std::max(lineLen, size_t(2000))); + char* dst = (char*)writable.first; + memcpy(dst, header.data(), header.length()); + dst += header.length(); + *dst++ = ':'; + *dst++ = ' '; + memcpy(dst, value.data(), value.length()); + dst += value.length(); + *dst++ = '\r'; + *dst = '\n'; + DCHECK(size_t(++dst - (char*)writable.first) == lineLen); + writeBuf.postallocate(lineLen); + len += lineLen; + }); + bool bodyCheck = + (downstream && keepalive_ && !expectNoResponseBody_ && !egressUpgrade_) || + // auto chunk POSTs and any request that came to us chunked + (upstream && ((msg.getMethod() == HTTPMethod::POST) || egressChunked_)); + // TODO: 400 a 1.0 POST with no content-length + // clear egressChunked_ if the header wasn't actually set + egressChunked_ &= hasTransferEncodingChunked; + if (bodyCheck && !egressChunked_ && !deferredContentLength) { + // On a connection that would otherwise be eligible for keep-alive, + // we're being asked to send a response message with no Content-Length, + // no chunked encoding, and no special circumstances that would eliminate + // the need for a response body. If the client supports chunking, turn + // on chunked encoding now. Otherwise, turn off keepalives on this + // connection. + if (!hasTransferEncodingChunked && mayChunkEgress_) { + appendLiteral(writeBuf, len, "Transfer-Encoding: chunked\r\n"); + egressChunked_ = true; + } else { + keepalive_ = false; + } + } + if (downstream && !hasDateHeader) { + addDateHeader(writeBuf, len); + } + if (!is1xxResponse_ || upstream || hasUpgradeHeader) { + appendLiteral(writeBuf, len, "Connection: "); + if (hasUpgradeHeader) { + // Upgrade header needs to have 'upgrade' keyword as the connection type + appendLiteral(writeBuf, len, "upgrade\r\n"); + } else if (keepalive_) { + appendLiteral(writeBuf, len, "keep-alive\r\n"); + } else { + appendLiteral(writeBuf, len, "close\r\n"); + } + } + if (deferredContentLength) { + appendLiteral(writeBuf, len, "Content-Length: "); + appendString(writeBuf, len, *deferredContentLength); + appendString(writeBuf, len, CRLF); + } + appendLiteral(writeBuf, len, CRLF); + + if (size) { + size->compressed = 0; + size->uncompressed = len; + } +} + +size_t +HTTP1xCodec::generateBody(IOBufQueue& writeBuf, + StreamID txn, + unique_ptr chain, + bool eom) { + DCHECK(txn == egressTxnID_); + if (!chain) { + return 0; + } + size_t buflen = chain->computeChainDataLength(); + if (buflen == 0) { + return buflen; + } + size_t totLen = buflen; + + if (egressChunked_ && !inChunk_) { + char chunkLenBuf[32]; + int rc = snprintf(chunkLenBuf, sizeof(chunkLenBuf), "%zx\r\n", buflen); + CHECK(rc > 0); + CHECK(size_t(rc) < sizeof(chunkLenBuf)); + + writeBuf.append(chunkLenBuf, rc); + totLen += rc; + + writeBuf.append(std::move(chain)); + writeBuf.append("\r\n", 2); + totLen += 2; + } else { + writeBuf.append(std::move(chain)); + } + if (eom) { + totLen += generateEOM(writeBuf, txn); + } + + return totLen; +} + +size_t HTTP1xCodec::generateChunkHeader(IOBufQueue& writeBuf, + StreamID txn, + size_t length) { + // TODO: Format directly into the IOBuf, rather than copying after the fact. + // IOBufQueue::append() currently forces us to copy. + + CHECK(length) << "use sendEOM to terminate the message using the " + << "standard zero-length chunk. Don't " + << "send zero-length chunks using this API."; + if (egressChunked_) { + CHECK(!inChunk_); + inChunk_ = true; + char chunkLenBuf[32]; + int rc = snprintf(chunkLenBuf, sizeof(chunkLenBuf), "%zx\r\n", length); + CHECK(rc > 0); + CHECK(size_t(rc) < sizeof(chunkLenBuf)); + + writeBuf.append(chunkLenBuf, rc); + return rc; + } + + return 0; +} + +size_t HTTP1xCodec::generateChunkTerminator(IOBufQueue& writeBuf, + StreamID txn) { + if (egressChunked_ && inChunk_) { + inChunk_ = false; + writeBuf.append("\r\n", 2); + return 2; + } + + return 0; +} + +size_t +HTTP1xCodec::generateTrailers(IOBufQueue& writeBuf, + StreamID txn, + const HTTPHeaders& trailers) { + DCHECK(txn == egressTxnID_); + size_t len = 0; + if (egressChunked_) { + CHECK(!inChunk_); + appendLiteral(writeBuf, len, "0\r\n"); + lastChunkWritten_ = true; + trailers.forEach([&] (const string& trailer, const string& value) { + appendString(writeBuf, len, trailer); + appendLiteral(writeBuf, len, ": "); + appendString(writeBuf, len, value); + appendLiteral(writeBuf, len, CRLF); + }); + } + return len; +} + +size_t HTTP1xCodec::generateEOM(IOBufQueue& writeBuf, StreamID txn) { + DCHECK(txn == egressTxnID_); + size_t len = 0; + if (egressChunked_) { + CHECK(!inChunk_); + if (!lastChunkWritten_) { + lastChunkWritten_ = true; + appendLiteral(writeBuf, len, "0\r\n"); + } + appendLiteral(writeBuf, len, CRLF); + } + switch (transportDirection_) { + case TransportDirection::DOWNSTREAM: + responsePending_ = false; + break; + case TransportDirection::UPSTREAM: + requestPending_ = false; + break; + } + return len; +} + +size_t HTTP1xCodec::generateRstStream(IOBufQueue& writeBuf, + StreamID txn, + ErrorCode statusCode) { + // statusCode ignored for HTTP/1.1 + // We won't be able to send anything else on the transport after this. + disableKeepalivePending_ = true; + return 0; +} + +size_t HTTP1xCodec::generateGoaway(IOBufQueue& writeBuf, + StreamID lastStream, + ErrorCode statusCode) { + // statusCode ignored for HTTP/1.1 + // We won't be able to send anything else on the transport after this. + disableKeepalivePending_ = true; + return 0; +} + +int +HTTP1xCodec::onMessageBegin() { + headersComplete_ = false; + headerSize_.uncompressed = 0; + headerParseState_ = HeaderParseState::kParsingHeaderStart; + msg_.reset(new HTTPMessage()); + trailers_.reset(); + if (transportDirection_ == TransportDirection::DOWNSTREAM) { + requestPending_ = true; + responsePending_ = true; + } + // If there was a 1xx on this connection, don't increment the ingress txn id + if (transportDirection_ == TransportDirection::DOWNSTREAM || + !is1xxResponse_) { + ++ingressTxnID_; + } + is1xxResponse_ = false; + callback_->onMessageBegin(ingressTxnID_, msg_.get()); + return 0; +} + +int +HTTP1xCodec::onURL(const char* buf, size_t len) { + url_.append(buf, len); + return 0; +} + +int +HTTP1xCodec::onReason(const char* buf, size_t len) { + reason_.append(buf, len); + return 0; +} + +void HTTP1xCodec::pushHeaderNameAndValue(HTTPHeaders& hdrs) { + if (LIKELY(currentHeaderName_.empty())) { + hdrs.addFromCodec(currentHeaderNameStringPiece_.begin(), + currentHeaderNameStringPiece_.size(), + std::move(currentHeaderValue_)); + } else { + hdrs.add(currentHeaderName_, + std::move(currentHeaderValue_)); + currentHeaderName_.clear(); + } + currentHeaderNameStringPiece_.clear(); + currentHeaderValue_.clear(); +} + +int +HTTP1xCodec::onHeaderField(const char* buf, size_t len) { + if (headerParseState_ == HeaderParseState::kParsingHeaderValue) { + pushHeaderNameAndValue(msg_->getHeaders()); + } else if (headerParseState_ == HeaderParseState::kParsingTrailerValue) { + if (!trailers_) { + trailers_.reset(new HTTPHeaders()); + } + pushHeaderNameAndValue(*trailers_); + } + + if (isParsingHeaderOrTrailerName()) { + + // we're already parsing a header name + if (currentHeaderName_.empty()) { + // but we've been keeping it in currentHeaderNameStringPiece_ until now + if (currentHeaderNameStringPiece_.end() == buf) { + // the header name we are currently reading is contiguous in memory, + // and so we just adjust the right end of our StringPiece; + // this is likely because onIngress() hasn't been called since we got + // the previous chunk (otherwise currentHeaderName_ would be nonempty) + currentHeaderNameStringPiece_.advance(len); + } else { + // this is just for safety - if for any reason there is a discontinuity + // even though we are during the same onIngress() call, + // we fall back to currentHeaderName_ + currentHeaderName_.assign(currentHeaderNameStringPiece_.begin(), + currentHeaderNameStringPiece_.size()); + currentHeaderName_.append(buf, len); + } + } else { + // we had already fallen back to currentHeaderName_ before + currentHeaderName_.append(buf, len); + } + + } else { + // we're not yet parsing a header name - this is the first chunk + // (typically, there is only one) + currentHeaderNameStringPiece_.reset(buf, len); + + if (headerParseState_ >= HeaderParseState::kParsingHeadersComplete) { + headerParseState_ = HeaderParseState::kParsingTrailerName; + } else { + headerParseState_ = HeaderParseState::kParsingHeaderName; + } + } + return 0; +} + +int +HTTP1xCodec::onHeaderValue(const char* buf, size_t len) { + if (isParsingHeaders()) { + headerParseState_ = HeaderParseState::kParsingHeaderValue; + } else { + headerParseState_ = HeaderParseState::kParsingTrailerValue; + } + currentHeaderValue_.append(buf, len); + return 0; +} + +int +HTTP1xCodec::onHeadersComplete(size_t len) { + if (headerParseState_ == HeaderParseState::kParsingHeaderValue) { + pushHeaderNameAndValue(msg_->getHeaders()); + } + + // Update the HTTPMessage with the values parsed from the header + msg_->setHTTPVersion(parser_.http_major, parser_.http_minor); + msg_->setIsChunked((parser_.flags & F_CHUNKED)); + + if (transportDirection_ == TransportDirection::DOWNSTREAM) { + // Set the method type + msg_->setMethod(http_method_str(static_cast(parser_.method))); + + connectRequest_ = (msg_->getMethod() == HTTPMethod::CONNECT); + + // If this is a headers-only request, we shouldn't send + // an entity-body in the response. + headRequest_ = (msg_->getMethod() == HTTPMethod::HEAD); + + ParseURL parseUrl = msg_->setURL(std::move(url_)); + url_.clear(); + + if (parseUrl.hasHost()) { + // RFC 2616 5.2.1 states "If Request-URI is an absoluteURI, the host + // is part of the Request-URI. Any Host header field value in the + // request MUST be ignored." + auto hostAndPort = parseUrl.hostAndPort(); + VLOG(4) << "Adding inferred host header: " << hostAndPort; + msg_->getHeaders().set(HTTP_HEADER_HOST, hostAndPort); + } + + // If the client sent us an HTTP/1.x with x >= 1, we may send + // chunked responses. + mayChunkEgress_ = ((parser_.http_major == 1) && (parser_.http_minor >= 1)); + } else { + msg_->setStatusCode(parser_.status_code); + msg_->setStatusMessage(std::move(reason_)); + reason_.clear(); + } + + if (transportDirection_ == TransportDirection::UPSTREAM) { + if (connectRequest_ && + (parser_.status_code >= 200 && parser_.status_code < 300)) { + // Enable upgrade if this is a 200 response to a CONNECT + // request we sent earlier + ingressUpgrade_ = true; + } else if (parser_.status_code == 101) { + // Set the upgrade flags if the server has upgraded. + ingressUpgrade_ = true; + egressUpgrade_ = true; + } + } + else { + if (connectRequest_) { + // Enable upgrade by default for the CONNECT requests. + // If we locally reject CONNECT, we will disable this flag while + // sending the reject response. If we forward the req to upstream proxy, + // we will start forwarding data to the proxy without waiting for + // the response from the proxy server. + ingressUpgrade_ = true; + } + } + msg_->setIsUpgraded(ingressUpgrade_); + + headerParseState_ = HeaderParseState::kParsingHeadersComplete; + bool msgKeepalive = msg_->computeKeepalive(); + if (!msgKeepalive) { + keepalive_ = false; + } + if (transportDirection_ == TransportDirection::DOWNSTREAM) { + // Remember whether this was an HTTP 1.0 request with keepalive enabled + if (msgKeepalive && msg_->isHTTP1_0() && + (keepaliveRequested_ == KeepaliveRequested::UNSET || + keepaliveRequested_ == KeepaliveRequested::YES)) { + keepaliveRequested_ = KeepaliveRequested::YES; + } else { + keepaliveRequested_ = KeepaliveRequested::NO; + } + } + + // Determine whether the HTTP parser should ignore any headers + // that indicate the presence of a message body. This is needed, + // for example, if the message is a response to a request with + // method==HEAD. + bool ignoreBody; + if (transportDirection_ == TransportDirection::DOWNSTREAM) { + ignoreBody = false; + } else { + is1xxResponse_ = msg_->is1xxResponse(); + if (expectNoResponseBody_) { + ignoreBody = true; + } else { + ignoreBody = RFC2616::responseBodyMustBeEmpty(msg_->getStatusCode()); + } + } + + headersComplete_ = true; + headerSize_.uncompressed += len; + msg_->setIngressHeaderSize(headerSize_); + + callback_->onHeadersComplete(ingressTxnID_, std::move(msg_)); + + // 1 is a magic value that tells the http_parser not to expect a + // message body even if the message header implied the presence + // of one (e.g., via a Content-Length) + return (ignoreBody) ? 1 : 0; +} + +int +HTTP1xCodec::onBody(const char* buf, size_t len) { + DCHECK(!isParsingHeaders()); + DCHECK(!inRecvLastChunk_); + CHECK(currentIngressBuf_ != nullptr); + const char* dataStart = (const char*)currentIngressBuf_->data(); + const char* dataEnd = dataStart + currentIngressBuf_->length(); + DCHECK_GE(buf, dataStart); + DCHECK_LE(buf + len, dataEnd); + unique_ptr clone(currentIngressBuf_->clone()); + clone->trimStart(buf - dataStart); + clone->trimEnd(dataEnd - (buf + len)); + callback_->onBody(ingressTxnID_, std::move(clone)); + return 0; +} + +int HTTP1xCodec::onChunkHeader(size_t len) { + if (len > 0) { + callback_->onChunkHeader(ingressTxnID_, len); + } else { + VLOG(5) << "Suppressed onChunkHeader callback for final zero length " + << "chunk"; + inRecvLastChunk_ = true; + } + return 0; +} + +int HTTP1xCodec::onChunkComplete() { + if (inRecvLastChunk_) { + inRecvLastChunk_ = false; + } else { + callback_->onChunkComplete(ingressTxnID_); + } + return 0; +} + +int HTTP1xCodec::onMessageComplete() { + DCHECK(!isParsingHeaders()); + DCHECK(!inRecvLastChunk_); + if (headerParseState_ == HeaderParseState::kParsingTrailerValue) { + if (!trailers_) { + trailers_.reset(new HTTPHeaders()); + } + pushHeaderNameAndValue(*trailers_); + } + + headerParseState_ = HeaderParseState::kParsingHeaderIdle; + if (trailers_) { + callback_->onTrailersComplete(ingressTxnID_, std::move(trailers_)); + } + + switch (transportDirection_) { + case TransportDirection::DOWNSTREAM: + requestPending_ = false; + break; + case TransportDirection::UPSTREAM: + responsePending_ = is1xxResponse_; + } + + callback_->onMessageComplete(ingressTxnID_, ingressUpgrade_); + + if (ingressUpgrade_) { + ingressUpgradeComplete_ = true; + // If upgrade is complete, any pending data should not be parsed. + // It must be forwarded directly to the handler. + setParserPaused(true); + } + + return 0; +} + +int +HTTP1xCodec::onMessageBeginCB(http_parser* parser) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onMessageBegin(); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 1; + } +} + +int +HTTP1xCodec::onUrlCB(http_parser* parser, const char* buf, size_t len) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onURL(buf, len); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 1; + } +} + +int +HTTP1xCodec::onReasonCB(http_parser* parser, const char* buf, size_t len) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onReason(buf, len); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 1; + } +} + +int +HTTP1xCodec::onHeaderFieldCB(http_parser* parser, const char* buf, size_t len) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onHeaderField(buf, len); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 1; + } +} + +int +HTTP1xCodec::onHeaderValueCB(http_parser* parser, const char* buf, size_t len) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onHeaderValue(buf, len); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 1; + } +} + +int +HTTP1xCodec::onHeadersCompleteCB(http_parser* parser, + const char* buf, size_t len) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onHeadersComplete(len); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 3; + } +} + +int +HTTP1xCodec::onBodyCB(http_parser* parser, const char* buf, size_t len) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onBody(buf, len); + } catch (const std::exception& ex) { + // Note: http_parser appears to completely ignore the return value from the + // on_body() callback. There seems to be no way to abort parsing after an + // error in on_body(). + // + // We handle this by checking if error_ is set after each call to + // http_parser_execute(). + codec->onParserError(ex.what()); + return 1; + } +} + +int HTTP1xCodec::onChunkHeaderCB(http_parser* parser) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onChunkHeader(parser->content_length); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 1; + } +} + +int HTTP1xCodec::onChunkCompleteCB(http_parser* parser) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onChunkComplete(); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 1; + } +} + +int +HTTP1xCodec::onMessageCompleteCB(http_parser* parser) { + HTTP1xCodec* codec = static_cast(parser->data); + DCHECK(codec != nullptr); + DCHECK_EQ(&codec->parser_, parser); + + try { + return codec->onMessageComplete(); + } catch (const std::exception& ex) { + codec->onParserError(ex.what()); + return 1; + } +} + +void +HTTP1xCodec::initParserSettings() { + kParserSettings.on_message_begin = HTTP1xCodec::onMessageBeginCB; + kParserSettings.on_url = HTTP1xCodec::onUrlCB; + kParserSettings.on_header_field = HTTP1xCodec::onHeaderFieldCB; + kParserSettings.on_header_value = HTTP1xCodec::onHeaderValueCB; + kParserSettings.on_headers_complete = HTTP1xCodec::onHeadersCompleteCB; + kParserSettings.on_body = HTTP1xCodec::onBodyCB; + kParserSettings.on_message_complete = HTTP1xCodec::onMessageCompleteCB; + kParserSettings.on_reason = HTTP1xCodec::onReasonCB; + kParserSettings.on_chunk_header = HTTP1xCodec::onChunkHeaderCB; + kParserSettings.on_chunk_complete = HTTP1xCodec::onChunkCompleteCB; +} + +bool HTTP1xCodec::supportsNextProtocol(const std::string& npn) { + return npn.length() == 8 && (npn == "http/1.0" || npn == "http/1.1"); +} + +} // proxygen diff --git a/proxygen/lib/http/codec/HTTP1xCodec.h b/proxygen/lib/http/codec/HTTP1xCodec.h new file mode 100644 index 0000000000..b4fac26628 --- /dev/null +++ b/proxygen/lib/http/codec/HTTP1xCodec.h @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/external/http_parser/http_parser.h" +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/http/codec/HTTPCodec.h" +#include "proxygen/lib/http/codec/TransportDirection.h" + +#include + +namespace proxygen { + +struct HTTPHeaderSize; + +class HTTP1xCodec : public HTTPCodec { + public: + explicit HTTP1xCodec(TransportDirection direction, + bool forceUpstream1_1 = false); + ~HTTP1xCodec() override; + + // HTTPCodec API + CodecProtocol getProtocol() const override { + return CodecProtocol::HTTP_1_1; + } + TransportDirection getTransportDirection() const override { + return transportDirection_; + } + StreamID createStream() override; + void setCallback(Callback* callback) override { callback_ = callback; } + bool isBusy() const override; + void setParserPaused(bool paused) override; + size_t onIngress(const folly::IOBuf& buf) override; + void onIngressEOF() override; + bool isReusable() const override; + bool isWaitingToDrain() const override { return false; } + // If the session has been upgraded we will send EOF (or RST if needed) + // on egress complete + bool closeOnEgressComplete() const override { return egressUpgrade_; } + bool supportsParallelRequests() const override { return false; } + bool supportsPushTransactions() const override { return false; } + void generateHeader(folly::IOBufQueue& writeBuf, + StreamID txn, + const HTTPMessage& msg, + StreamID assocStream = 0, + HTTPHeaderSize* size = nullptr) override; + size_t generateBody(folly::IOBufQueue& writeBuf, + StreamID txn, + std::unique_ptr chain, + bool eom) override; + size_t generateChunkHeader(folly::IOBufQueue& writeBuf, + StreamID txn, + size_t length) override; + size_t generateChunkTerminator(folly::IOBufQueue& writeBuf, + StreamID txn) override; + size_t generateTrailers(folly::IOBufQueue& writeBuf, + StreamID txn, + const HTTPHeaders& trailers) override; + size_t generateEOM(folly::IOBufQueue& writeBuf, + StreamID txn) override; + size_t generateRstStream(folly::IOBufQueue& writeBuf, + StreamID txn, + ErrorCode statusCode) override; + size_t generateGoaway(folly::IOBufQueue& writeBuf, + StreamID lastStream, + ErrorCode statusCode) override; + + /** + * @returns true if the codec supports the given NPN protocol. + */ + static bool supportsNextProtocol(const std::string& npn); + + private: + /** Simple state model used to track the parsing of HTTP headers */ + enum class HeaderParseState : uint8_t { + kParsingHeaderIdle, + kParsingHeaderStart, + kParsingHeaderName, + kParsingHeaderValue, + kParsingHeadersComplete, + kParsingTrailerName, + kParsingTrailerValue + }; + + /** Used to keep track of whether a client requested keep-alive. This is + * only useful to support HTTP 1.0 keep-alive for a downstream connection + * where keep-alive is disabled unless the client requested it. */ + enum class KeepaliveRequested : uint8_t { + UNSET, + YES, // incoming message requested keepalive + NO, // incoming message disabled keepalive + }; + + void addDateHeader(folly::IOBufQueue& writeBuf, size_t& len); + + /** Check whether we're currently parsing ingress message headers */ + bool isParsingHeaders() const { + return (headerParseState_ > HeaderParseState::kParsingHeaderIdle) && + (headerParseState_ < HeaderParseState::kParsingHeadersComplete); + } + + /** Check whether we're currently parsing ingress header-or-trailer name */ + bool isParsingHeaderOrTrailerName() const { + return (headerParseState_ == HeaderParseState::kParsingHeaderName) || + (headerParseState_ == HeaderParseState::kParsingTrailerName); + } + + /** Invoked when a parsing error occurs. It will send an exception to + the callback object to report the error and do any other cleanup + needed. It optionally takes a message to pass to the generated + HTTPException passed to callback_. */ + void onParserError(const char* what = nullptr); + + /** Push out header name-value pair to hdrs and clear currentHeader*_ */ + void pushHeaderNameAndValue(HTTPHeaders& hdrs); + + // Parser callbacks + int onMessageBegin(); + int onURL(const char* buf, size_t len); + int onReason(const char* buf, size_t len); + int onHeaderField(const char* buf, size_t len); + int onHeaderValue(const char* buf, size_t len); + int onHeadersComplete(size_t len); + int onBody(const char* buf, size_t len); + int onChunkHeader(size_t len); + int onChunkComplete(); + int onMessageComplete(); + + HTTPCodec::Callback* callback_; + StreamID ingressTxnID_; + StreamID egressTxnID_; + http_parser parser_; + const folly::IOBuf* currentIngressBuf_; + std::unique_ptr msg_; + std::unique_ptr trailers_; + std::string currentHeaderName_; + folly::StringPiece currentHeaderNameStringPiece_; + std::string currentHeaderValue_; + std::string url_; + std::string reason_; + HTTPHeaderSize headerSize_; + HeaderParseState headerParseState_; + TransportDirection transportDirection_; + KeepaliveRequested keepaliveRequested_; // only used in DOWNSTREAM mode + bool forceUpstream1_1_:1; // Use HTTP/1.1 upstream even if msg is 1.0 + bool parserActive_:1; + bool pendingEOF_:1; + bool parserPaused_:1; + bool parserError_:1; + bool requestPending_:1; + bool responsePending_:1; + bool egressChunked_:1; + bool inChunk_:1; + bool lastChunkWritten_:1; + bool keepalive_:1; + bool disableKeepalivePending_:1; + // TODO: replace the 2 booleans below with an enum "request method" + bool connectRequest_:1; + bool headRequest_:1; + bool expectNoResponseBody_:1; + bool mayChunkEgress_:1; + bool is1xxResponse_:1; + bool inRecvLastChunk_:1; + bool ingressUpgrade_:1; + bool ingressUpgradeComplete_:1; + bool egressUpgrade_:1; + bool headersComplete_:1; + + // C-callable wrappers for the http_parser callbacks + static int onMessageBeginCB(http_parser* parser); + static int onPathCB(http_parser* parser, const char* buf, size_t len); + static int onQueryStringCB(http_parser* parser, const char* buf, size_t len); + static int onUrlCB(http_parser* parser, const char* buf, size_t len); + static int onReasonCB(http_parser* parser, const char* buf, size_t len); + static int onHeaderFieldCB(http_parser* parser, const char* buf, size_t len); + static int onHeaderValueCB(http_parser* parser, const char* buf, size_t len); + static int onHeadersCompleteCB(http_parser* parser, + const char* buf, size_t len); + static int onBodyCB(http_parser* parser, const char* buf, size_t len); + static int onChunkHeaderCB(http_parser* parser); + static int onChunkCompleteCB(http_parser* parser); + static int onMessageCompleteCB(http_parser* parser); + + static void initParserSettings() __attribute__ ((__constructor__)); + + static http_parser_settings kParserSettings; +}; + +} // proxygen diff --git a/proxygen/lib/http/codec/HTTPChecks.cpp b/proxygen/lib/http/codec/HTTPChecks.cpp new file mode 100644 index 0000000000..cfba2f6426 --- /dev/null +++ b/proxygen/lib/http/codec/HTTPChecks.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/HTTPChecks.h" + +#include "proxygen/lib/http/RFC2616.h" + +namespace proxygen { + +void HTTPChecks::onHeadersComplete(StreamID stream, + std::unique_ptr msg) { + + if (msg->isRequest() && (RFC2616::isRequestBodyAllowed(msg->getMethod()) + == RFC2616::BodyAllowed::NOT_ALLOWED) && + RFC2616::bodyImplied(msg->getHeaders())) { + HTTPException ex(HTTPException::Direction::INGRESS); + ex.setProxygenError(kErrorParseHeader); + // setting the status code means that the error is at the HTTP layer and + // that parsing succeeded. + ex.setHttpStatusCode(400); + callback_->onError(stream, ex, true); + return; + } + + callback_->onHeadersComplete(stream, std::move(msg)); +} + +void HTTPChecks::generateHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPMessage& msg, + StreamID assocStream, + HTTPHeaderSize* sizeOut) { + if (msg.isRequest() && RFC2616::bodyImplied(msg.getHeaders())) { + CHECK(RFC2616::isRequestBodyAllowed(msg.getMethod()) != + RFC2616::BodyAllowed::NOT_ALLOWED); + // We could also add a "strict" mode that disallows sending body on GET + // requests here too. + } + + call_->generateHeader(writeBuf, stream, msg, assocStream, sizeOut); +} + +} diff --git a/proxygen/lib/http/codec/HTTPChecks.h b/proxygen/lib/http/codec/HTTPChecks.h new file mode 100644 index 0000000000..bfef66762e --- /dev/null +++ b/proxygen/lib/http/codec/HTTPChecks.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/HTTPCodecFilter.h" + +namespace proxygen { + +/** + * This class enforces certain higher-level HTTP semantics. It does not enforce + * conditions that require state to decide. That is, this class is stateless and + * only examines the calls and callbacks that go through it. + */ + +class HTTPChecks: public PassThroughHTTPCodecFilter { + public: + // HTTPCodec::Callback methods + + void onHeadersComplete(StreamID stream, + std::unique_ptr msg) override; + + // HTTPCodec methods + + void generateHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPMessage& msg, + StreamID assocStream, + HTTPHeaderSize* sizeOut) override; +}; + +} diff --git a/proxygen/lib/http/codec/HTTPCodec.h b/proxygen/lib/http/codec/HTTPCodec.h new file mode 100644 index 0000000000..44e7db5bdf --- /dev/null +++ b/proxygen/lib/http/codec/HTTPCodec.h @@ -0,0 +1,468 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPException.h" +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/codec/CodecProtocol.h" +#include "proxygen/lib/http/codec/ErrorCode.h" +#include "proxygen/lib/http/codec/HTTPSettings.h" +#include "proxygen/lib/http/codec/TransportDirection.h" +#include "proxygen/lib/http/codec/compress/HeaderCodec.h" + +#include + +namespace proxygen { + +class HTTPHeaders; +class HTTPMessage; +class HTTPTransactionHandler; +class HTTPErrorPage; + +/** + * Interface for a parser&generator that can translate between an internal + * representation of an HTTP request and a wire format. The details of the + * wire format (e.g., HTTP/1.x encoding vs. SPDY encoding) are left for + * subclasses to implement. + */ +class HTTPCodec { + public: + + /** + * Key that uniquely identifies a request/response pair within + * (and only within) the scope of the codec. Code outside the + * codec should regard the StreamID as an opaque data + * structure; different subclasses of HTTPCodec are likely to + * use different conventions for generating StreamID values. + * + * A value of zero indicates an uninitialized/unknown/unspecified + * StreamID. + */ + typedef uint32_t StreamID; + + static const StreamID NoStream{0}; + + /** + * Callback interface that users of HTTPCodec must implement + */ + class Callback { + public: + /** + * Called when a new message is seen while parsing the ingress + * @param stream The stream ID + * @param msg A newly allocated HTTPMessage + */ + virtual void onMessageBegin(StreamID stream, HTTPMessage* msg) = 0; + + /** + * Called when a new push message is seen while parsing the ingress. + * + * @param stream The stream ID + * @param assocStream The stream ID of the associated stream, + * which can never be 0 + * @param msg A newly allocated HTTPMessage + */ + virtual void onPushMessageBegin(StreamID stream, + StreamID assocStream, + HTTPMessage* msg) {} + + /** + * Called when all the headers of an ingress message have been parsed + * @param stream The stream ID + * @param msg The message + * @param size Size of the ingress header + */ + virtual void onHeadersComplete(StreamID stream, + std::unique_ptr msg) = 0; + + /** + * Called for each block of message body data + * @param stream The stream ID + * @param chain One or more buffers of body data. The codec will + * remove any protocol framing, such as HTTP/1.1 chunk + * headers, from the buffers before calling this function. + */ + virtual void onBody(StreamID stream, + std::unique_ptr chain) = 0; + + /** + * Called for each HTTP chunk header. + * + * onChunkHeader() will be called when the chunk header is received. As + * the chunk data arrives, it will be passed to the callback normally with + * onBody() calls. Note that the chunk data may arrive in multiple + * onBody() calls: it is not guaranteed to arrive in a single onBody() + * call. + * + * After the chunk data has been received and the terminating CRLF has been + * received, onChunkComplete() will be called. + * + * @param stream The stream ID + * @param length The chunk length. + */ + virtual void onChunkHeader(StreamID stream, size_t length) {} + + /** + * Called when the terminating CRLF is received to end a chunk of HTTP body + * data. + * + * @param stream The stream ID + */ + virtual void onChunkComplete(StreamID stream) {} + + /** + * Called when all the trailers of an ingress message have been + * parsed, but only if the number of trailers is nonzero. + * @param stream The stream ID + * @param trailers The message trailers + */ + virtual void onTrailersComplete(StreamID stream, + std::unique_ptr trailers) = 0; + + /** + * Called at end of a message (including body and trailers, if applicable) + * @param stream The stream ID + * @param upgrade Whether the connection has been upgraded to another + * protocol. + */ + virtual void onMessageComplete(StreamID stream, bool upgrade) = 0; + + /** + * Called when a parsing or protocol error has occurred + * @param stream The stream ID + * @param error Description of the error + * @param newTxn true if onMessageBegin has not been called for txn + */ + virtual void onError(StreamID stream, + const HTTPException& error, + bool newTxn = false) = 0; + + /** + * Called when the peer has asked to shut down a stream + * immediately. + * @param stream The stream ID + * @param code The code the stream was aborted with + * @note Not applicable to all protocols. + */ + virtual void onAbort(StreamID stream, + ErrorCode code) {} + + /** + * Called upon receipt of a goaway. + * @param lastGoodStreamID Last successful stream created by the receiver + * @param code The code the connection was aborted with + * @note Not all protocols have goaways. SPDY does, but HTTP/1.1 doesn't. + */ + virtual void onGoaway(uint64_t lastGoodStreamID, + ErrorCode code) {} + + /** + * Called upon receipt of a ping request + * @param uniqueID Unique identifier for the ping + * @note Not all protocols have pings. SPDY does, but HTTP/1.1 doesn't. + */ + virtual void onPingRequest(uint64_t uniqueID) {} + + /** + * Called upon receipt of a ping reply + * @param uniqueID Unique identifier for the ping + * @note Not all protocols have pings. SPDY does, but HTTP/1.1 doesn't. + */ + virtual void onPingReply(uint64_t uniqueID) {} + + /** + * Called upon receipt of a window update, for protocols that support + * flow control. For instance spdy/3 and higher. + */ + virtual void onWindowUpdate(StreamID stream, uint32_t amount) {} + + /** + * Called upon receipt of a settings frame, for protocols that support + * settings. + * + * @param settings a list of settings that were sent in the settings frame + */ + virtual void onSettings(const SettingsList& settings) {} + + /** + * Called upon receipt of a settings frame with ACK set, for + * protocols that support settings ack. + */ + virtual void onSettingsAck() {} + + /** + * Return the number of open streams started by this codec callback. + * Parallel codecs with a maximum number of streams will invoke this + * to determine if a new stream exceeds the limit. + */ + virtual uint32_t numOutgoingStreams() const { return 0; } + + /** + * Return the number of open streams started by the remote side. + * Parallel codecs with a maximum number of streams will invoke this + * to determine if a new stream exceeds the limit. + */ + virtual uint32_t numIncomingStreams() const { return 0; } + + virtual ~Callback() {} + }; + + virtual ~HTTPCodec() {} + + /** + * Gets the session protocol currently used by the codec. This can be + * mapped to a string for logging and diagnostic use. + */ + virtual CodecProtocol getProtocol() const = 0; + + /** + * Get the transport direction of this codec: + * DOWNSTREAM if the codec receives requests from clients or + * UPSTREAM if the codec sends requests to servers. + */ + virtual TransportDirection getTransportDirection() const = 0; + + /** + * Returns true iff this codec supports per stream flow control + */ + virtual bool supportsStreamFlowControl() const { + return false; + } + + /** + * Returns true iff this codec supports session level flow control + */ + virtual bool supportsSessionFlowControl() const { + return false; + } + + /** + * Reserve a stream ID. + * @return A stream ID on success, or zero on error. + */ + virtual StreamID createStream() = 0; + + /** + * Set the callback to notify on ingress events + * @param callback The callback object + */ + virtual void setCallback(Callback* callback) = 0; + + /** + * Check whether the codec still has at least one HTTP + * stream to parse. + */ + virtual bool isBusy() const = 0; + + /** + * Pause or resume the ingress parser + * @param paused Whether the caller wants the parser to be paused + */ + virtual void setParserPaused(bool paused) = 0; + + /** + * Parse ingress data. + * @param buf A single IOBuf of data to parse + * @return Number of bytes consumed. + */ + virtual size_t onIngress(const folly::IOBuf& buf) = 0; + + /** + * Finish parsing when the ingress stream has ended. + */ + virtual void onIngressEOF() = 0; + + /** + * Check whether the codec can process new streams. Typically, + * an implementing subclass will return true when a new codec is + * created and false once it encounters a situation that would + * prevent reuse of the underlying transport (e.g., a "Connection: close" + * in HTTP/1.x). + * @note A return value of true means that the codec can process new + * connections at some reasonable point in the future; that may + * mean "immediately," for codecs that support pipelined or + * interleaved requests, or "upon completion of the current + * stream" for codecs that do not. + */ + virtual bool isReusable() const = 0; + + /** + * Returns true if this codec is in a state where it accepting new + * requests but will soon begin to reject new requests. For SPDY and + * HTTP/2, this is true when the first GOAWAY NO_ERROR is sent during + * graceful shutdown. + */ + virtual bool isWaitingToDrain() const = 0; + + /** + * Checks whether the socket needs to be closed when EOM is sent. This is used + * during CONNECT when EOF needs to be sent after upgrade to notify the server + */ + virtual bool closeOnEgressComplete() const = 0; + + /** + * Check whether the codec supports the processing of multiple + * requests in parallel. + */ + virtual bool supportsParallelRequests() const = 0; + + /** + * Check whether the codec supports pushing resources from server to + * client. + */ + virtual bool supportsPushTransactions() const = 0; + + /** + * Write an egress message header. For pushed streams, you must specify + * the assocStream. + * @retval size the size of the generated message, both the actual size + * and the size of the uncompressed data. + * @return None + */ + virtual void generateHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPMessage& msg, + StreamID assocStream = NoStream, + HTTPHeaderSize* size = nullptr) = 0; + + /** + * Write part of an egress message body. + * + * This will automatically generate a chunk header and footer around the data + * if necessary (e.g. you haven't manually sent a chunk header and the + * message should be chunked). + * + * @param eom implicitly generate the EOM marker with this body frame + * + * @return number of bytes written + */ + virtual size_t generateBody(folly::IOBufQueue& writeBuf, + StreamID stream, + std::unique_ptr chain, + bool eom) = 0; + + /** + * Write a body chunk header, if relevant. + */ + virtual size_t generateChunkHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + size_t length) = 0; + + /** + * Write a body chunk terminator, if relevant. + */ + virtual size_t generateChunkTerminator(folly::IOBufQueue& writeBuf, + StreamID stream) = 0; + + /** + * Write the message trailers + * @return number of bytes written + */ + virtual size_t generateTrailers(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPHeaders& trailers) = 0; + + /** + * Generate any protocol framing needed to finalize an egress + * message. This method must be called to complete a stream. + * + * @return number of bytes written + */ + virtual size_t generateEOM(folly::IOBufQueue& writeBuf, + StreamID stream) = 0; + + /** + * Generate any protocol framing needed to abort a connection. + * @return number of bytes written + */ + virtual size_t generateRstStream(folly::IOBufQueue& writeBuf, + StreamID stream, + ErrorCode code) = 0; + + /** + * Generate any protocol framing needed to abort a stream. + * @return number of bytes written + */ + virtual size_t generateGoaway(folly::IOBufQueue& writeBuf, + StreamID lastStream, + ErrorCode code) = 0; + + /** + * If the protocol supports it, generate a ping message that the other + * side should respond to. + */ + virtual size_t generatePingRequest(folly::IOBufQueue& writeBuf) { return 0; } + + /** + * Generate a reply to a ping message, if supported in the + * protocol implemented by the codec. + */ + virtual size_t generatePingReply(folly::IOBufQueue& writeBuf, + uint64_t uniqueID) { return 0; } + + /** + * Generate a settings message, if supported in the + * protocol implemented by the codec. + */ + virtual size_t generateSettings(folly::IOBufQueue& writeBuf) { + return 0; + } + + /** + * Generate a settings ack message, if supported in the + * protocol implemented by the codec. + */ + virtual size_t generateSettingsAck(folly::IOBufQueue& writeBuf) { + return 0; + } + + /* + * Generate a WINDOW_UPDATE message, if supported. The delta is the amount + * of ingress bytes we processed and freed from the current receive window. + * Returns the number of bytes written on the wire as a result of invoking + * this function. + */ + virtual size_t generateWindowUpdate(folly::IOBufQueue& writeBuf, + StreamID stream, + uint32_t delta) { + return 0; + } + + /* + * The below interfaces need only be implemented if the codec supports + * settings + */ + virtual HTTPSettings* getEgressSettings() { + return nullptr; + } + + virtual const HTTPSettings* getIngressSettings() const { + return nullptr; + } + + /** + * This interface is only implemented by SPDYCodec. This enables some + * HTTP/2 style behavior during graceful shutdown that allows 2 GOAWAYs + * to be sent during shutdown. + */ + virtual void enableDoubleGoawayDrain() {} + + /** + * set stats for the header codec, if the protocol supports header compression + */ + virtual void setHeaderCodecStats(HeaderCodec::Stats* stats) {} + + /** + * Get the identifier of the last stream started by the remote. + */ + virtual StreamID getLastIncomingStreamID() const { return NoStream; } +}; + +} diff --git a/proxygen/lib/http/codec/HTTPCodecFilter.cpp b/proxygen/lib/http/codec/HTTPCodecFilter.cpp new file mode 100644 index 0000000000..33f9a90fd2 --- /dev/null +++ b/proxygen/lib/http/codec/HTTPCodecFilter.cpp @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/HTTPCodecFilter.h" + +namespace proxygen { + +// HTTPCodec::Callback methods +void PassThroughHTTPCodecFilter::onMessageBegin(StreamID stream, + HTTPMessage* msg) { + callback_->onMessageBegin(stream, msg); +} + +void PassThroughHTTPCodecFilter::onPushMessageBegin(StreamID stream, + StreamID assocStream, + HTTPMessage* msg) { + callback_->onPushMessageBegin(stream, assocStream, msg); +} + +void PassThroughHTTPCodecFilter::onHeadersComplete( + StreamID stream, + std::unique_ptr msg) { + callback_->onHeadersComplete(stream, std::move(msg)); +} + +void PassThroughHTTPCodecFilter::onBody(StreamID stream, + std::unique_ptr chain) { + callback_->onBody(stream, std::move(chain)); +} + +void PassThroughHTTPCodecFilter::onChunkHeader(StreamID stream, + size_t length) { + callback_->onChunkHeader(stream, length); +} + +void PassThroughHTTPCodecFilter::onChunkComplete(StreamID stream) { + callback_->onChunkComplete(stream); +} + +void PassThroughHTTPCodecFilter::onTrailersComplete( + StreamID stream, + std::unique_ptr trailers) { + callback_->onTrailersComplete(stream, std::move(trailers)); +} + +void PassThroughHTTPCodecFilter::onMessageComplete(StreamID stream, + bool upgrade) { + callback_->onMessageComplete(stream, upgrade); +} + +void PassThroughHTTPCodecFilter::onError( + StreamID stream, + const HTTPException& error, + bool newStream) { + callback_->onError(stream, error, newStream); +} + +void PassThroughHTTPCodecFilter::onAbort(StreamID stream, + ErrorCode code) { + callback_->onAbort(stream, code); +} + +void PassThroughHTTPCodecFilter::onGoaway(uint64_t lastGoodStreamID, + ErrorCode code) { + callback_->onGoaway(lastGoodStreamID, code); +} + +void PassThroughHTTPCodecFilter::onPingRequest(uint64_t uniqueID) { + callback_->onPingRequest(uniqueID); +} + +void PassThroughHTTPCodecFilter::onPingReply(uint64_t uniqueID) { + callback_->onPingReply(uniqueID); +} + +void PassThroughHTTPCodecFilter::onWindowUpdate(StreamID stream, + uint32_t amount) { + callback_->onWindowUpdate(stream, amount); +} + +void PassThroughHTTPCodecFilter::onSettings(const SettingsList& settings) { + callback_->onSettings(settings); +} + +void PassThroughHTTPCodecFilter::onSettingsAck() { + callback_->onSettingsAck(); +} + +uint32_t PassThroughHTTPCodecFilter::numOutgoingStreams() const { + return callback_->numOutgoingStreams(); +} + +uint32_t PassThroughHTTPCodecFilter::numIncomingStreams() const { + return callback_->numIncomingStreams(); +} + +// PassThroughHTTPCodec methods +CodecProtocol PassThroughHTTPCodecFilter::getProtocol() const { + return call_->getProtocol(); +} + +TransportDirection PassThroughHTTPCodecFilter::getTransportDirection() const { + return call_->getTransportDirection(); +} + +bool PassThroughHTTPCodecFilter::supportsStreamFlowControl() const { + return call_->supportsStreamFlowControl(); +} + +bool PassThroughHTTPCodecFilter::supportsSessionFlowControl() const { + return call_->supportsSessionFlowControl(); +} + +HTTPCodec::StreamID PassThroughHTTPCodecFilter::createStream() { + return call_->createStream(); +} + +void PassThroughHTTPCodecFilter::setCallback(HTTPCodec::Callback* callback) { + setCallbackInternal(callback); +} + +bool PassThroughHTTPCodecFilter::isBusy() const { + return call_->isBusy(); +} + +void PassThroughHTTPCodecFilter::setParserPaused(bool paused) { + call_->setParserPaused(paused); +} + +size_t PassThroughHTTPCodecFilter::onIngress(const folly::IOBuf& buf) { + return call_->onIngress(buf); +} + +void PassThroughHTTPCodecFilter::onIngressEOF() { + call_->onIngressEOF(); +} + +bool PassThroughHTTPCodecFilter::isReusable() const { + return call_->isReusable(); +} + +bool PassThroughHTTPCodecFilter::isWaitingToDrain() const { + return call_->isWaitingToDrain(); +} + +bool PassThroughHTTPCodecFilter::closeOnEgressComplete() const { + return call_->closeOnEgressComplete(); +} + +bool PassThroughHTTPCodecFilter::supportsParallelRequests() const { + return call_->supportsParallelRequests(); +} + +bool PassThroughHTTPCodecFilter::supportsPushTransactions() const { + return call_->supportsPushTransactions(); +} + +void PassThroughHTTPCodecFilter::generateHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPMessage& msg, + StreamID assocStream, + HTTPHeaderSize* size) { + return call_->generateHeader(writeBuf, stream, msg, assocStream, size); +} + +size_t PassThroughHTTPCodecFilter::generateBody( + folly::IOBufQueue& writeBuf, + StreamID stream, + std::unique_ptr chain, + bool eom) { + return call_->generateBody(writeBuf, stream, std::move(chain), eom); +} + +size_t PassThroughHTTPCodecFilter::generateChunkHeader( + folly::IOBufQueue& writeBuf, + StreamID stream, + size_t length) { + return call_->generateChunkHeader(writeBuf, stream, length); +} + +size_t PassThroughHTTPCodecFilter::generateChunkTerminator( + folly::IOBufQueue& writeBuf, + StreamID stream) { + return call_->generateChunkTerminator(writeBuf, stream); +} + +size_t PassThroughHTTPCodecFilter::generateTrailers( + folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPHeaders& trailers) { + return call_->generateTrailers(writeBuf, stream, trailers); +} + +size_t PassThroughHTTPCodecFilter::generateEOM(folly::IOBufQueue& writeBuf, + StreamID stream) { + return call_->generateEOM(writeBuf, stream); +} + +size_t PassThroughHTTPCodecFilter::generateRstStream( + folly::IOBufQueue& writeBuf, + StreamID stream, + ErrorCode code) { + return call_->generateRstStream(writeBuf, stream, code); +} + +size_t PassThroughHTTPCodecFilter::generateGoaway( + folly::IOBufQueue& writeBuf, + StreamID lastStream, + ErrorCode statusCode) { + return call_->generateGoaway(writeBuf, lastStream, statusCode); +} + +size_t PassThroughHTTPCodecFilter::generatePingRequest( + folly::IOBufQueue& writeBuf) { + return call_->generatePingRequest(writeBuf); +} + +size_t PassThroughHTTPCodecFilter::generatePingReply( + folly::IOBufQueue& writeBuf, + uint64_t uniqueID) { + return call_->generatePingReply(writeBuf, uniqueID); +} + +size_t PassThroughHTTPCodecFilter::generateSettings(folly::IOBufQueue& buf) { + return call_->generateSettings(buf); +} + +size_t PassThroughHTTPCodecFilter::generateWindowUpdate( + folly::IOBufQueue& buf, + StreamID stream, + uint32_t delta) { + return call_->generateWindowUpdate(buf, stream, delta); +} + +HTTPSettings* PassThroughHTTPCodecFilter::getEgressSettings() { + return call_->getEgressSettings(); +} + +const HTTPSettings* PassThroughHTTPCodecFilter::getIngressSettings() const { + return call_->getIngressSettings(); +} + +void PassThroughHTTPCodecFilter::enableDoubleGoawayDrain() { + return call_->enableDoubleGoawayDrain(); +} + +void PassThroughHTTPCodecFilter::setHeaderCodecStats( + HeaderCodec::Stats* stats) { + call_->setHeaderCodecStats(stats); +} + +HTTPCodec::StreamID +PassThroughHTTPCodecFilter::getLastIncomingStreamID() const { + return call_->getLastIncomingStreamID(); +} + +} diff --git a/proxygen/lib/http/codec/HTTPCodecFilter.h b/proxygen/lib/http/codec/HTTPCodecFilter.h new file mode 100644 index 0000000000..bda895a031 --- /dev/null +++ b/proxygen/lib/http/codec/HTTPCodecFilter.h @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/HTTPCodec.h" +#include "proxygen/lib/utils/FilterChain.h" + +namespace proxygen { + +typedef GenericFilter< + HTTPCodec, + HTTPCodec::Callback, + &HTTPCodec::setCallback, + true> HTTPCodecFilter; + +/** + * An implementation of HTTPCodecFilter that passes through all calls. This is + * useful to subclass if you aren't interested in intercepting every function. + * See HTTPCodec.h for documentation on these methods. + */ +class PassThroughHTTPCodecFilter: public HTTPCodecFilter { + public: + /** + * By default, the filter gets both calls and callbacks + */ + explicit PassThroughHTTPCodecFilter(bool calls = true, + bool callbacks = true): + HTTPCodecFilter(calls, callbacks) {} + + // HTTPCodec::Callback methods + void onMessageBegin(StreamID stream, HTTPMessage* msg) override; + + void onPushMessageBegin(StreamID stream, StreamID assocStream, + HTTPMessage* msg) override; + + void onHeadersComplete(StreamID stream, + std::unique_ptr msg) override; + + void onBody(StreamID stream, + std::unique_ptr chain) override; + + void onChunkHeader(StreamID stream, size_t length) override; + + void onChunkComplete(StreamID stream) override; + + void onTrailersComplete(StreamID stream, + std::unique_ptr trailers) override; + + void onMessageComplete(StreamID stream, bool upgrade) override; + + void onError(StreamID stream, + const HTTPException& error, + bool newStream = false) override; + + void onAbort(StreamID stream, + ErrorCode code) override; + + void onGoaway(uint64_t lastGoodStreamID, + ErrorCode code) override; + + void onPingRequest(uint64_t uniqueID) override; + + void onPingReply(uint64_t uniqueID) override; + + void onWindowUpdate(StreamID stream, uint32_t amount) override; + + void onSettings(const SettingsList& settings) override; + + void onSettingsAck() override; + + uint32_t numOutgoingStreams() const override; + + uint32_t numIncomingStreams() const override; + + // HTTPCodec methods + CodecProtocol getProtocol() const override; + + TransportDirection getTransportDirection() const override; + + bool supportsStreamFlowControl() const override; + + bool supportsSessionFlowControl() const override; + + StreamID createStream() override; + + void setCallback(HTTPCodec::Callback* callback) override; + + bool isBusy() const override; + + void setParserPaused(bool paused) override; + + size_t onIngress(const folly::IOBuf& buf) override; + + void onIngressEOF() override; + + bool isReusable() const override; + + bool isWaitingToDrain() const override; + + bool closeOnEgressComplete() const override; + + bool supportsParallelRequests() const override; + + bool supportsPushTransactions() const override; + + void generateHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPMessage& msg, + StreamID assocStream, + HTTPHeaderSize* size) override; + + size_t generateBody(folly::IOBufQueue& writeBuf, + StreamID stream, + std::unique_ptr chain, + bool eom) override; + + size_t generateChunkHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + size_t length) override; + + size_t generateChunkTerminator(folly::IOBufQueue& writeBuf, + StreamID stream) override; + + size_t generateTrailers(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPHeaders& trailers) override; + + size_t generateEOM(folly::IOBufQueue& writeBuf, + StreamID stream) override; + + size_t generateRstStream(folly::IOBufQueue& writeBuf, + StreamID stream, + ErrorCode statusCode) override; + + size_t generateGoaway(folly::IOBufQueue& writeBuf, + StreamID lastStream, + ErrorCode statusCode) override; + + size_t generatePingRequest(folly::IOBufQueue& writeBuf) override; + + size_t generatePingReply(folly::IOBufQueue& writeBuf, + uint64_t uniqueID) override; + + size_t generateSettings(folly::IOBufQueue& writeBuf) override; + + size_t generateWindowUpdate(folly::IOBufQueue& writeBuf, + StreamID stream, + uint32_t delta) override; + + HTTPSettings* getEgressSettings() override; + + const HTTPSettings* getIngressSettings() const override; + + void setHeaderCodecStats(HeaderCodec::Stats* stats) override; + + void enableDoubleGoawayDrain() override; + + HTTPCodec::StreamID getLastIncomingStreamID() const override; +}; + +typedef FilterChain< + HTTPCodec, + HTTPCodec::Callback, + PassThroughHTTPCodecFilter, + &HTTPCodec::setCallback, + true> HTTPCodecFilterChain; + +} diff --git a/proxygen/lib/http/codec/HTTPSettings.cpp b/proxygen/lib/http/codec/HTTPSettings.cpp new file mode 100644 index 0000000000..9977845853 --- /dev/null +++ b/proxygen/lib/http/codec/HTTPSettings.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/HTTPSettings.h" + +namespace proxygen { + +void HTTPSettings::setSetting(SettingsId id, uint32_t val) { + auto s = findSetting(id); + if (!s) { + // Create the setting + settings_.emplace_back( + id, val); + ++numSettings_; + } else { + // Enable and/or update the setting + if (!s->isSet) { + s->isSet = true; + ++numSettings_; + } + s->value = val; + } +} + +void HTTPSettings::unsetSetting(SettingsId id) { + auto s = findSetting(id); + if (s && s->isSet) { + s->isSet = false; + --numSettings_; + } +} + +const HTTPSetting* HTTPSettings::getSetting(SettingsId id) const { + auto ret = findSettingConst(id); + if (!ret || !ret->isSet) { + return nullptr; + } + return ret; +} + +uint32_t HTTPSettings::getSetting(SettingsId id, + uint32_t defaultValue) const { + auto ret = findSettingConst(id); + if (!ret || !ret->isSet) { + return defaultValue; + } + return ret->value; +} + +HTTPSetting* HTTPSettings::findSetting(SettingsId id) { + for (auto& setting: settings_) { + if (setting.id == id) { + return &setting; + } + } + return nullptr; +} + +const HTTPSetting* HTTPSettings::findSettingConst(SettingsId id) const { + for (auto& setting: settings_) { + if (setting.id == id) { + return &setting; + } + } + return nullptr; +} + +} diff --git a/proxygen/lib/http/codec/HTTPSettings.h b/proxygen/lib/http/codec/HTTPSettings.h new file mode 100644 index 0000000000..4212551fc8 --- /dev/null +++ b/proxygen/lib/http/codec/HTTPSettings.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/SettingsId.h" + +#include +#include + +namespace proxygen { + +struct HTTPSetting { + HTTPSetting(SettingsId i, + uint32_t v): + id(i), + value(v), + isSet(true) {} + + SettingsId id; + uint32_t value; + bool isSet; +}; + +class HTTPSettings { + public: + // HTTP/2 Defaults + HTTPSettings() : + settings_( + {{ SettingsId::HEADER_TABLE_SIZE, 4096 }, + { SettingsId::ENABLE_PUSH, 1 }, + { SettingsId::MAX_FRAME_SIZE, 16384 }}), + numSettings_(3) { + } + HTTPSettings( + const std::initializer_list& initialSettings) + : numSettings_(0) { + for (auto& setting: initialSettings) { + setSetting(setting.first, setting.second); + } + } + void setSetting(SettingsId id, uint32_t val); + void unsetSetting(SettingsId id); + const HTTPSetting* getSetting(SettingsId id) const; + uint32_t getSetting(SettingsId id, uint32_t defaultVal) const; + // Note: this does not count disabled settings + uint8_t getNumSettings() const { return numSettings_; } + // The length of the returned vector may be greater than getNumSettings() + // TODO: replace this with an iterator that skips disabled settings + const std::vector& getAllSettings() { return settings_; } + void clearSettings() { + settings_.clear(); + numSettings_ = 0; + } + private: + HTTPSetting* findSetting(SettingsId id); + const HTTPSetting* findSettingConst(SettingsId id) const; + + std::vector settings_; + uint8_t numSettings_{0}; +}; + +typedef std::vector SettingsList; + +} diff --git a/proxygen/lib/http/codec/Makefile.am b/proxygen/lib/http/codec/Makefile.am new file mode 100644 index 0000000000..845be43bdb --- /dev/null +++ b/proxygen/lib/http/codec/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = . test diff --git a/proxygen/lib/http/codec/SPDYCodec.cpp b/proxygen/lib/http/codec/SPDYCodec.cpp new file mode 100644 index 0000000000..71ce003ad8 --- /dev/null +++ b/proxygen/lib/http/codec/SPDYCodec.cpp @@ -0,0 +1,1584 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/SPDYCodec.h" + +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/http/codec/CodecDictionaries.h" +#include "proxygen/lib/http/codec/SPDYUtil.h" +#include "proxygen/lib/http/codec/compress/GzipHeaderCodec.h" +#include "proxygen/lib/http/codec/compress/HPACKCodec.h" +#include "proxygen/lib/utils/ParseURL.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using folly::IOBuf; +using folly::IOBufQueue; +using folly::io::Appender; +using folly::io::Cursor; +using folly::io::RWPrivateCursor; +using proxygen::compress::Header; +using proxygen::compress::HeaderPieceList; +using std::pair; +using std::string; +using std::unique_ptr; +using std::vector; + +namespace proxygen { + +namespace { + +// Sizes, in bytes, of various types and parts of SPDY frames +const size_t kFrameSizeDataCommon = 8; // common prefix of all data frames +const size_t kFrameSizeControlCommon = 8; // common prefix of all ctrl frames +const size_t kFrameSizeSynStream = 10; // SYN_STREAM +const size_t kFrameSizeSynReplyv2 = 6; // SYN_REPLY, SPDYv2 +const size_t kFrameSizeSynReplyv3 = 4; // SPDYv3's SYN_REPLY is shorter +const size_t kFrameSizeRstStream = 8; // RST_STREAM +const size_t kFrameSizeGoawayv2 = 4; // GOAWAY, SPDYv2 +const size_t kFrameSizeGoawayv3 = 8; // GOAWAY, SPDYv3 +const size_t kFrameSizeHeaders = 4; // HEADERS +const size_t kFrameSizePing = 4; // PING +const size_t kFrameSizeSettings = 4; // SETTINGS +const size_t kFrameSizeSettingsEntry = 8; // Each entry in SETTINGS +const size_t kFrameSizeWindowUpdate = 8; // WINDOW_UPDATE +const size_t kFrameSizeNameValuev2 = 2; // The size in bytes of a + // name/value pair +const size_t kFrameSizeNameValuev3 = 4; // The size in bytes of a + // name/value pair +const size_t kPriShiftv2 = 6; // How many bits to shift pri, v2 +const size_t kPriShiftv3 = 5; // How many bits to shift pri, v3 + +#define CTRL_MASK 0x80 +#define FLAGS_MASK 0xff000000 +#define STREAM_ID_MASK 0x7fffffff +#define VERSION_MASK 0x7fff +#define DELTA_WINDOW_SIZE_MASK 0x7fffffff + +/* The number of bytes of the frame header. */ +#define FRAME_HEADER_LEN 8 + +// SPDY flags +const uint8_t kFlagFin = 0x01; + +/** + * Convenience function to pack SPDY's 8-bit flags field and + * 24-bit length field into a single uint32_t so we can write + * them out more easily. (This function packs the flags into + * the high order 8 bits of a 32-bit int; because SPDY uses + * network byte ordering for these fields, the flag thus ends + * up in the right place - in front of the length - when we + * serialize the returned uint32_t.) + */ +uint32_t flagsAndLength(uint8_t flags, uint32_t length) { + length &= 0x00ffffff; + length |= (int32_t(flags) << 24); + return length; +} + +void appendUint16(uint8_t*& dst, size_t value) { + *(uint16_t*)dst = htons(uint16_t(value)); + dst += 2; +} + +void appendUint32(uint8_t*& dst, size_t value) { + *(uint32_t*)dst = htonl(uint32_t(value)); + dst += 4; +} + +uint32_t parseUint16(Cursor* cursor) { + auto chunk = cursor->peek(); + if (LIKELY(chunk.second >= sizeof(uint16_t))) { + cursor->skip(sizeof(uint16_t)); + return ntohs(*(uint16_t*)chunk.first); + } + return cursor->readBE(); +} + +uint32_t parseUint32(Cursor* cursor) { + auto chunk = cursor->peek(); + if (LIKELY(chunk.second >= sizeof(uint32_t))) { + cursor->skip(sizeof(uint32_t)); + return ntohl(*(uint32_t*)chunk.first); + } + return cursor->readBE(); +} + +class SPDYSessionFailed : public std::exception { + public: + explicit SPDYSessionFailed(spdy::GoawayStatusCode inStatus) + : statusCode(inStatus) {} + + spdy::GoawayStatusCode statusCode; +}; + +class SPDYStreamFailed : public std::exception { + public: + SPDYStreamFailed(bool inIsNew, uint32_t inStreamID, + uint32_t inStatus, + const std::string& inMsg = empty_string) + : isNew(inIsNew), + streamID(inStreamID), + statusCode(inStatus) { + message = folly::to("new=", isNew, " streamID=", streamID, + " statusCode=", statusCode, " message=", + inMsg); + } + + ~SPDYStreamFailed() throw() {} + + virtual const char* what() const throw() { + return message.c_str(); + } + + bool isNew; + uint32_t streamID; + uint32_t statusCode; + std::string message; +}; + +void printCtrlHeader(uint16_t version, uint8_t flags, uint32_t length) { + std::cout << "CTRL FRAME: version=" << version << ", flags=" + << std::hex << folly::to(flags) << std::dec + << ", length=" << length << std::endl; +} + +void printNV(const compress::HeaderPieceList& headers) { + for (size_t i = 0; i < headers.size(); i += 2) { + std::cout << "\t" << headers[i].str << ": " + << headers[i + 1].str << std::endl; + } +} + +void printHeaders(uint32_t stream_id, + const compress::HeaderPieceList& headers) { + std::cout << "HEADERS: stream_id=" << stream_id + << "numHeaders=" << headers.size() / 2 << std::endl; + printNV(headers); +} + +void printSynStream(uint32_t stream_id, uint32_t assocStream, uint8_t pri, + uint8_t slot, + const compress::HeaderPieceList& headers) { + std::cout << "SYN_STREAM: stream_id=" << stream_id << ", assocStream=" + << assocStream << ", pri=" << folly::to(pri) + << ", slot=" << folly::to(slot) + << ", numHeaders=" << (headers.size() / 2) << std::endl; + printNV(headers); +} + +void printSynReply(uint32_t stream_id, + const compress::HeaderPieceList& headers) { + std::cout << "SYN_REPLY: stream_id=" << stream_id + << ", numHeaders=" << headers.size() / 2 << std::endl; + + printNV(headers); +} + +void printRstStream(uint32_t stream_id, uint32_t statusCode) { + std::cout << "RST_STREAM: stream_id=" << stream_id << ", statusCode=" + << statusCode << std::endl; +} + +void printSettings(const SPDYCodec::SettingList& settings) { + std::cout << "SETTINGS: num=" << settings.size() << std::endl; + for (const auto& setting: settings) { + std::cout << "\tflags=" + << std::hex << folly::to(setting.flags) << std::dec + << ", id=" << setting.id + << ", value=" << setting.value << std::endl; + } +} + +void printPing(uint32_t unique_id) { + std::cout << "PING: unique_id=" << unique_id << std::endl; +} + +void printGoaway(uint32_t lastGoodStream, uint32_t statusCode) { + std::cout << "GOAWAY: lastGoodStream=" << lastGoodStream + << ", statusCode=" << statusCode << std::endl; +} + +void printWindowUpdate(uint32_t stream_id, uint32_t delta) { + std::cout << "WINDOW_UPDATE: stream_id=" << stream_id + << "delta_window_size=" << delta << std::endl; +} + +void printDataFrame(uint32_t stream_id, uint8_t flags, uint32_t length) { + std::cout << "DATA: stream_id=" << stream_id << ", flags=" + << std::hex << folly::to(flags) << std::dec + << ", length=" << length << std::endl; +} + +void printException(const std::exception& ex) { + std::cout << "Exception: " << folly::exceptionStr(ex) << std::endl; +} + +} // anonynous namespace + +std::bitset<256> SPDYCodec::perHopHeaderCodes_; + +void SPDYCodec::initPerHopHeaders() { + // SPDY per-hop headers + perHopHeaderCodes_[HTTP_HEADER_CONNECTION] = true; + perHopHeaderCodes_[HTTP_HEADER_HOST] = true; + perHopHeaderCodes_[HTTP_HEADER_KEEP_ALIVE] = true; + perHopHeaderCodes_[HTTP_HEADER_PROXY_CONNECTION] = true; + perHopHeaderCodes_[HTTP_HEADER_TRANSFER_ENCODING] = true; + perHopHeaderCodes_[HTTP_HEADER_UPGRADE] = true; +} + +const string& SPDYCodec::getHpackNpn() { + static const std::string hpackNpn = "spdy/3.1-fb-" + + folly::to(kHPACKMajorVersion) + "." + + folly::to(kHPACKMinorVersion); + return hpackNpn; +} + +const SPDYVersionSettings& SPDYCodec::getVersionSettings(SPDYVersion version) { + // Indexed by SPDYVersion + static const SPDYVersionSettings spdyVersions[] = { + // SPDY2 + {spdy::kNameVersionv2, spdy::kNameStatusv2, spdy::kNameMethodv2, + spdy::kNamePathv2, spdy::kNameSchemev2, "", + spdy::kSessionProtoNameSPDY2, parseUint16, appendUint16, + (const unsigned char*)kSPDYv2Dictionary, sizeof(kSPDYv2Dictionary), + 0x8002, kFrameSizeSynReplyv2, kFrameSizeNameValuev2, + kFrameSizeGoawayv2, kPriShiftv2, 2, 0, SPDYVersion::SPDY2}, + // SPDY3 + {spdy::kNameVersionv3, spdy::kNameStatusv3, spdy::kNameMethodv3, + spdy::kNamePathv3, spdy::kNameSchemev3, spdy::kNameHostv3, + spdy::kSessionProtoNameSPDY3, parseUint32, appendUint32, + (const unsigned char*)kSPDYv3Dictionary, sizeof(kSPDYv3Dictionary), + 0x8003, kFrameSizeSynReplyv3, kFrameSizeNameValuev3, + kFrameSizeGoawayv3, kPriShiftv3, 3, 0, SPDYVersion::SPDY3}, + // SPDY3.1 + {spdy::kNameVersionv3, spdy::kNameStatusv3, spdy::kNameMethodv3, + spdy::kNamePathv3, spdy::kNameSchemev3, spdy::kNameHostv3, + spdy::kSessionProtoNameSPDY3, parseUint32, appendUint32, + (const unsigned char*)kSPDYv3Dictionary, sizeof(kSPDYv3Dictionary), + 0x8003, kFrameSizeSynReplyv3, kFrameSizeNameValuev3, + kFrameSizeGoawayv3, kPriShiftv3, 3, 1, SPDYVersion::SPDY3_1} + }; + // SPDY3_1_HPACK is identical to SPDY3 in terms of version settings structure + if (version == SPDYVersion::SPDY3_1_HPACK) { + version = SPDYVersion::SPDY3_1; + } + auto intVersion = static_cast(version); + CHECK(intVersion < (sizeof(spdyVersions) / sizeof(SPDYVersionSettings))); + return spdyVersions[intVersion]; +} + +SPDYCodec::SPDYCodec(TransportDirection direction, SPDYVersion version, + int spdyCompressionLevel /* = Z_NO_COMPRESSION */) + : callback_(nullptr), + transportDirection_(direction), + lastStreamID_(0), + versionSettings_(getVersionSettings(version)), + maxFrameLength_(spdy::kMaxFrameLength), + currentIngressBuf_(nullptr), + frameState_(FrameState::FRAME_HEADER), + version_(0), + type_(0xffff), + streamId_(0), + length_(0), + flags_(0), + sessionClosing_(ClosingState::OPEN), + printer_(false), + ctrl_(false) { + VLOG(4) << "creating SPDY/" << static_cast(versionSettings_.majorVersion) + << "." << static_cast(versionSettings_.minorVersion) + << " codec"; + if (version == SPDYVersion::SPDY3_1_HPACK) { + headerCodec_ = folly::make_unique(transportDirection_); + } else { + headerCodec_ = folly::make_unique( + spdyCompressionLevel, versionSettings_); + // Use the default value. + headerCodec_->setMaxUncompressed(proxygen::spdy::kMaxFrameLength); + } + + switch (transportDirection_) { + case TransportDirection::DOWNSTREAM: + nextEgressStreamID_ = 2; + nextEgressPingID_ = 2; + break; + case TransportDirection::UPSTREAM: + nextEgressStreamID_ = 1; + nextEgressPingID_ = 1; + break; + default: + LOG(FATAL) << "Unknown transport direction."; + } +} + +SPDYCodec::~SPDYCodec() { +} + +void SPDYCodec::setMaxFrameLength(uint32_t maxFrameLength) { + maxFrameLength_ = maxFrameLength; +} + +void SPDYCodec::setMaxUncompressedHeaders(uint32_t maxUncompressed) { + headerCodec_->setMaxUncompressed(maxUncompressed); +} + +CodecProtocol SPDYCodec::getProtocol() const { + switch (versionSettings_.version) { + case SPDYVersion::SPDY2: return CodecProtocol::SPDY_2; + case SPDYVersion::SPDY3: return CodecProtocol::SPDY_3; + case SPDYVersion::SPDY3_1: return CodecProtocol::SPDY_3_1; + case SPDYVersion::SPDY3_1_HPACK: return CodecProtocol::SPDY_3_1_HPACK; + }; + LOG(FATAL) << "unreachable"; + return CodecProtocol::SPDY_3_1; +} + +bool SPDYCodec::supportsStreamFlowControl() const { + return versionSettings_.majorVersion > 2; +} + +bool SPDYCodec::supportsSessionFlowControl() const { + return versionSettings_.majorVersion > 3 || + (versionSettings_.majorVersion == 3 && versionSettings_.minorVersion > 0); +} + +HTTPCodec::StreamID SPDYCodec::createStream() { + auto ret = nextEgressStreamID_; + nextEgressStreamID_ += 2; + return ret; +} + +bool SPDYCodec::isBusy() const { + return false; +} + +void SPDYCodec::setParserPaused(bool paused) { + // Not applicable +} + +void SPDYCodec::checkLength(uint32_t expectedLength, const std::string& msg) { + if (length_ != expectedLength) { + LOG(ERROR) << msg << ": invalid length " << length_ << " != " << + expectedLength; + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } +} + +void SPDYCodec::checkMinLength(uint32_t minLength, const std::string& msg) { + if (length_ < minLength) { + LOG(ERROR) << msg << ": invalid length " << length_ << " < " << minLength; + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } +} + +size_t SPDYCodec::onIngress(const folly::IOBuf& buf) { + size_t bytesParsed = 0; + currentIngressBuf_ = &buf; + try { + bytesParsed = parseIngress(buf); + } catch (const SPDYSessionFailed& ex) { + if (printer_) { + printException(ex); + } + failSession(ex.statusCode); + bytesParsed = buf.computeChainDataLength(); + } + return bytesParsed; +} + +size_t SPDYCodec::parseIngress(const folly::IOBuf& buf) { + const size_t chainLength = buf.computeChainDataLength(); + Cursor cursor(&buf); + size_t avail = cursor.totalLength(); + + // This can parse beyond the current IOBuf + for (; avail > 0; avail = cursor.totalLength()) { + if (frameState_ == FrameState::FRAME_HEADER) { + if (avail < FRAME_HEADER_LEN) { + // Make the caller buffer until we get a full frame header + break; + } + auto data = cursor.peek(); + ctrl_ = (data.first[0] & CTRL_MASK); + if (ctrl_) { + version_ = cursor.readBE() & VERSION_MASK; + type_ = cursor.readBE(); + if (version_ != versionSettings_.majorVersion) { + LOG(ERROR) << "Invalid version=" << version_; + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } + } else { + streamId_ = cursor.readBE() & STREAM_ID_MASK; + } + length_ = cursor.readBE(); + flags_ = (length_ & FLAGS_MASK) >> 24; + length_ &= ~FLAGS_MASK; + if (ctrl_) { + if (length_ > maxFrameLength_) { + if (type_ == spdy::SYN_STREAM || type_ == spdy::SYN_REPLY || + type_ == spdy::HEADERS) { + uint32_t stream_id = cursor.readBE() & STREAM_ID_MASK; + failStream(true, stream_id, spdy::RST_FRAME_TOO_LARGE); + // Compression/stream state is out of sync now + } + // Since maxFrameLength_ must be at least 8kb and most control frames + // have fixed size, only an invalid settings or credential frame can + // land here. For invalid credential frames we must send a goaway, + // and a settings frame would have > 1023 pairs, of which none are + // allowed to be duplicates. Just fail everything. + LOG(ERROR) << "excessive frame size length_=" << length_; + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } + frameState_ = FrameState::CTRL_FRAME_DATA; + if (printer_) { + printCtrlHeader(version_, flags_, length_); + } + } else { + frameState_ = FrameState::DATA_FRAME_DATA; + if (printer_) { + printDataFrame(streamId_, flags_, length_); + } + } + } else if (frameState_ == FrameState::CTRL_FRAME_DATA) { + if (avail < length_) { + // Make the caller buffer the rest of the control frame. + // We could attempt to decompress incomplete name/value blocks, + // but for now we're favoring simplicity. + VLOG(6) << "Need more data: length_=" << length_ << " avail=" << avail; + break; + } + try { + onControlFrame(cursor); + } catch (const SPDYStreamFailed& ex) { + if (printer_) { + printException(ex); + } + failStream(ex.isNew, ex.streamID, ex.statusCode, ex.what()); + } + frameState_ = FrameState::FRAME_HEADER; + } else if (avail > 0 || length_ == 0) { + // Data frame data. Pass everything we have up to the frame boundary + DCHECK(FrameState::DATA_FRAME_DATA == frameState_); + + uint32_t toClone = (avail > std::numeric_limits::max()) ? + std::numeric_limits::max() : static_cast(avail); + toClone = std::min(toClone, length_); + std::unique_ptr chunk; + cursor.clone(chunk, toClone); + callback_->onBody(StreamID(streamId_), std::move(chunk)); + length_ -= toClone; + } + + // Fin handling + if (length_ == 0) { + if (flags_ & spdy::CTRL_FLAG_FIN) { + callback_->onMessageComplete(StreamID(streamId_), false); + } + frameState_ = FrameState::FRAME_HEADER; + } + } + return chainLength - avail; +} + +void SPDYCodec::onControlFrame(Cursor& cursor) { + uint32_t stream_id = 0; + switch (type_) { + case spdy::SYN_STREAM: + { + checkMinLength(kFrameSizeSynStream, "SYN_STREAM"); + streamId_ = cursor.readBE() & STREAM_ID_MASK; + uint32_t assocStream = cursor.readBE(); + uint8_t pri = cursor.read() >> versionSettings_.priShift; + uint8_t slot = cursor.read(); + length_ -= kFrameSizeSynStream; + auto result = decodeHeaders(cursor); + checkLength(0, "SYN_STREAM"); + onSynStream(assocStream, pri, slot, + result.headers, headerCodec_->getDecodedSize()); + break; + } + case spdy::SYN_REPLY: + { + checkMinLength(versionSettings_.synReplySize, "SYN_REPLY"); + streamId_ = cursor.readBE() & STREAM_ID_MASK; + length_ -= versionSettings_.synReplySize; + if (version_ == 2) { + // 2 byte unused + cursor.skip(2); + } + auto result = decodeHeaders(cursor); + checkLength(0, "SYN_REPLY"); + onSynReply(result.headers, + headerCodec_->getDecodedSize()); + break; + } + case spdy::RST_STREAM: + { + checkLength(kFrameSizeRstStream, "RST"); + streamId_ = cursor.readBE() & STREAM_ID_MASK; + uint32_t statusCode = cursor.readBE(); + onRstStream(statusCode); + break; + } + case spdy::SETTINGS: + { + checkMinLength(kFrameSizeSettings, "SETTINGS"); + uint32_t numSettings = cursor.readBE(); + length_ -= sizeof(uint32_t); + if (length_ / 8 < numSettings) { + LOG(ERROR) << "SETTINGS: number of settings to high. " + << length_ << " < 8 * " << numSettings; + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } + SettingList settings; + for (uint32_t i = 0; i < numSettings; i++) { + uint32_t id = 0; + if (version_ == 2) { + id = cursor.readLE(); + } else { + id = cursor.readBE(); + } + uint32_t value = cursor.readBE(); + uint8_t flags = (id & FLAGS_MASK) >> 24; + id &= ~FLAGS_MASK; + settings.emplace_back(flags, id, value); + } + onSettings(settings); + break; + } + case spdy::NOOP: + VLOG(4) << "Noop received. Doing nothing."; + checkLength(0, "NOOP"); + break; + case spdy::PING: + { + checkLength(kFrameSizePing, "PING"); + uint32_t unique_id = cursor.readBE(); + onPing(unique_id); + break; + } + case spdy::GOAWAY: + { + checkLength(versionSettings_.goawaySize, "GOAWAY"); + uint32_t lastStream = cursor.readBE(); + uint32_t statusCode = 0; + if (version_ == 3) { + statusCode = cursor.readBE(); + } + onGoaway(lastStream, statusCode); + break; + } + case spdy::HEADERS: + { + // Note: this is for the HEADERS frame type, not the initial headers + checkMinLength(kFrameSizeHeaders, "HEADERS"); + streamId_ = cursor.readBE() & STREAM_ID_MASK; + length_ -= kFrameSizeHeaders; + if (version_ == 2) { + // 2 byte unused + cursor.skip(2); + length_ -= 2; + } + auto result = decodeHeaders(cursor); + checkLength(0, "HEADERS"); + onHeaders(result.headers); + break; + } + case spdy::WINDOW_UPDATE: + { + checkLength(kFrameSizeWindowUpdate, "WINDOW_UPDATE"); + streamId_ = cursor.readBE() & STREAM_ID_MASK; + uint32_t delta = cursor.readBE() & DELTA_WINDOW_SIZE_MASK; + onWindowUpdate(delta); + break; + } + case spdy::CREDENTIAL: + { + VLOG(4) << "Skipping unsupported/deprecated CREDENTIAL frame"; + // Fall through to default case + } + default: + VLOG(3) << "unimplemented control frame type " << type_ + << ", frame length: " << length_; + // From spdy spec: + // Control frame processing requirements: + // If an endpoint receives a control frame for a type it does not + // recognize, it must ignore the frame. + + // Consume rest of the frame to skip processing it further + cursor.skip(length_); + length_ = 0; + return; + } +} + +HeaderDecodeResult SPDYCodec::decodeHeaders(Cursor& cursor) { + auto result = headerCodec_->decode(cursor, length_); + if (result.isError()) { + auto err = result.error(); + if (err == HeaderDecodeError::HEADERS_TOO_LARGE || + err == HeaderDecodeError::INFLATE_DICTIONARY || + err == HeaderDecodeError::BAD_ENCODING) { + // Fail stream only for FRAME_TOO_LARGE error + if (err == HeaderDecodeError::HEADERS_TOO_LARGE) { + failStream(true, streamId_, spdy::RST_FRAME_TOO_LARGE); + } + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } + // For other types of errors we throw a stream error + bool newStream = (type_ != spdy::HEADERS); + throw SPDYStreamFailed(newStream, streamId_, spdy::RST_PROTOCOL_ERROR, + "Error parsing header: " + folly::to(err)); + } + + length_ -= result.ok().bytesConsumed; + return result.ok(); +} + +void SPDYCodec::onIngressEOF() { + // SPDY does not report errors for partial frames +} + +bool SPDYCodec::isReusable() const { + // This codec can process new streams if it is open, or if it is a + // server and it has only sent the first of two goaways. + // TODO move ingressGoawayAck_ into ClosingState and simplify this logic. + return (sessionClosing_ == ClosingState::OPEN || + sessionClosing_ == ClosingState::OPEN_WITH_GRACEFUL_DRAIN_ENABLED || + (transportDirection_ == TransportDirection::DOWNSTREAM && + isWaitingToDrain())) + && (ingressGoawayAck_ == std::numeric_limits::max()); +} + +bool SPDYCodec::isWaitingToDrain() const { + return sessionClosing_ == ClosingState::FIRST_GOAWAY_SENT; +} + +bool SPDYCodec::isSPDYReserved(const std::string& name) { + return (versionSettings_.majorVersion == 2 && + ((transportDirection_ == TransportDirection::DOWNSTREAM && + (boost::iequals(name, spdy::kNameStatusv2) || + boost::iequals(name, spdy::kNameVersionv2))) || + (transportDirection_ == TransportDirection::UPSTREAM && + (boost::iequals(name, spdy::kNameMethodv2) || + boost::iequals(name, spdy::kNameSchemev2) || + boost::iequals(name, spdy::kNamePathv2) || + boost::iequals(name, spdy::kNameVersionv2))))); +} + +// Add the SPDY-specific header fields that hold the +// equivalent of the HTTP/1.x request-line or status-line. +unique_ptr SPDYCodec::encodeHeaders( + const HTTPMessage& msg, vector
& allHeaders, + uint32_t headroom, HTTPHeaderSize* size) { + + allHeaders.emplace_back(versionSettings_.versionStr, spdy::httpVersion); + + // Add the HTTP headers supplied by the caller, but skip + // any per-hop headers that aren't supported in SPDY. + msg.getHeaders().forEachWithCode([&] (HTTPHeaderCode code, + const string& name, + const string& value) { + if (perHopHeaderCodes_[code] || isSPDYReserved(name)) { + VLOG(3) << "Dropping SPDY reserved header " << name; + return; + } + if (name.length() == 0) { + VLOG(2) << "Dropping header with empty name"; + return; + } + if (versionSettings_.majorVersion == 2 && value.length() == 0) { + VLOG(2) << "Dropping header \"" << name + << "\" with empty value for spdy/2"; + return; + } + allHeaders.emplace_back(code, name, value); + }); + + headerCodec_->setEncodeHeadroom(headroom); + auto out = headerCodec_->encode(allHeaders); + if (size) { + *size = headerCodec_->getEncodedSize(); + } + + return out; +} + +unique_ptr SPDYCodec::serializeResponseHeaders( + const HTTPMessage& msg, uint32_t headroom, HTTPHeaderSize* size) { + + // Note: the header-sorting code works with pointers to strings. + // The role of this local status string is to hold the generated + // status code long enough for the sort (done later within the + // same scope) to be able to access it. + string status; + + const HTTPHeaders& headers = msg.getHeaders(); + vector
allHeaders; + allHeaders.reserve(headers.size() + 4); + + if (msg.getStatusMessage().empty()) { + status = folly::to(msg.getStatusCode()); + } else { + status = folly::to(msg.getStatusCode(), " ", + msg.getStatusMessage()); + } + allHeaders.emplace_back(versionSettings_.statusStr, status); + // See comment above regarding status + string date; + if (!headers.exists(HTTP_HEADER_DATE)) { + date = std::move(HTTPMessage::formatDateHeader()); + allHeaders.emplace_back(HTTP_HEADER_DATE, date); + } + + return encodeHeaders(msg, allHeaders, headroom, size); +} + +unique_ptr SPDYCodec::serializeRequestHeaders( + const HTTPMessage& msg, + bool isPushed, + uint32_t headroom, + HTTPHeaderSize* size) { + + const HTTPHeaders& headers = msg.getHeaders(); + vector
allHeaders; + allHeaders.reserve(headers.size() + 6); + + const string& method = msg.getMethodString(); + const string& scheme = msg.isSecure() ? spdy::https : spdy::http; + string path = msg.getURL(); + + if (versionSettings_.majorVersion == 2 && path[0] == '/') { + // We don't send the host header, SPDY/2 requires absolute URLs + const string& host = msg.getHeaders().getSingleOrEmpty(HTTP_HEADER_HOST); + if (!host.empty()) { + string url = scheme; + url.append("://"); + url.append(host); + url.append(path); + path = std::move(url); + } // oh well. roll the dice + } + + if (isPushed) { + static const string ok("200"); + allHeaders.emplace_back(versionSettings_.statusStr, ok); + } else { + allHeaders.emplace_back(versionSettings_.methodStr, method); + } + allHeaders.emplace_back(versionSettings_.schemeStr, scheme); + allHeaders.emplace_back(versionSettings_.pathStr, path); + if (versionSettings_.majorVersion == 3) { + DCHECK(headers.exists(HTTP_HEADER_HOST)); + const string& host = headers.getSingleOrEmpty(HTTP_HEADER_HOST); + allHeaders.emplace_back(versionSettings_.hostStr, host); +} + + return encodeHeaders(msg, allHeaders, headroom, size); +} + +void SPDYCodec::generateHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPMessage& msg, + StreamID assocStream, + HTTPHeaderSize* size) { + if (transportDirection_ == TransportDirection::UPSTREAM || + assocStream != HTTPCodec::NoStream) { + generateSynStream(stream, assocStream, writeBuf, msg, size); + } else { + generateSynReply(stream, writeBuf, msg, size); + } +} + +void SPDYCodec::generateSynStream(StreamID stream, + StreamID assocStream, + folly::IOBufQueue& writeBuf, + const HTTPMessage& msg, + HTTPHeaderSize* size) { + // Pushed streams must have an even streamId and an odd assocStream + CHECK((assocStream == HTTPCodec::NoStream && (stream % 2 == 1)) || + ((stream % 2 == 0) && (assocStream % 2 == 1))) << + "Invalid stream ids stream=" << stream << " assocStream=" << assocStream; + + // Serialize the compressed representation of the headers + // first because we need to write its length. The + // serializeRequestHeaders() method allocates an IOBuf to + // hold the headers, but we can tell it to reserve + // enough headroom at the start of the IOBuf to hold + // the metadata we'll need to add once we know the + // length. + uint32_t fieldsSize = kFrameSizeSynStream; + uint32_t headroom = kFrameSizeControlCommon + fieldsSize; + bool isPushed = (assocStream != HTTPCodec::NoStream); + unique_ptr out(serializeRequestHeaders(msg, isPushed, + headroom, size)); + + // The length field in the SYN_STREAM header holds the number + // of bytes that follow it. That's the length of the fields + // specific to the SYN_STREAM message (all of which come after + // the length field) plus the length of the serialized header + // name/value block. + uint32_t len = fieldsSize + out->computeChainDataLength(); + + // Generate a control frame header of type SYN_STREAM within + // the headroom that serializeRequestHeaders() reserved for us + // at the start of the IOBuf. + uint8_t flags = (assocStream != HTTPCodec::NoStream) ? + spdy::CTRL_FLAG_UNIDIRECTIONAL : spdy::CTRL_FLAG_NONE; + out->prepend(headroom); + RWPrivateCursor cursor(out.get()); + cursor.writeBE(versionSettings_.controlVersion); + cursor.writeBE(uint16_t(spdy::SYN_STREAM)); + cursor.writeBE(flagsAndLength(flags, len)); + cursor.writeBE(uint32_t(stream)); + cursor.writeBE(uint32_t(assocStream)); + cursor.writeBE(uint16_t( + msg.getPriority() << (versionSettings_.priShift + 8))); + + // Now that we have a complete SYN_STREAM control frame, append + // it to the writeBuf. + writeBuf.append(std::move(out)); +} + +void SPDYCodec::generateSynReply(StreamID stream, + folly::IOBufQueue& writeBuf, + const HTTPMessage& msg, + HTTPHeaderSize* size) { + // Serialize the compressed representation of the headers + // first because we need to write its length. The + // serializeResponseHeaders() method allocates an IOBuf to + // hold the headers, but we can tell it to reserve + // enough headroom at the start of the IOBuf to hold + // the metadata we'll need to add once we know the + // length. + uint32_t headroom = kFrameSizeControlCommon + versionSettings_.synReplySize; + unique_ptr out(serializeResponseHeaders(msg, headroom, size)); + + // The length field in the SYN_REPLY header holds the number + // of bytes that follow it. That's the length of the fields + // specific to the SYN_REPLY message (all of which come after + // the length field) plus the length of the serialized header + // name/value block. + uint32_t len = versionSettings_.synReplySize + out->computeChainDataLength(); + + // Generate a control frame header of type SYN_REPLY within + // the headroom that we serializeResponseHeaders() reserved for us + // at the start of the IOBuf.1 + out->prepend(headroom); + RWPrivateCursor cursor(out.get()); + cursor.writeBE(versionSettings_.controlVersion); + cursor.writeBE(uint16_t(spdy::SYN_REPLY)); + cursor.writeBE(flagsAndLength(0, len)); + cursor.writeBE(uint32_t(stream)); // TODO: stream should never be bigger than 2^31 + if (versionSettings_.majorVersion == 2) { + cursor.writeBE(uint16_t(0)); + } + + // Now that we have a complete SYN_REPLY control frame, append + // it to the writeBuf. + writeBuf.append(std::move(out)); +} + +size_t SPDYCodec::generateBody(folly::IOBufQueue& writeBuf, + StreamID stream, + std::unique_ptr chain, + bool eom) { + size_t len = chain->computeChainDataLength(); + if (len == 0) { + return len; + } + + // TODO if the data length is 2^24 or greater, split it into + // multiple data frames. Proxygen should never be writing that + // much data at once, but other apps that use this codec might. + CHECK(len < (1 << 24)); + + uint8_t flags = (eom) ? kFlagFin : 0; + generateDataFrame(writeBuf, uint32_t(stream), flags, len); + writeBuf.append(std::move(chain)); + return len; +} + +size_t SPDYCodec::generateChunkHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + size_t length) { + // SPDY chunk headers are built into the data frames + return 0; +} + +size_t SPDYCodec::generateChunkTerminator(folly::IOBufQueue& writeBuf, + StreamID stream) { + // SPDY has no chunk terminator + return 0; +} + +size_t SPDYCodec::generateTrailers(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPHeaders& trailers) { + // TODO generate a HEADERS frame? An additional HEADERS frame + // somewhere after the SYN_REPLY seems to be the SPDY equivalent + // of HTTP/1.1's trailers. + return 0; +} + +size_t SPDYCodec::generateEOM(folly::IOBufQueue& writeBuf, + StreamID stream) { + VLOG(4) << "sending EOM for stream=" << stream; + generateDataFrame(writeBuf, uint32_t(stream), kFlagFin, 0); + return 8; // size of data frame header +} + +size_t SPDYCodec::generateRstStream(IOBufQueue& writeBuf, + StreamID stream, + ErrorCode code) { + DCHECK(stream > 0); + VLOG(4) << "sending RST_STREAM for stream=" << stream + << " with code=" << getErrorCodeString(code); + + // Suppress any EOM callback for the current frame. + if (stream == streamId_) { + flags_ &= ~spdy::CTRL_FLAG_FIN; + } + + const uint32_t statusCode = (uint32_t) spdy::errorCodeToReset(code); + unique_ptr frame = IOBuf::create(kFrameSizeControlCommon + + kFrameSizeRstStream); + size_t written = 0; + Appender appender(frame.get(), 0); + appender.writeBE(versionSettings_.controlVersion); + appender.writeBE(uint16_t(spdy::RST_STREAM)); + appender.writeBE(flagsAndLength(0, kFrameSizeRstStream)); + appender.writeBE(uint32_t(stream)); + appender.writeBE(rstStatusSupported(statusCode) ? + statusCode : spdy::RST_PROTOCOL_ERROR); + written = frame->length(); + writeBuf.append(std::move(frame)); + return written; +} + +size_t SPDYCodec::generateGoaway(IOBufQueue& writeBuf, + StreamID lastStream, + ErrorCode code) { + const uint32_t statusCode = (uint32_t) spdy::errorCodeToGoaway(code); + unique_ptr frame = IOBuf::create(kFrameSizeControlCommon + + (size_t)versionSettings_.goawaySize); + size_t written = 0; + Appender appender(frame.get(), 0); + appender.writeBE(versionSettings_.controlVersion); + + if (sessionClosing_ == ClosingState::CLOSING) { + VLOG(4) << "Not sending GOAWAY for closed session"; + return 0; + } + if (code != ErrorCode::NO_ERROR) { + sessionClosing_ = ClosingState::CLOSING; + } + + VLOG(4) << "Sending GOAWAY with last acknowledged stream=" + << lastStream << " with code=" << getErrorCodeString(code); + + appender.writeBE(uint16_t(spdy::GOAWAY)); + appender.writeBE(flagsAndLength(0, versionSettings_.goawaySize)); + appender.writeBE(uint32_t(lastStream)); + if (versionSettings_.majorVersion == 3) { + appender.writeBE(statusCode); + } + switch (sessionClosing_) { + case ClosingState::OPEN: + sessionClosing_ = ClosingState::CLOSING; + break; + case ClosingState::OPEN_WITH_GRACEFUL_DRAIN_ENABLED: + if (lastStream == std::numeric_limits::max()) { + sessionClosing_ = ClosingState::FIRST_GOAWAY_SENT; + } else { + // The user of this codec decided not to do the double goaway + // drain + sessionClosing_ = ClosingState::CLOSING; + } + break; + case ClosingState::FIRST_GOAWAY_SENT: + sessionClosing_ = ClosingState::CLOSING; + break; + case ClosingState::CLOSING: + break; + } + written = frame->length(); + writeBuf.append(std::move(frame)); + return written; +} + +size_t SPDYCodec::generatePingRequest(IOBufQueue& writeBuf) { + const auto id = nextEgressPingID_; + nextEgressPingID_ += 2; + VLOG(4) << "Generating ping request with id=" << id; + return generatePingCommon(writeBuf, id); +} + +size_t SPDYCodec::generatePingReply(IOBufQueue& writeBuf, uint64_t uniqueID) { + VLOG(4) << "Generating ping reply with id=" << uniqueID; + return generatePingCommon(writeBuf, uniqueID); +} + +size_t SPDYCodec::generatePingCommon(IOBufQueue& writeBuf, uint64_t uniqueID) { + unique_ptr frame = IOBuf::create(kFrameSizeControlCommon + + kFrameSizePing); + Appender appender(frame.get(), 0); + appender.writeBE(versionSettings_.controlVersion); + appender.writeBE(uint16_t(spdy::PING)); + appender.writeBE(flagsAndLength(0, kFrameSizePing)); + appender.writeBE(uint32_t(uniqueID)); + size_t encodedSize = frame->length(); + writeBuf.append(std::move(frame)); + return encodedSize; +} + +size_t SPDYCodec::generateSettings(folly::IOBufQueue& writeBuf) { + auto numSettings = egressSettings_.getNumSettings(); + VLOG(4) << "generating " << (unsigned)numSettings << " settings"; + unique_ptr frame = IOBuf::create( + kFrameSizeControlCommon + kFrameSizeSettings + + (kFrameSizeSettingsEntry * numSettings)); + Appender appender(frame.get(), 0); + appender.writeBE(versionSettings_.controlVersion); + appender.writeBE(uint16_t(spdy::SETTINGS)); + appender.writeBE(flagsAndLength(spdy::FLAG_SETTINGS_CLEAR_SETTINGS, + kFrameSizeSettings + + kFrameSizeSettingsEntry * numSettings)); + appender.writeBE(uint32_t(numSettings)); + for (const auto& setting: egressSettings_.getAllSettings()) { + if (!setting.isSet) { + continue; + } + auto settingId = spdy::httpToSpdySettingsId(setting.id); + if (!settingId) { + LOG(WARNING) << "Invalid SpdySetting " << (uint32_t)setting.id; + continue; + } + VLOG(5) << " writing setting with id=" << *settingId + << ", value=" << setting.value; + if (versionSettings_.majorVersion == 2) { + // ID: 24-bits in little-endian byte order. + // This is inconsistent with other values in SPDY and + // is the result of a bug in the initial v2 implementation. + appender.writeLE(flagsAndLength(0, *settingId)); + } else { + appender.writeBE(flagsAndLength(0, *settingId)); + } + appender.writeBE(setting.value); + } + size_t written = frame->length(); + writeBuf.append(std::move(frame)); + return written; +} + +size_t SPDYCodec::generateWindowUpdate(folly::IOBufQueue& writeBuf, + StreamID stream, + uint32_t delta) { + if (versionSettings_.majorVersion < 3 || + (stream == 0 && versionSettings_.majorVersion == 3 && + versionSettings_.minorVersion == 0)) { + return 0; + } + + VLOG(4) << "generating window update for stream=" << stream + << ": Processed " << delta << " bytes"; + unique_ptr frame = IOBuf::create(kFrameSizeControlCommon + + kFrameSizeWindowUpdate); + Appender appender(frame.get(), 0); + appender.writeBE(versionSettings_.controlVersion); + appender.writeBE(uint16_t(spdy::WINDOW_UPDATE)); + appender.writeBE(flagsAndLength(0, kFrameSizeWindowUpdate)); + appender.writeBE(uint32_t(stream)); // TODO: ensure stream < 2^31 + appender.writeBE(delta); // TODO: delta should never be bigger than 2^31 + size_t written = frame->length(); + writeBuf.append(std::move(frame)); + return written; +} + +void SPDYCodec::enableDoubleGoawayDrain() { + CHECK_EQ(sessionClosing_, ClosingState::OPEN); + sessionClosing_ = ClosingState::OPEN_WITH_GRACEFUL_DRAIN_ENABLED; +} + +uint8_t SPDYCodec::getVersion() const { + return versionSettings_.majorVersion; +} + +uint8_t SPDYCodec::getMinorVersion() const { + return versionSettings_.minorVersion; +} + +size_t SPDYCodec::generateDataFrame(folly::IOBufQueue& writeBuf, + uint32_t streamID, uint8_t flags, uint32_t length) { + const size_t frameSize = kFrameSizeDataCommon; + unique_ptr frame = IOBuf::create(frameSize); + frame->append(frameSize); + RWPrivateCursor cursor(frame.get()); + cursor.writeBE(uint32_t(streamID)); + cursor.writeBE(flagsAndLength(flags, length)); + size_t encodedSize = frame->length(); + writeBuf.append(std::move(frame)); + return encodedSize; +} + +unique_ptr +SPDYCodec::parseHeaders(TransportDirection direction, StreamID streamID, + StreamID assocStreamID, + const HeaderPieceList& inHeaders) { + unique_ptr msg(new HTTPMessage()); + HTTPHeaders& headers = msg->getHeaders(); + bool newStream = (type_ != spdy::HEADERS); + + bool hasScheme = false; + bool hasPath = false; + + // Number of fields must be even + CHECK((inHeaders.size() & 1) == 0); + for (unsigned i = 0; i < inHeaders.size(); i += 2) { + uint8_t off = 0; + uint32_t len = inHeaders[i].str.size(); + if (len > 1 && inHeaders[i].str[0] == ':') { + off = 1; // also signals control header + len--; + } + folly::StringPiece name(inHeaders[i].str, off, len); + folly::StringPiece value = inHeaders[i + 1].str; + VLOG(5) << "Header " << name << ": " << value; + bool nameOk = SPDYUtil::validateHeaderName(name); + bool valueOk = false; + bool isPath = false; + bool isMethod = false; + if (nameOk) { + if ((version_ == 2 && name == "url") || + (version_ == 3 && off && name == "path")) { + valueOk = SPDYUtil::validateURL(value); + isPath = true; + hasPath = true; + } else if ((version_ == 2 || off) && name == "method") { + valueOk = SPDYUtil::validateMethod(value); + isMethod = true; + if (value == "CONNECT") { + // We don't support CONNECT request for SPDY + valueOk = false; + } + } else { + valueOk = SPDYUtil::validateHeaderValue(value, SPDYUtil::STRICT); + } + } + if (!nameOk || !valueOk) { + if (newStream) { + if (assocStreamID) { + callback_->onPushMessageBegin(streamID, assocStreamID, nullptr); + } else { + callback_->onMessageBegin(streamID, nullptr); + } + } + headers.add(name, value); + partialMsg_ = std::move(msg); + throw SPDYStreamFailed(false, streamID, 400, "Bad header value"); + } + bool add = false; + if (off || version_ == 2) { + if (isMethod) { + msg->setMethod(value); + } else if (isPath) { + msg->setURL(value.str()); + } else if (name == "version") { + if (caseInsensitiveEqual(value, "http/1.0")) { + msg->setHTTPVersion(1, 0); + } else { + msg->setHTTPVersion(1, 1); + } + } else if (version_ == 3 && name == "host") { + headers.add(HTTP_HEADER_HOST, value.str()); + } else if (name == "scheme") { + hasScheme = true; + if (value == "https") { + msg->setSecure(true); + } + } else if (name == "status") { + if (direction == TransportDirection::UPSTREAM && !assocStreamID) { + folly::StringPiece codePiece; + folly::StringPiece reasonPiece; + if (value.contains(' ')) { + folly::split(' ', value, codePiece, reasonPiece); + } else { + codePiece = value; + } + int32_t code = -1; + try { + code = folly::to(codePiece); + } catch (const std::range_error& ex) { + } + if (code >= 100 && code <= 999) { + msg->setStatusCode(code); + msg->setStatusMessage(reasonPiece.str()); + } else { + msg->setStatusCode(0); + headers.add(name, value); + partialMsg_ = std::move(msg); + throw SPDYStreamFailed(newStream, streamID, + spdy::RST_PROTOCOL_ERROR, + "Invalid status code"); + } + } else if (!assocStreamID) { + if (version_ == 2) { + headers.add("Status", value); + } + } // else eat the status header because it fails a check in HTTPMessage + } else if (version_ == 2) { + add = true; + } + } else { + add = true; + } + if (add) { + if (!inHeaders[i].isMultiValued() && headers.exists(name)) { + headers.add(name, value); + partialMsg_ = std::move(msg); + throw SPDYStreamFailed(newStream, streamID, spdy::RST_PROTOCOL_ERROR, + "Duplicate header value"); + } + headers.add(name, value); + } + } + if (assocStreamID && + (!headers.exists(HTTP_HEADER_HOST) || !hasScheme || !hasPath)) { + // Fail a server push without host, scheme or path headers + throw SPDYStreamFailed(newStream, streamID, 400, "Bad Request"); + } + if (direction == TransportDirection::DOWNSTREAM) { + if (version_ == 2 && !headers.exists(HTTP_HEADER_HOST)) { + ParseURL url(msg->getURL()); + if (url.valid()) { + headers.add(HTTP_HEADER_HOST, url.hostAndPort()); + } + } + + const string& accept_encoding = + headers.getSingleOrEmpty(HTTP_HEADER_ACCEPT_ENCODING); + if (accept_encoding.empty()) { + headers.add(HTTP_HEADER_ACCEPT_ENCODING, "gzip, deflate"); + } else { + bool hasGzip = false; + bool hasDeflate = false; + if (!SPDYUtil::hasGzipAndDeflate(accept_encoding, hasGzip, hasDeflate)) { + string new_encoding = accept_encoding; + if (!hasGzip) { + new_encoding.append(", gzip"); + } + if (!hasDeflate) { + new_encoding.append(", deflate"); + } + headers.set(HTTP_HEADER_ACCEPT_ENCODING, new_encoding); + } + } + } + return std::move(msg); +} + +void SPDYCodec::onSynCommon(StreamID streamID, + StreamID assocStreamID, + const HeaderPieceList& headers, + int8_t pri, + const HTTPHeaderSize& size) { + if (version_ != versionSettings_.majorVersion) { + LOG(ERROR) << "Invalid version=" << version_; + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } + + unique_ptr msg = parseHeaders(transportDirection_, + streamID, assocStreamID, headers); + msg->setIngressHeaderSize(size); + + msg->setSPDY(version_); + msg->setPriority(pri); + if (assocStreamID) { + callback_->onPushMessageBegin(streamID, assocStreamID, msg.get()); + } else { + callback_->onMessageBegin(streamID, msg.get()); + } + + callback_->onHeadersComplete(streamID, std::move(msg)); +} + +void SPDYCodec::onSynStream(uint32_t assocStream, + uint8_t pri, uint8_t slot, + const HeaderPieceList& headers, + const HTTPHeaderSize& size) { + VLOG(4) << "Got SYN_STREAM, stream=" << streamId_ + << " pri=" << folly::to(pri); + if (printer_) { + printSynStream(streamId_, assocStream, pri, slot, headers); + } + if (sessionClosing_ == ClosingState::CLOSING) { + VLOG(4) << "Dropping SYN_STREAM after final GOAWAY, stream=" << streamId_; + // Suppress any EOM callback for the current frame. + flags_ &= ~spdy::CTRL_FLAG_FIN; + return; + } + if (streamId_ == 0 || + streamId_ < lastStreamID_ || + (transportDirection_ == TransportDirection::UPSTREAM && + (streamId_ & 0x01) == 1) || + (transportDirection_ == TransportDirection::DOWNSTREAM && + ((streamId_ & 0x1) == 0)) || + (transportDirection_ == TransportDirection::UPSTREAM && + assocStream == 0)) { + LOG(ERROR) << " invalid syn stream stream_id=" << streamId_ + << " lastStreamID_=" << lastStreamID_ + << " assocStreamID=" << assocStream + << " direction=" << transportDirection_; + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } + + if (streamId_ == lastStreamID_) { + throw SPDYStreamFailed(true, streamId_, spdy::RST_PROTOCOL_ERROR); + } + if (callback_->numIncomingStreams() >= + egressSettings_.getSetting(SettingsId::MAX_CONCURRENT_STREAMS, + spdy::kMaxConcurrentStreams)) { + throw SPDYStreamFailed(true, streamId_, spdy::RST_REFUSED_STREAM); + } + if (assocStream != 0 && !(flags_ & spdy::CTRL_FLAG_UNIDIRECTIONAL)) { + throw SPDYStreamFailed(true, streamId_, spdy::RST_PROTOCOL_ERROR); + } + lastStreamID_ = streamId_; + onSynCommon(StreamID(streamId_), + StreamID(assocStream), headers, pri, size); +} + +void SPDYCodec::onSynReply(const HeaderPieceList& headers, + const HTTPHeaderSize& size) { + VLOG(4) << "Got SYN_REPLY, stream=" << streamId_; + if (printer_) { + printSynReply(streamId_, headers); + } + if (transportDirection_ == TransportDirection::DOWNSTREAM || + (streamId_ & 0x1) == 0) { + throw SPDYStreamFailed(true, streamId_, spdy::RST_PROTOCOL_ERROR); + } + // Server push transactions, short of any better heuristics, + // should have a background priority. Thus, we pick the largest + // numerical value for the SPDY priority, which no matter what + // protocol version this is can be conveyed to onSynCommon by -1. + onSynCommon(StreamID(streamId_), + HTTPCodec::NoStream, headers, -1, size); +} + +void SPDYCodec::onRstStream(uint32_t statusCode) noexcept { + VLOG(4) << "Got RST_STREAM, stream=" << streamId_ + << ", status=" << statusCode; + if (printer_) { + printRstStream(streamId_, statusCode); + } + StreamID streamID(streamId_); + callback_->onAbort(streamID, + spdy::rstToErrorCode(spdy::ResetStatusCode(statusCode))); +} + +void SPDYCodec::onSettings(const SettingList& settings) { + VLOG(4) << "Got " << settings.size() << " settings with " + << "version=" << version_ << " and flags=" + << std::hex << folly::to(flags_) << std::dec; + if (printer_) { + printSettings(settings); + } + SettingsList settingsList; + for (const SettingData& cur: settings) { + // For now, we never ask for anything to be persisted, so ignore anything + // coming back + if (cur.flags & spdy::ID_FLAG_SETTINGS_PERSISTED) { + VLOG(2) << "Ignoring bogus persisted setting: " << cur.id; + continue; + } + + switch (cur.id) { + case spdy::SETTINGS_UPLOAD_BANDWIDTH: + case spdy::SETTINGS_DOWNLOAD_BANDWIDTH: + case spdy::SETTINGS_ROUND_TRIP_TIME: + case spdy::SETTINGS_CURRENT_CWND: + case spdy::SETTINGS_DOWNLOAD_RETRANS_RATE: + case spdy::SETTINGS_CLIENT_CERTIFICATE_VECTOR_SIZE: + // These will be stored in ingressSettings_ and passed to the callback + // but we currently ignore the PERSIST flag + break; + case spdy::SETTINGS_MAX_CONCURRENT_STREAMS: + break; + case spdy::SETTINGS_INITIAL_WINDOW_SIZE: + if (cur.value > std::numeric_limits::max()) { + throw SPDYSessionFailed(spdy::GOAWAY_PROTOCOL_ERROR); + } + break; + default: + LOG(ERROR) << "Received unknown setting with ID=" << cur.id + << ", value=" << cur.value + << ", and flags=" << std::hex << cur.flags << std::dec; + } + auto id = spdy::spdyToHttpSettingsId((spdy::SettingsId)cur.id); + if (id) { + ingressSettings_.setSetting(*id, cur.value); + auto s = ingressSettings_.getSetting(*id); + settingsList.push_back(*s); + } + } + callback_->onSettings(settingsList); +} + +void SPDYCodec::onPing(uint32_t uniqueID) noexcept { + if (printer_) { + printPing(uniqueID); + } + bool odd = uniqueID & 0x1; + bool isReply = true; + if (transportDirection_ == TransportDirection::DOWNSTREAM) { + if (odd) { + isReply = false; + } + } else if (!odd) { + isReply = false; + } + + if (isReply) { + if (uniqueID >= nextEgressPingID_) { + LOG(INFO) << "Received reply for pingID=" << uniqueID + << " that was never sent"; + return; + } + callback_->onPingReply(uniqueID); + } else { + callback_->onPingRequest(uniqueID); + } +} + +void SPDYCodec::onGoaway(uint32_t lastGoodStream, + uint32_t statusCode) noexcept { + if (printer_) { + printGoaway(lastGoodStream, statusCode); + } + VLOG(4) << "Got GOAWAY, lastGoodStream=" << lastGoodStream + << ", statusCode=" << statusCode; + + if (lastGoodStream < ingressGoawayAck_) { + ingressGoawayAck_ = lastGoodStream; + // Drain all streams <= lastGoodStream + // and abort streams > lastGoodStream + callback_->onGoaway(lastGoodStream, spdy::goawayToErrorCode( + spdy::GoawayStatusCode(statusCode))); + } else { + LOG(WARNING) << "Received multiple GOAWAY with increasing ack"; + } +} + +void SPDYCodec::onHeaders(const HeaderPieceList& headers) noexcept { + if (printer_) { + printHeaders(streamId_, headers); + } + VLOG(3) << "onHeaders is unimplemented."; +} + +void SPDYCodec::onWindowUpdate(uint32_t delta) noexcept { + if (printer_) { + printWindowUpdate(streamId_, delta); + } + callback_->onWindowUpdate(streamId_, delta); +} + +void SPDYCodec::failStream(bool newStream, StreamID streamID, + uint32_t code, string excStr) { + // Suppress any EOM callback for the current frame. + if (streamID == streamId_) { + flags_ &= ~spdy::CTRL_FLAG_FIN; + } + + HTTPException err( + code >= 100 ? + HTTPException::Direction::INGRESS : + HTTPException::Direction::INGRESS_AND_EGRESS, + "SPDYCodec stream error: stream=", + streamID, " status=", code, " exception: ", excStr); + if (code >= 100) { + err.setHttpStatusCode(code); + } else { + err.setCodecStatusCode(spdy::rstToErrorCode(spdy::ResetStatusCode(code))); + } + err.setProxygenError(kErrorParseHeader); + + if (partialMsg_) { + err.setPartialMsg(std::move(partialMsg_)); + } + // store the ingress buffer + if (currentIngressBuf_) { + err.setCurrentIngressBuf(std::move(currentIngressBuf_->clone())); + } + callback_->onError(streamID, err, newStream); +} + +void SPDYCodec::failSession(uint32_t code) { + HTTPException err( + HTTPException::Direction::INGRESS_AND_EGRESS, + "SPDYCodec session error " + "lastGoodStream=", lastStreamID_, " status=", code); + err.setCodecStatusCode(spdy::goawayToErrorCode(spdy::GoawayStatusCode(code))); + err.setProxygenError(kErrorParseHeader); + + // store the ingress buffer + if (currentIngressBuf_) { + err.setCurrentIngressBuf(std::move(currentIngressBuf_->clone())); + } + callback_->onError(0, err); +} + +bool SPDYCodec::rstStatusSupported(int statusCode) const { + if (statusCode == 0) { + // 0 is not a valid status code for RST_STREAM + return false; + } + // SPDY/3 supports more status codes for RST_STREAM. For SPDY/2, + // we just use PROTOCOL_ERROR for these new higher numbered error codes. + return (versionSettings_.majorVersion != 2 || + statusCode <= spdy::RST_FLOW_CONTROL_ERROR); +} + +boost::optional +SPDYCodec::getVersion(const std::string& protocol) { + // Fail fast if it's not possible for the protocol string to define a + // SPDY protocol. strlen("spdy/1") == 6 + if (protocol.length() < 6) { + return boost::none; + } + + if (protocol == getHpackNpn()) { + return SPDYVersion::SPDY3_1_HPACK; + } + if (protocol == "spdy/3.1") { + return SPDYVersion::SPDY3_1; + } + if (protocol == "spdy/3") { + return SPDYVersion::SPDY3; + } + if (protocol == "spdy/2") { + return SPDYVersion::SPDY2; + } + + return boost::none; +} + +} diff --git a/proxygen/lib/http/codec/SPDYCodec.h b/proxygen/lib/http/codec/SPDYCodec.h new file mode 100644 index 0000000000..153a5d958b --- /dev/null +++ b/proxygen/lib/http/codec/SPDYCodec.h @@ -0,0 +1,378 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPHeaders.h" +#include "proxygen/lib/http/codec/HTTPCodec.h" +#include "proxygen/lib/http/codec/HTTPSettings.h" +#include "proxygen/lib/http/codec/SPDYConstants.h" +#include "proxygen/lib/http/codec/SPDYVersionSettings.h" +#include "proxygen/lib/http/codec/compress/HPACKCodec.h" +#include "proxygen/lib/http/codec/compress/HeaderCodec.h" + +#include +#include +#include +#include + +namespace folly { namespace io { +class Cursor; +}} + +namespace proxygen { + +/** + * An implementation of the framing layer for all versions of + * SPDY. Instances of this class must not be used from multiple threads + * concurrently. + */ +class SPDYCodec: public HTTPCodec { +public: + explicit SPDYCodec(TransportDirection direction, + SPDYVersion version, + int spdyCompressionLevel = Z_NO_COMPRESSION); + ~SPDYCodec() override; + + static const SPDYVersionSettings& getVersionSettings(SPDYVersion version); + + static const std::string& getHpackNpn(); + + // HTTPCodec API + CodecProtocol getProtocol() const override; + TransportDirection getTransportDirection() const override { + return transportDirection_; + } + bool supportsStreamFlowControl() const override; + bool supportsSessionFlowControl() const override; + StreamID createStream() override; + void setCallback(Callback* callback) override { callback_ = callback; } + bool isBusy() const override; + void setParserPaused(bool paused) override; + size_t onIngress(const folly::IOBuf& buf) override; + void onIngressEOF() override; + bool isReusable() const override; + bool isWaitingToDrain() const override; + bool closeOnEgressComplete() const override { return false; } + bool supportsParallelRequests() const override { return true; } + bool supportsPushTransactions() const override { return true; } + void generateHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPMessage& msg, + StreamID assocStream = 0, + HTTPHeaderSize* size = nullptr) override; + size_t generateBody(folly::IOBufQueue& writeBuf, + StreamID stream, + std::unique_ptr chain, + bool eom) override; + size_t generateChunkHeader(folly::IOBufQueue& writeBuf, + StreamID stream, + size_t length) override; + size_t generateChunkTerminator(folly::IOBufQueue& writeBuf, + StreamID stream) override; + size_t generateTrailers(folly::IOBufQueue& writeBuf, + StreamID stream, + const HTTPHeaders& trailers) override; + size_t generateEOM(folly::IOBufQueue& writeBuf, + StreamID stream) override; + size_t generateRstStream(folly::IOBufQueue& writeBuf, + StreamID txn, + ErrorCode statusCode) override; + size_t generateGoaway(folly::IOBufQueue& writeBuf, + StreamID lastStream, + ErrorCode statusCode) override; + size_t generatePingRequest(folly::IOBufQueue& writeBuf) override; + size_t generatePingReply(folly::IOBufQueue& writeBuf, + uint64_t uniqueID) override; + size_t generateSettings(folly::IOBufQueue& writeBuf) override; + size_t generateWindowUpdate(folly::IOBufQueue& writeBuf, + StreamID stream, + uint32_t delta) override; + void enableDoubleGoawayDrain() override; + StreamID getLastIncomingStreamID() const override { return lastStreamID_; } + + // SPDYCodec specific API + void setPrinter(bool printer) { printer_ = printer; } + /** + * Returns a const reference to the ingress settings. Since ingress + * settings are set by the remote end, it doesn't make sense for these + * to be mutable outside the codec. + */ + const HTTPSettings* getIngressSettings() const { return &ingressSettings_; } + /** + * Returns a reference to the egress settings + */ + HTTPSettings* getEgressSettings() { return &egressSettings_; } + + uint8_t getVersion() const; + + uint8_t getMinorVersion() const; + + void setMaxFrameLength(uint32_t maxFrameLength); + + /** + * Set the maximum size of the uncompressed headers + */ + void setMaxUncompressedHeaders(uint32_t maxUncompressed); + + void setHeaderCodecStats(HeaderCodec::Stats* stats) override { + headerCodec_->setStats(stats); + } + + struct SettingData { + SettingData(uint8_t inFlags, uint32_t inId, uint32_t inValue) + : flags(inFlags), + id(inId), + value(inValue) {} + uint8_t flags; + uint32_t id; + uint32_t value; + }; + + typedef std::vector SettingList; + + /** + * Returns the SPDYVersion for the given protocol string, or none otherwise. + */ + static boost::optional getVersion(const std::string& protocol); + + private: + + /** + * Determines whether header with a given code is on the SPDY per-hop + * header blacklist. + */ + static std::bitset<256> perHopHeaderCodes_; + + static void initPerHopHeaders() __attribute__ ((__constructor__)); + + // SPDY Frame parsing state + enum class FrameState { + FRAME_HEADER = 0, + CTRL_FRAME_DATA = 1, + DATA_FRAME_DATA = 2, + }; + + /** + * Generates a frame of type SYN_STREAM + */ + void generateSynStream(StreamID stream, + StreamID assocStream, + folly::IOBufQueue& writeBuf, + const HTTPMessage& msg, + HTTPHeaderSize* size); + /** + * Generates a frame of type SYN_REPLY + */ + void generateSynReply(StreamID stream, + folly::IOBufQueue& writeBuf, + const HTTPMessage& msg, + HTTPHeaderSize* size); + + /** + * Generates the shared parts of a ping request and reply. + */ + size_t generatePingCommon(folly::IOBufQueue& writeBuf, + uint64_t uniqueID); + /** + * Ingress parser, can throw exceptions + */ + size_t parseIngress(const folly::IOBuf& buf); + + /** + * Handle an ingress SYN_STREAM control frame. For a downstream-facing + * SPDY session, this frame is the equivalent of an HTTP request header. + */ + void onSynStream(uint32_t assocStream, + uint8_t pri, uint8_t slot, + const compress::HeaderPieceList& headers, + const HTTPHeaderSize& size); + /** + * Handle an ingress SYN_REPLY control frame. For an upstream-facing + * SPDY session, this frame is the equivalent of an HTTP response header. + */ + void onSynReply(const compress::HeaderPieceList& headers, + const HTTPHeaderSize& size); + /** + * Handle an ingress RST_STREAM control frame. + */ + void onRstStream(uint32_t statusCode) noexcept; + /** + * Handle a SETTINGS message that changes/updates settings for the + * entire SPDY connection (across all transactions) + */ + void onSettings(const SettingList& settings); + + void onPing(uint32_t uniqueID) noexcept; + + void onGoaway(uint32_t lastGoodStream, + uint32_t statusCode) noexcept; + /** + * Handle a HEADERS frame. This is *not* invoked when the first headers + * on a stream are received. This is called when the remote endpoint + * sends us any additional headers. + */ + void onHeaders(const compress::HeaderPieceList& headers) noexcept; + + void onWindowUpdate(uint32_t delta) noexcept; + + // Helpers + + /** + * Parses the headers in the nameValues array and creates an HTTPMessage + * object initialized for this transaction. + */ + std::unique_ptr parseHeaders( + TransportDirection direction, StreamID streamID, + StreamID assocStreamID, const compress::HeaderPieceList& headers); + + /** + * Helper function to parse out a control frame and execute its handler. + * All errors are thrown as exceptions. + */ + void onControlFrame(folly::io::Cursor& cursor); + + /** + * Helper function that contains the common implementation details of + * calling the same callbacks for onSynStream() and onSynReply() + * + * Negative values of pri are interpreted much like negative array + * indexes in python, so -1 will be the largest numerical priority + * value for this SPDY version (i.e., 3 for SPDY/2 or 7 for SPDY/3), + * -2 the second largest (i.e., 2 for SPDY/2 or 6 for SPDY/3). + */ + void onSynCommon(StreamID streamID, + StreamID assocStreamID, + const compress::HeaderPieceList& headers, + int8_t pri, + const HTTPHeaderSize& size); + + /** + * Generate the header for a SPDY data frame + * @param writeBuf Buffer queue to which the control frame is written. + * @param streamID Stream ID. + * @param flags Bitmap of flags, as defined in the SPDY spec. + * @param length Length of the data, in bytes. + * @return length Length of the encoded bytes + */ + size_t generateDataFrame(folly::IOBufQueue& writeBuf, + uint32_t streamID, + uint8_t flags, + uint32_t length); + + /** + * Serializes headers for requests (aka SYN_STREAM) + * @param msg The message to serialize. + * @param isPushed true if this is a push message + * @param size Size of the serialized headers before and after compression + * @param headroom Optional amount of headroom to reserve at the + * front of the returned IOBuf, in case the caller + * wants to put some other data there. + */ + std::unique_ptr serializeRequestHeaders( + const HTTPMessage& msg, + bool isPushed, + uint32_t headroom = 0, + HTTPHeaderSize* size = nullptr); + + /** + * Serializes headers for responses (aka SYN_REPLY) + * @param msg The message to serialize. + * @param size Size of the serialized headers before and after compression + * @param headroom Optional amount of headroom to reserve at the + * front of the returned IOBuf, in case the caller + * wants to put some other data there. + */ + std::unique_ptr serializeResponseHeaders( + const HTTPMessage& msg, + uint32_t headroom = 0, + HTTPHeaderSize* size = nullptr); + + /** + * Helper function to create the compressed Name/Value representation of + * a message's headers. + * @param msg The message containing headers to serialize. + * @param headers A vector containing any extra headers to serialize + * @param size Size of the serialized headers before and after compression + * @param headroom Optional amount of headroom to reserve at the + * front of the returned IOBuf, in case the caller + * wants to put some other data there. + */ + std::unique_ptr encodeHeaders( + const HTTPMessage& msg, std::vector& headers, + uint32_t headroom = 0, + HTTPHeaderSize* size = nullptr); + + void failStream(bool newTxn, StreamID streamID, uint32_t code, + std::string excStr = empty_string); + + void failSession(uint32_t statusCode); + + /** + * Decodes the headers from the cursor and returns the result. + */ + HeaderDecodeResult decodeHeaders(folly::io::Cursor& cursor); + + void checkLength(uint32_t expectedLength, const std::string& msg); + + void checkMinLength(uint32_t minLength, const std::string& msg); + + bool isSPDYReserved(const std::string& name); + + /** + * Helper function to check if the status code is supported by the + * SPDY version being used + */ + bool rstStatusSupported(int statusCode) const; + + HTTPCodec::Callback* callback_; + TransportDirection transportDirection_; + StreamID nextEgressStreamID_; + StreamID nextEgressPingID_; + StreamID lastStreamID_; + // StreamID's are 31 bit unsigned integers, so all received goaways will + // be lower than this. + StreamID ingressGoawayAck_{std::numeric_limits::max()}; + folly::fbvector closedStreams_; + const SPDYVersionSettings& versionSettings_; + uint32_t maxFrameLength_; + + const folly::IOBuf* currentIngressBuf_; + std::unique_ptr partialMsg_; + + HTTPSettings ingressSettings_{ + {SettingsId::MAX_CONCURRENT_STREAMS, spdy::kMaxConcurrentStreams}, + {SettingsId::INITIAL_WINDOW_SIZE, spdy::kInitialWindow} + }; + HTTPSettings egressSettings_{ + {SettingsId::MAX_CONCURRENT_STREAMS, spdy::kMaxConcurrentStreams}, + {SettingsId::INITIAL_WINDOW_SIZE, spdy::kInitialWindow} + }; + + FrameState frameState_; + uint16_t version_; + uint16_t type_; + uint32_t streamId_; + uint32_t length_; + uint8_t flags_; + + enum ClosingState { + OPEN = 0, + OPEN_WITH_GRACEFUL_DRAIN_ENABLED = 1, + FIRST_GOAWAY_SENT = 2, + CLOSING = 3, + }; + ClosingState sessionClosing_:2; + bool printer_:1; + bool ctrl_:1; + + std::unique_ptr headerCodec_; +}; + +} // proxygen diff --git a/proxygen/lib/http/codec/SPDYConstants.cpp b/proxygen/lib/http/codec/SPDYConstants.cpp new file mode 100644 index 0000000000..0dee1f9213 --- /dev/null +++ b/proxygen/lib/http/codec/SPDYConstants.cpp @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/SPDYConstants.h" + +namespace proxygen { namespace spdy { + +GoawayStatusCode errorCodeToGoaway(ErrorCode code) { + switch (code) { + case ErrorCode::NO_ERROR: return GOAWAY_OK; + case ErrorCode::INTERNAL_ERROR: return GOAWAY_INTERNAL_ERROR; + case ErrorCode::FLOW_CONTROL_ERROR: return GOAWAY_FLOW_CONTROL_ERROR; + case ErrorCode::PROTOCOL_ERROR: break; + case ErrorCode::SETTINGS_TIMEOUT: break; + case ErrorCode::STREAM_CLOSED: break; + case ErrorCode::FRAME_SIZE_ERROR: break; + case ErrorCode::REFUSED_STREAM: break; + case ErrorCode::CANCEL: break; + case ErrorCode::COMPRESSION_ERROR: break; + case ErrorCode::CONNECT_ERROR: break; + case ErrorCode::ENHANCE_YOUR_CALM: break; + case ErrorCode::INADEQUATE_SECURITY: break; + case ErrorCode::_SPDY_INVALID_STREAM: break; + } + return GOAWAY_PROTOCOL_ERROR; +} + +ResetStatusCode errorCodeToReset(ErrorCode code) { + switch (code) { + case ErrorCode::NO_ERROR: break; + case ErrorCode::INTERNAL_ERROR: return RST_INTERNAL_ERROR; + case ErrorCode::FLOW_CONTROL_ERROR: return RST_FLOW_CONTROL_ERROR; + case ErrorCode::PROTOCOL_ERROR: return RST_PROTOCOL_ERROR; + case ErrorCode::SETTINGS_TIMEOUT: break; + case ErrorCode::STREAM_CLOSED: return RST_STREAM_ALREADY_CLOSED; + case ErrorCode::FRAME_SIZE_ERROR: return RST_FRAME_TOO_LARGE; + case ErrorCode::REFUSED_STREAM: return RST_REFUSED_STREAM; + case ErrorCode::CANCEL: return RST_CANCEL; + case ErrorCode::COMPRESSION_ERROR: return RST_INTERNAL_ERROR; + case ErrorCode::CONNECT_ERROR: break; + case ErrorCode::ENHANCE_YOUR_CALM: break; + case ErrorCode::INADEQUATE_SECURITY: return RST_INVALID_CREDENTIALS; + case ErrorCode::_SPDY_INVALID_STREAM: return RST_INVALID_STREAM; + } + return RST_PROTOCOL_ERROR; +} + +ErrorCode goawayToErrorCode(GoawayStatusCode code) { + switch (code) { + case GOAWAY_OK: return ErrorCode::NO_ERROR; + case GOAWAY_PROTOCOL_ERROR: return ErrorCode::PROTOCOL_ERROR; + case GOAWAY_INTERNAL_ERROR: return ErrorCode::INTERNAL_ERROR; + case GOAWAY_FLOW_CONTROL_ERROR: return ErrorCode::FLOW_CONTROL_ERROR; + } + return ErrorCode::PROTOCOL_ERROR; +} + +ErrorCode rstToErrorCode(ResetStatusCode code) { + switch (code) { + case RST_PROTOCOL_ERROR: break; + case RST_INVALID_STREAM: return ErrorCode::_SPDY_INVALID_STREAM; + case RST_REFUSED_STREAM: return ErrorCode::REFUSED_STREAM; + case RST_UNSUPPORTED_VERSION: break; // not used anyway + case RST_CANCEL: return ErrorCode::CANCEL; + case RST_INTERNAL_ERROR: return ErrorCode::INTERNAL_ERROR; + case RST_FLOW_CONTROL_ERROR: return ErrorCode::FLOW_CONTROL_ERROR; + case RST_STREAM_IN_USE: return ErrorCode::FLOW_CONTROL_ERROR; + case RST_STREAM_ALREADY_CLOSED: return ErrorCode::STREAM_CLOSED; + case RST_INVALID_CREDENTIALS: return ErrorCode::INADEQUATE_SECURITY; + case RST_FRAME_TOO_LARGE: return ErrorCode::FRAME_SIZE_ERROR; + } + return ErrorCode::PROTOCOL_ERROR; +} + +boost::optional httpToSpdySettingsId( + proxygen::SettingsId id) { + switch (id) { + // no mapping + case proxygen::SettingsId::HEADER_TABLE_SIZE: + case proxygen::SettingsId::ENABLE_PUSH: + case proxygen::SettingsId::MAX_FRAME_SIZE: + case proxygen::SettingsId::MAX_HEADER_LIST_SIZE: + return boost::none; + case proxygen::SettingsId::MAX_CONCURRENT_STREAMS: + return SETTINGS_MAX_CONCURRENT_STREAMS; + case proxygen::SettingsId::INITIAL_WINDOW_SIZE: + return SETTINGS_INITIAL_WINDOW_SIZE; + case proxygen::SettingsId::_SPDY_UPLOAD_BANDWIDTH: + return SETTINGS_UPLOAD_BANDWIDTH; + case proxygen::SettingsId::_SPDY_DOWNLOAD_BANDWIDTH: + return SETTINGS_DOWNLOAD_BANDWIDTH; + case proxygen::SettingsId::_SPDY_ROUND_TRIP_TIME: + return SETTINGS_ROUND_TRIP_TIME; + case proxygen::SettingsId::_SPDY_CURRENT_CWND: + return SETTINGS_CURRENT_CWND; + case proxygen::SettingsId::_SPDY_DOWNLOAD_RETRANS_RATE: + return SETTINGS_DOWNLOAD_RETRANS_RATE; + case proxygen::SettingsId::_SPDY_CLIENT_CERTIFICATE_VECTOR_SIZE: + return SETTINGS_CLIENT_CERTIFICATE_VECTOR_SIZE; + } + return boost::none; +} + +boost::optional spdyToHttpSettingsId( + proxygen::spdy::SettingsId id) { + switch (id) { + case SETTINGS_UPLOAD_BANDWIDTH: + case SETTINGS_DOWNLOAD_BANDWIDTH: + case SETTINGS_ROUND_TRIP_TIME: + case SETTINGS_CURRENT_CWND: + case SETTINGS_DOWNLOAD_RETRANS_RATE: + case SETTINGS_CLIENT_CERTIFICATE_VECTOR_SIZE: + // These mappings are possible, but not needed right now + return boost::none; + case SETTINGS_MAX_CONCURRENT_STREAMS: + return proxygen::SettingsId::MAX_CONCURRENT_STREAMS; + case SETTINGS_INITIAL_WINDOW_SIZE: + return proxygen::SettingsId::INITIAL_WINDOW_SIZE; + } + return boost::none; +} + +const uint32_t kInitialWindow = 65536; +const uint32_t kMaxConcurrentStreams = 100; +const uint32_t kMaxFrameLength = (1 << 24) - 1; + +const std::string kSessionProtoNameSPDY2("spdy/2"); +const std::string kSessionProtoNameSPDY3("spdy/3"); + +const std::string httpVersion("HTTP/1.1"); +const std::string kNameVersionv2("version"); +const std::string kNameVersionv3(":version"); +const std::string kNameStatusv2("status"); +const std::string kNameStatusv3(":status"); +const std::string kNameMethodv2("method"); +const std::string kNameMethodv3(":method"); +const std::string kNamePathv2("url"); +const std::string kNamePathv3(":path"); +const std::string kNameSchemev2("scheme"); +const std::string kNameSchemev3(":scheme"); +const std::string kNameHostv3(":host"); // SPDY v3 only +const std::string https("https"); +const std::string http("http"); + +// In the future, we may be shifting the SPDY wire priority +// by this much so we can easily use the lower bits to do our +// own priority queueing within the bands defined by the SPDY +// protocol... +// +// so far: +// +// lower 2 LSB: used to randomly approximate some fairness within +// priority bands, relying on the poisson events of extracting or +// appending a frame to gather "entropy". +const size_t SPDY_PRIO_SHIFT_FACTOR = 2; // up to 60 + +}} diff --git a/proxygen/lib/http/codec/SPDYConstants.h b/proxygen/lib/http/codec/SPDYConstants.h new file mode 100644 index 0000000000..46f02fa15d --- /dev/null +++ b/proxygen/lib/http/codec/SPDYConstants.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/ErrorCode.h" +#include "proxygen/lib/http/codec/SettingsId.h" + +#include +#include + +namespace proxygen { namespace spdy { + +enum FrameType { + SYN_STREAM = 1, + SYN_REPLY = 2, + RST_STREAM = 3, + SETTINGS = 4, + NOOP = 5, + PING = 6, + GOAWAY = 7, + HEADERS = 8, + WINDOW_UPDATE = 9, + // The CREDENTIAL frame is removed in SPDY/3.1 + CREDENTIAL = 10 +}; + +enum CtrlFlag { + CTRL_FLAG_NONE = 0, + CTRL_FLAG_FIN = 1, + CTRL_FLAG_UNIDIRECTIONAL = 2 +}; + +enum SettingsFlag { + FLAG_SETTINGS_NONE = 0, + FLAG_SETTINGS_CLEAR_SETTINGS = 1 +}; + +enum SettingsIdFlag { + ID_FLAG_SETTINGS_NONE = 0, + ID_FLAG_SETTINGS_PERSIST_VALUE = 1, + ID_FLAG_SETTINGS_PERSISTED = 2 +}; + +enum SettingsId { + SETTINGS_UPLOAD_BANDWIDTH = 1, + SETTINGS_DOWNLOAD_BANDWIDTH = 2, + SETTINGS_ROUND_TRIP_TIME = 3, + SETTINGS_MAX_CONCURRENT_STREAMS = 4, + SETTINGS_CURRENT_CWND = 5, + SETTINGS_DOWNLOAD_RETRANS_RATE = 6, + SETTINGS_INITIAL_WINDOW_SIZE = 7, + SETTINGS_CLIENT_CERTIFICATE_VECTOR_SIZE = 8 +}; + +/** + * The status codes for the RST_STREAM control frame. + */ +enum ResetStatusCode { + RST_PROTOCOL_ERROR = 1, + RST_INVALID_STREAM = 2, + RST_REFUSED_STREAM = 3, + RST_UNSUPPORTED_VERSION = 4, + RST_CANCEL = 5, + RST_INTERNAL_ERROR = 6, + RST_FLOW_CONTROL_ERROR = 7, + // The following status codes were added in SPDY/3 + // and are not supported in SPDY/2 + RST_STREAM_IN_USE = 8, + RST_STREAM_ALREADY_CLOSED = 9, + RST_INVALID_CREDENTIALS = 10, + RST_FRAME_TOO_LARGE = 11 +}; + +/** + * The status codes for the GOAWAY control frame. + */ +enum GoawayStatusCode { + GOAWAY_OK = 0, + GOAWAY_PROTOCOL_ERROR = 1, + GOAWAY_INTERNAL_ERROR = 2, + // Only for SPDY/3.1 for connection-level flow control errors + GOAWAY_FLOW_CONTROL_ERROR = 7 +}; + +// Functions + +extern GoawayStatusCode errorCodeToGoaway(ErrorCode code); +extern ResetStatusCode errorCodeToReset(ErrorCode code); +extern ErrorCode goawayToErrorCode(spdy::GoawayStatusCode); +extern ErrorCode rstToErrorCode(spdy::ResetStatusCode); + +boost::optional httpToSpdySettingsId( + proxygen::SettingsId id); +boost::optional spdyToHttpSettingsId( + proxygen::spdy::SettingsId id); + +// Constants + +extern const uint32_t kInitialWindow; +extern const uint32_t kMaxConcurrentStreams; +extern const uint32_t kMaxFrameLength; + +extern const std::string kSessionProtoNameSPDY2; +extern const std::string kSessionProtoNameSPDY3; + +extern const std::string httpVersion; +extern const std::string kNameVersionv2; +extern const std::string kNameVersionv3; +extern const std::string kNameStatusv2; +extern const std::string kNameStatusv3; +extern const std::string kNameMethodv2; +extern const std::string kNameMethodv3; +extern const std::string kNamePathv2; +extern const std::string kNamePathv3; +extern const std::string kNameSchemev2; +extern const std::string kNameSchemev3; +extern const std::string kNameHostv3; +extern const std::string https; +extern const std::string http; + +extern const size_t SPDY_PRIO_SHIFT_FACTOR; + +}} diff --git a/proxygen/lib/http/codec/SPDYUtil.cpp b/proxygen/lib/http/codec/SPDYUtil.cpp new file mode 100644 index 0000000000..e7e3f7b0f5 --- /dev/null +++ b/proxygen/lib/http/codec/SPDYUtil.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/SPDYUtil.h" + +#include "proxygen/lib/http/RFC2616.h" + +#include + +namespace proxygen { + +/** + * Tokens as defined by rfc 2616. Also lowercases them. + * token = 1* + * separators = "(" | ")" | "<" | ">" | "@" + * | "," | ";" | ":" | "\" | <"> + * | "/" | "[" | "]" | "?" | "=" + * | "{" | "}" | SP | HT + */ +const char SPDYUtil::http_tokens[256] = { +/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */ + 0, 0, 0, 0, 0, 0, 0, 0, +/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */ + ' ', '!', '"', '#', '$', '%', '&', '\'', +/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */ + 0, 0, '*', '+', 0, '-', '.', '/', +/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */ + '0', '1', '2', '3', '4', '5', '6', '7', +/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */ + '8', '9', 0, 0, 0, 0, 0, 0, +/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */ + 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g', +/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */ + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', +/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */ + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', +/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */ + 'x', 'y', 'z', 0, 0, 0, '^', '_', +/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */ + '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', +/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */ + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', +/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */ + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', +/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */ + 'x', 'y', 'z', 0, '|', '}', '~', 0 +}; + +bool SPDYUtil::hasGzipAndDeflate(const std::string& value, bool& hasGzip, + bool& hasDeflate) { + static folly::ThreadLocal> output; + output->clear(); + hasGzip = false; + hasDeflate = false; + RFC2616::parseQvalues(value, *output); + for (const auto& encodingQ: *output) { + std::string lower(std::move(encodingQ.first.str())); + std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); + // RFC says 3 sig figs + if (lower == "gzip" && encodingQ.second >= 0.001) { + hasGzip = true; + } else if (lower == "deflate" && encodingQ.second >= 0.001) { + hasDeflate = true; + } + } + return hasGzip && hasDeflate; +} + +} diff --git a/proxygen/lib/http/codec/SPDYUtil.h b/proxygen/lib/http/codec/SPDYUtil.h new file mode 100644 index 0000000000..7637e89c9d --- /dev/null +++ b/proxygen/lib/http/codec/SPDYUtil.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include +#include +#include + +namespace proxygen { + +class SPDYUtil { + public: + // If these are needed elsewhere, we can move them to a more generic + // namespace/class later + static const char http_tokens[256]; + + static bool validateURL(folly::ByteRange url) { + for (auto p: url) { + if (p <= 0x20 || p == 0x7f) { + // no controls or unescaped spaces + return false; + } + } + return true; + } + + static bool validateMethod(folly::ByteRange method) { + for (auto p: method) { + if (!isalpha(p)) { + // methods are all characters + return false; + } + } + return true; + } + + static bool validateHeaderName(folly::ByteRange name) { + for (auto p: name) { + if (p < 0x80 && http_tokens[(uint8_t)p] != p) { + return false; + } + } + return true; + } + + /** + * RFC2616 allows certain control chars in header values if they are + * quoted and escaped. + * When mode is COMPLIANT, then this is allowed. + * When mode is STRICT, no escaped CTLs are allowed + */ + enum CtlEscapeMode { + COMPLIANT, + STRICT + }; + + static bool validateHeaderValue(folly::ByteRange value, + CtlEscapeMode mode) { + bool escape = false; + bool quote = false; + enum { lws_none, + lws_expect_nl, + lws_expect_ws1, + lws_expect_ws2 } state = lws_none; + + for (auto p = std::begin(value); p != std::end(value); ++p) { + if (escape) { + escape = false; + if (mode == COMPLIANT) { + // prev char escaped. Turn off escape and go to next char + // COMPLIANT mode only + assert(quote); + continue; + } + } + switch (state) { + case lws_none: + switch (*p) { + case '\\': + if (quote) { + escape = true; + } + break; + case '\"': + quote = !quote; + break; + case '\r': + state = lws_expect_nl; + break; + default: + if (*p < 0x20 || *p == 0x7f) { + // unexpected ctl per rfc2616 + return false; + } + break; + } + break; + case lws_expect_nl: + if (*p != '\n') { + // unescaped \r must be LWS + return false; + } + state = lws_expect_ws1; + break; + case lws_expect_ws1: + if (*p != ' ' && *p != '\t') { + // unescaped \r\n must be LWS + return false; + } + state = lws_expect_ws2; + break; + case lws_expect_ws2: + if (*p != ' ' && *p != '\t') { + // terminated LWS + state = lws_none; + // check this char again + p--; + } + break; + } + } + // Unterminated quotes are OK, since the value can be* TEXT which treats + // the " like any other char. + // Unterminated escapes are bad because it will escape the next character + // when converting to HTTP + // Unterminated LWS (dangling \r or \r\n) is bad because it could + // prematurely terminate the headers when converting to HTTP + return !escape && (state == lws_none || state == lws_expect_ws2); + } + + static bool hasGzipAndDeflate(const std::string& value, bool& hasGzip, + bool& hasDeflate); +}; + +} diff --git a/proxygen/lib/http/codec/SPDYVersion.h b/proxygen/lib/http/codec/SPDYVersion.h new file mode 100644 index 0000000000..5f989de176 --- /dev/null +++ b/proxygen/lib/http/codec/SPDYVersion.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +enum class SPDYVersion { SPDY2 = 0, SPDY3, SPDY3_1, SPDY3_1_HPACK }; + +} diff --git a/proxygen/lib/http/codec/SPDYVersionSettings.h b/proxygen/lib/http/codec/SPDYVersionSettings.h new file mode 100644 index 0000000000..13ae3653b5 --- /dev/null +++ b/proxygen/lib/http/codec/SPDYVersionSettings.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/SPDYVersion.h" + +#include + +namespace folly { namespace io { +class Cursor; +}} + +namespace proxygen { + +/** + * Helper struct to carry common settings for a spdy version + */ +struct SPDYVersionSettings { + const std::string versionStr; + const std::string statusStr; + const std::string methodStr; + const std::string pathStr; + const std::string schemeStr; + const std::string hostStr; + const std::string protoName; + uint32_t (*parseSizeFun)(folly::io::Cursor*); + void (*appendSizeFun)(uint8_t*&, size_t); + const unsigned char* dict; + size_t dictSize; + uint16_t controlVersion; + uint16_t synReplySize; + uint16_t nameValueSize; + uint16_t goawaySize; + uint8_t priShift; + uint8_t majorVersion; + uint8_t minorVersion; + SPDYVersion version; +}; + +} diff --git a/proxygen/lib/http/codec/SettingsId.cpp b/proxygen/lib/http/codec/SettingsId.cpp new file mode 100644 index 0000000000..b4e3836582 --- /dev/null +++ b/proxygen/lib/http/codec/SettingsId.cpp @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/SettingsId.h" + +namespace proxygen { + +const uint8_t kMaxSettingIdentifier = 5; + +} diff --git a/proxygen/lib/http/codec/SettingsId.h b/proxygen/lib/http/codec/SettingsId.h new file mode 100644 index 0000000000..aab885fd8b --- /dev/null +++ b/proxygen/lib/http/codec/SettingsId.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +// Will never be valid HTTP/2 which only has 16 bits +#define SPDY_SETTINGS_MASK (1 << 16) + +enum class SettingsId: uint32_t { + // From HTTP/2 + HEADER_TABLE_SIZE = 1, + ENABLE_PUSH = 2, + MAX_CONCURRENT_STREAMS = 3, + INITIAL_WINDOW_SIZE = 4, + MAX_FRAME_SIZE = 5, + MAX_HEADER_LIST_SIZE = 6, + + // From SPDY, mostly unused + _SPDY_UPLOAD_BANDWIDTH = SPDY_SETTINGS_MASK | 1, + _SPDY_DOWNLOAD_BANDWIDTH = SPDY_SETTINGS_MASK | 2, + _SPDY_ROUND_TRIP_TIME = SPDY_SETTINGS_MASK | 3, + // MAX_CONCURRENT_STREAMS = 4, + _SPDY_CURRENT_CWND = SPDY_SETTINGS_MASK | 5, + _SPDY_DOWNLOAD_RETRANS_RATE = SPDY_SETTINGS_MASK | 6, + // INITIAL_WINDOW_SIZE = 7, + _SPDY_CLIENT_CERTIFICATE_VECTOR_SIZE = SPDY_SETTINGS_MASK | 8 +}; + +extern const uint8_t kMaxSettingIdentifier; + +typedef std::pair SettingPair; + +} diff --git a/proxygen/lib/http/codec/TransportDirection.cpp b/proxygen/lib/http/codec/TransportDirection.cpp new file mode 100644 index 0000000000..b8949f9cbf --- /dev/null +++ b/proxygen/lib/http/codec/TransportDirection.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/TransportDirection.h" + +namespace proxygen { + +const char* getTransportDirectionString(TransportDirection dir) { + switch (dir) { + case TransportDirection::UPSTREAM: return "upstream"; + case TransportDirection::DOWNSTREAM: return "downstream"; + } + // unreachable + return ""; +} + +TransportDirection operator!(TransportDirection dir) { + return dir == TransportDirection::DOWNSTREAM ? + TransportDirection::UPSTREAM : TransportDirection::DOWNSTREAM; +} + +std::ostream& operator<<(std::ostream& os, const TransportDirection dir) { + os << getTransportDirectionString(dir); + return os; +} + +} diff --git a/proxygen/lib/http/codec/TransportDirection.h b/proxygen/lib/http/codec/TransportDirection.h new file mode 100644 index 0000000000..1f669b0247 --- /dev/null +++ b/proxygen/lib/http/codec/TransportDirection.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +enum class TransportDirection : uint8_t { + DOWNSTREAM, // toward the client + UPSTREAM // toward the origin application or data +}; + +const char* getTransportDirectionString(TransportDirection dir); + +TransportDirection operator!(TransportDirection dir); + +std::ostream& operator<<(std::ostream& os, const TransportDirection dir); + +} // proxygen diff --git a/proxygen/lib/http/codec/compress/GzipHeaderCodec.cpp b/proxygen/lib/http/codec/compress/GzipHeaderCodec.cpp new file mode 100644 index 0000000000..a0d90782fa --- /dev/null +++ b/proxygen/lib/http/codec/compress/GzipHeaderCodec.cpp @@ -0,0 +1,400 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/GzipHeaderCodec.h" + +#include "proxygen/lib/http/codec/SPDYCodec.h" +#include "proxygen/lib/http/codec/SPDYConstants.h" + +#include +#include +#include +#include +#include + +using folly::IOBuf; +using folly::ThreadLocalPtr; +using folly::io::Cursor; +using proxygen::compress::Header; +using proxygen::compress::HeaderPiece; +using proxygen::compress::HeaderPieceList; +using proxygen::spdy::kMaxFrameLength; +using std::string; +using std::unique_ptr; +using std::vector; + +namespace { + +// Maximum size of header names+values after expanding multi-value headers +const size_t kMaxExpandedHeaderLineBytes = 80 * 1024; + +folly::IOBuf& getStaticHeaderBufSpace(size_t size) { + static ThreadLocalPtr buf; + if (!buf) { + buf.reset(new IOBuf(IOBuf::CREATE, size)); + } else { + if (size > buf->capacity()) { + buf.reset(new IOBuf(IOBuf::CREATE, size)); + } else { + buf->clear(); + } + } + DCHECK(!buf->isShared()); + return *buf; +} + +void appendString(uint8_t*& dst, const string& str) { + size_t len = str.length(); + memcpy(dst, str.data(), len); + dst += len; +} + +} // anonymous namespace + +namespace proxygen { + +GzipHeaderCodec::GzipHeaderCodec(int compressionLevel, + const SPDYVersionSettings& versionSettings) + : versionSettings_(versionSettings) { + // Create compression and decompression contexts by cloning thread-local + // copies of the initial SPDY compression state + auto context = getZlibContext(versionSettings, compressionLevel); + deflateCopy(&deflater_, const_cast(&(context->deflater))); + inflateCopy(&inflater_, const_cast(&(context->inflater))); +} + +GzipHeaderCodec::GzipHeaderCodec(int compressionLevel, + SPDYVersion version) + : GzipHeaderCodec( + compressionLevel, + SPDYCodec::getVersionSettings(version)) {} + +GzipHeaderCodec::~GzipHeaderCodec() { + deflateEnd(&deflater_); + inflateEnd(&inflater_); +} + +folly::IOBuf& GzipHeaderCodec::getHeaderBuf() { + return getStaticHeaderBufSpace(maxUncompressed_); +} + +const GzipHeaderCodec::ZlibContext* GzipHeaderCodec::getZlibContext( + SPDYVersionSettings versionSettings, int compressionLevel) { + static folly::ThreadLocal zlibContexts_; + ZlibConfig zlibConfig(versionSettings.version, compressionLevel); + auto match = zlibContexts_->find(zlibConfig); + if (match != zlibContexts_->end()) { + return match->second.get(); + } else { + // This is the first request for the specified SPDY version and compression + // level in this thread, so we need to construct the initial compressor and + // decompressor contexts. + auto newContext = folly::make_unique(); + newContext->deflater.zalloc = Z_NULL; + newContext->deflater.zfree = Z_NULL; + newContext->deflater.opaque = Z_NULL; + newContext->deflater.avail_in = 0; + newContext->deflater.next_in = Z_NULL; + int windowBits = (compressionLevel == Z_NO_COMPRESSION) ? 8 : 11; + int r = deflateInit2( + &(newContext->deflater), + compressionLevel, + Z_DEFLATED, // compression method + windowBits, // log2 of the compression window size, negative value + // means raw deflate output format w/o libz header + 1, // memory size for internal compression state, 1-9 + Z_DEFAULT_STRATEGY); + CHECK(r == Z_OK); + if (compressionLevel != Z_NO_COMPRESSION) { + r = deflateSetDictionary(&(newContext->deflater), versionSettings.dict, + versionSettings.dictSize); + CHECK(r == Z_OK); + } + + newContext->inflater.zalloc = Z_NULL; + newContext->inflater.zfree = Z_NULL; + newContext->inflater.opaque = Z_NULL; + newContext->inflater.avail_in = 0; + newContext->inflater.next_in = Z_NULL; + r = inflateInit(&(newContext->inflater)); + CHECK(r == Z_OK); + + auto result = newContext.get(); + zlibContexts_->emplace(zlibConfig, std::move(newContext)); + return result; + } +} + +unique_ptr GzipHeaderCodec::encode(vector
& headers) noexcept { + // Build a sequence of the header names and values, sorted by name. + // The purpose of the sort is to make it easier to combine the + // values of multiple headers with the same name. The SPDY spec + // prohibits any header name from appearing more than once in the + // Name/Value list, so we must combine values when serializing. + std::sort(headers.begin(), headers.end()); + + auto& uncompressed = getHeaderBuf(); + // Compute the amount of space needed to hold the uncompressed + // representation of the headers. This is an upper bound on the + // amount of space we'll actually need, because if we end up + // combining any headers with the same name, the combined + // representation will be smaller than the original. + size_t maxUncompressedSize = versionSettings_.nameValueSize; + for (const Header& header : headers) { + maxUncompressedSize += versionSettings_.nameValueSize; + maxUncompressedSize += header.name->length(); + maxUncompressedSize += versionSettings_.nameValueSize; + maxUncompressedSize += header.value->length(); + } + + // TODO: give on 'onError()' callback if the space in uncompressed buf + // cannot fit the headers and then skip the "reserve" code below. We + // have already reserved the maximum legal amount of space for + // uncompressed headers. + + VLOG(4) << "reserving " << maxUncompressedSize + << " bytes for uncompressed headers"; + uncompressed.reserve(0, maxUncompressedSize); + + // Serialize the uncompressed representation of the headers. + uint8_t* dst = uncompressed.writableData(); + dst += versionSettings_.nameValueSize; // Leave space for count of headers. + HTTPHeaderCode lastCode = HTTP_HEADER_OTHER; + const string* lastName = &empty_string; + uint8_t* lastValueLenPtr = nullptr; + size_t lastValueLen = 0; + unsigned numHeaders = 0; + for (const Header& header : headers) { + if ((header.code != lastCode) || (*header.name != *lastName)) { + // Simple case: this header name is different from the previous + // one, so we don't need to combine values. + numHeaders++; + versionSettings_.appendSizeFun(dst, header.name->length()); + + // lowercasing the header name inline + char* nameBegin = (char *)dst; + appendString(dst, *header.name); + folly::toLowerAscii((char *)nameBegin, header.name->size()); + + lastValueLenPtr = dst; + lastValueLen = header.value->length(); + versionSettings_.appendSizeFun(dst, header.value->length()); + appendString(dst, *header.value); + lastCode = header.code; + lastName = header.name; + } else { + // More complicated case: we do need to combine values. + *dst++ = 0; // SPDY uses a null byte as a separator + appendString(dst, *header.value); + // Go back and rewrite the length field in front of the value + lastValueLen += (1 + header.value->length()); + uint8_t* tmp = lastValueLenPtr; + versionSettings_.appendSizeFun(tmp, lastValueLen); + } + } + + // Compute the uncompressed length; if we combined any header values, + // we will have used less space than originally estimated. + size_t uncompressedLen = dst - uncompressed.writableData(); + + // Go back and write the count of unique header names at the start. + dst = uncompressed.writableData(); + versionSettings_.appendSizeFun(dst, numHeaders); + + // Allocate a contiguous space big enough to hold the compressed headers, + // plus any headroom requested by the caller. + size_t maxDeflatedSize = deflateBound(&deflater_, uncompressedLen); + unique_ptr out(IOBuf::create(maxDeflatedSize + encodeHeadroom_)); + out->advance(encodeHeadroom_); + + // Compress + deflater_.next_in = uncompressed.writableData(); + deflater_.avail_in = uncompressedLen; + deflater_.next_out = out->writableData(); + deflater_.avail_out = maxDeflatedSize; + int r = deflate(&deflater_, Z_SYNC_FLUSH); + CHECK(r == Z_OK); + CHECK(deflater_.avail_in == 0); + out->append(maxDeflatedSize - deflater_.avail_out); + + VLOG(4) << "header size orig=" << uncompressedLen + << ", max deflated=" << maxDeflatedSize + << ", actual deflated=" << out->length(); + + encodedSize_.compressed = out->length(); + encodedSize_.uncompressed = uncompressedLen; + if (stats_) { + stats_->recordEncode(Type::GZIP, encodedSize_); + } + + return std::move(out); +} + +Result +GzipHeaderCodec::decode(Cursor& cursor, uint32_t length) noexcept { + outHeaders_.clear(); + + // empty header block + if (length == 0) { + return HeaderDecodeResult{outHeaders_, 0}; + } + + // Get the thread local buffer space to use + auto& uncompressed = getHeaderBuf(); + uint32_t consumed = 0; + // Decompress the headers + while (length > 0) { + auto next = cursor.peek(); + uint32_t chunkLen = std::min((uint32_t)next.second, length); + inflater_.avail_in = chunkLen; + inflater_.next_in = (uint8_t *)next.first; + do { + if (uncompressed.tailroom() == 0) { + // This code should not execute, since we throw an error if the + // decompressed size of the headers is too large and we initialize + // the buffer to that size. + LOG(ERROR) << "Doubling capacity of SPDY headers buffer"; + uncompressed.reserve(0, uncompressed.capacity()); + } + + inflater_.next_out = uncompressed.writableTail(); + inflater_.avail_out = uncompressed.tailroom(); + int r = inflate(&inflater_, Z_NO_FLUSH); + if (r == Z_NEED_DICT) { + // we cannot initialize the inflater dictionary before calling inflate() + // as it checks the adler-32 checksum of the supplied dictionary + r = inflateSetDictionary(&inflater_, versionSettings_.dict, + versionSettings_.dictSize); + if (r != Z_OK) { + LOG(ERROR) << "inflate set dictionary failed with error=" << r; + return HeaderDecodeError::INFLATE_DICTIONARY; + } + inflater_.avail_out = 0; + continue; + } + if (r != 0) { + // probably bad encoding + LOG(ERROR) << "inflate failed with error=" << r; + return HeaderDecodeError::BAD_ENCODING; + } + uncompressed.append(uncompressed.tailroom() - inflater_.avail_out); + if (uncompressed.length() > maxUncompressed_) { + LOG(ERROR) << "Decompressed headers too large"; + return HeaderDecodeError::HEADERS_TOO_LARGE; + } + } while (inflater_.avail_in > 0 && inflater_.avail_out == 0); + length -= chunkLen; + consumed += chunkLen; + cursor.skip(chunkLen); + } + + decodedSize_.compressed = consumed; + decodedSize_.uncompressed = uncompressed.computeChainDataLength(); + if (stats_) { + stats_->recordDecode(Type::GZIP, decodedSize_); + } + + size_t expandedHeaderLineBytes = 0; + try { + auto result = parseNameValues(uncompressed); + if (result.isError()) { + return result.error(); + } + expandedHeaderLineBytes = result.ok(); + } catch (const std::out_of_range& ex) { + return HeaderDecodeError::BAD_ENCODING; + } + + if (UNLIKELY(expandedHeaderLineBytes > kMaxExpandedHeaderLineBytes)) { + LOG(ERROR) << "expanded headers too large"; + return HeaderDecodeError::HEADERS_TOO_LARGE; + } + + return HeaderDecodeResult{outHeaders_, consumed}; +} + +Result +GzipHeaderCodec::parseNameValues(const folly::IOBuf& uncompressed) { + + size_t expandedHeaderLineBytes = 0; + Cursor headerCursor(&uncompressed); + uint32_t numNV = versionSettings_.parseSizeFun(&headerCursor); + const HeaderPiece* headerName = nullptr; + + for (uint32_t i = 0; i < numNV * 2; i++) { + uint32_t len = versionSettings_.parseSizeFun(&headerCursor); + if (len == 0 && !headerName) { + LOG(ERROR) << "empty header name"; + return HeaderDecodeError::EMPTY_HEADER_NAME; + } + auto next = headerCursor.peek(); + if (next.second >= len) { + // string is contiguous, just put a pointer into the headers structure + outHeaders_.emplace_back((char *)next.first, len, false, false); + headerCursor.skip(len); + } else { + // string is not contiguous, allocate a buffer and pull into it + unique_ptr data (new char[len]); + headerCursor.pull(data.get(), len); + outHeaders_.emplace_back(data.release(), len, true, false); + } + if (i % 2 == 0) { + headerName = &outHeaders_.back(); + for (const char c: headerName->str) { + if (c < 0x20 || c > 0x7e || ('A' <= c && c <= 'Z')) { + LOG(ERROR) << "invalid header value"; + return HeaderDecodeError::INVALID_HEADER_VALUE; + } + } + } else { + HeaderPiece& headerValue = outHeaders_.back(); + bool first = true; + const char* valueStart = headerValue.str.data(); + const char* pos = valueStart; + const char* stop = valueStart + headerValue.str.size(); + while(pos < stop) { + if (*pos == '\0') { + if (pos - valueStart == 0) { + LOG(ERROR) << "empty header value"; + return HeaderDecodeError::EMPTY_HEADER_VALUE; + } + if (first) { + headerValue.str.reset(valueStart, pos - valueStart); + first = false; + } else { + outHeaders_.emplace_back(headerName->str.data(), + headerName->str.size(), + false, true); + outHeaders_.emplace_back(valueStart, pos - valueStart, false, true); + expandedHeaderLineBytes += ((pos - valueStart) + + headerName->str.size()); + } + valueStart = pos + 1; + } + pos++; + } + if (!first) { + // value contained at least one \0, add the last value + if (pos - valueStart == 0) { + LOG(ERROR) << "empty header value"; + return HeaderDecodeError::EMPTY_HEADER_VALUE; + } + outHeaders_.emplace_back(headerName->str.data(), + headerName->str.size(), + false, true); + outHeaders_.emplace_back(valueStart, pos - valueStart, false, true); + expandedHeaderLineBytes += (pos - valueStart) + headerName->str.size(); + } + headerName = nullptr; + } + } + return expandedHeaderLineBytes; +} + +} diff --git a/proxygen/lib/http/codec/compress/GzipHeaderCodec.h b/proxygen/lib/http/codec/compress/GzipHeaderCodec.h new file mode 100644 index 0000000000..2dba7e2c93 --- /dev/null +++ b/proxygen/lib/http/codec/compress/GzipHeaderCodec.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/SPDYVersionSettings.h" +#include "proxygen/lib/http/codec/compress/HeaderCodec.h" + +#include +#include +#include +#include +#include +#include + +namespace proxygen { + +class GzipHeaderCodec : public HeaderCodec { + + public: + GzipHeaderCodec(int compressionLevel, + const SPDYVersionSettings& versionSettings); + explicit GzipHeaderCodec(int compressionLevel, + SPDYVersion version = SPDYVersion::SPDY3_1); + virtual ~GzipHeaderCodec(); + + std::unique_ptr encode( + std::vector& headers) noexcept override; + + Result + decode(folly::io::Cursor& cursor, uint32_t length) noexcept override; + + private: + + folly::IOBuf& getHeaderBuf(); + + /** + * Parse the decompressed name/value header block. Note that this function can + * throw std::out_of_range + */ + Result + parseNameValues(const folly::IOBuf&); + + const SPDYVersionSettings& versionSettings_; + z_stream deflater_; + z_stream inflater_; + + // Pre-initialized compression contexts seeded with the + // starting dictionary for different SPDY versions - cloning + // one of these is faster than initializing and seeding a + // brand new deflate context. + struct ZlibConfig { + + ZlibConfig(SPDYVersion inVersion, int inCompressionLevel) + : version(inVersion), compressionLevel(inCompressionLevel) {} + + bool operator==(const ZlibConfig& lhs) const { + return (version == lhs.version) && + (compressionLevel == lhs.compressionLevel); + } + + bool operator<(const ZlibConfig& lhs) const { + return (version < lhs.version) || + ((version == lhs.version) && + (compressionLevel < lhs.compressionLevel)); + } + SPDYVersion version; + int compressionLevel; + }; + + struct ZlibContext { + ~ZlibContext() { + deflateEnd(&deflater); + inflateEnd(&inflater); + } + + z_stream deflater; + z_stream inflater; + }; + + /** + * get the thread local cached zlib context + */ + static const ZlibContext* getZlibContext(SPDYVersionSettings versionSettings, + int compressionLevel); + + typedef std::map> ZlibContextMap; +}; + +} diff --git a/proxygen/lib/http/codec/compress/HPACKCodec.cpp b/proxygen/lib/http/codec/compress/HPACKCodec.cpp new file mode 100644 index 0000000000..f55d4376da --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKCodec.cpp @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKCodec.h" + +#include "proxygen/lib/http/codec/compress/HPACKHeader.h" + +#include +#include +#include + +using folly::IOBuf; +using folly::io::Cursor; +using proxygen::compress::Header; +using proxygen::compress::HeaderPiece; +using proxygen::compress::HeaderPieceList; +using std::unique_ptr; +using std::vector; + +namespace proxygen { + +HPACKCodec::HPACKCodec(TransportDirection direction) { + HPACK::MessageType encoderType; + HPACK::MessageType decoderType; + if (direction == TransportDirection::DOWNSTREAM) { + decoderType = HPACK::MessageType::REQ; + encoderType = HPACK::MessageType::RESP; + } else { + // UPSTREAM + decoderType = HPACK::MessageType::RESP; + encoderType = HPACK::MessageType::REQ; + } + encoder_ = folly::make_unique(encoderType, true); + decoder_ = folly::make_unique(decoderType); +} + +unique_ptr HPACKCodec::encode(vector
& headers) noexcept { + vector converted; + // convert to HPACK API format + uint32_t uncompressed = 0; + for (const auto& h : headers) { + HPACKHeader header(*h.name, *h.value); + // This is ugly but since we're not changing the size + // of the string I'm assuming this is OK + char* mutableName = const_cast(header.name.data()); + folly::toLowerAscii(mutableName, header.name.size()); + converted.push_back(header); + uncompressed += header.name.size() + header.value.size() + 2; + } + auto buf = encoder_->encode(converted, encodeHeadroom_); + encodedSize_.compressed = 0; + if (buf) { + encodedSize_.compressed = buf->computeChainDataLength(); + } + encodedSize_.uncompressed = uncompressed; + if (stats_) { + stats_->recordEncode(Type::HPACK, encodedSize_); + } + return std::move(buf); +} + +Result +HPACKCodec::decode(Cursor& cursor, uint32_t length) noexcept { + outHeaders_.clear(); + decodedHeaders_.clear(); + auto consumed = decoder_->decode(cursor, length, decodedHeaders_); + if (decoder_->hasError()) { + if (stats_) { + stats_->recordDecodeError(Type::HPACK); + } + return HeaderDecodeError::BAD_ENCODING; + } + // convert to HeaderPieceList + uint32_t uncompressed = 0; + // sort the headers to detect duplicates/multi-valued headers + // * this is really unrelated to HPACK, just a need to mimic GzipHeaderCodec + sort(decodedHeaders_.begin(), decodedHeaders_.end()); + + for (uint32_t i = 0; i < decodedHeaders_.size(); i++) { + const HPACKHeader& h = decodedHeaders_[i]; + bool multiValued = + (i > 0 && decodedHeaders_[i - 1].name == h.name) || + (i < decodedHeaders_.size() - 1 && decodedHeaders_[i + 1].name == h.name); + // one entry for the name and one for the value + outHeaders_.push_back(HeaderPiece((char *)h.name.c_str(), h.name.size(), + false, multiValued)); + outHeaders_.push_back(HeaderPiece((char *)h.value.c_str(), h.value.size(), + false, multiValued)); + uncompressed += h.name.size() + h.value.size() + 2; + } + decodedSize_.compressed = consumed; + decodedSize_.uncompressed = uncompressed; + if (stats_) { + stats_->recordDecode(Type::HPACK, decodedSize_); + } + return HeaderDecodeResult{outHeaders_, consumed}; +} + +} diff --git a/proxygen/lib/http/codec/compress/HPACKCodec.h b/proxygen/lib/http/codec/compress/HPACKCodec.h new file mode 100644 index 0000000000..a5e95ca134 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKCodec.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/TransportDirection.h" +#include "proxygen/lib/http/codec/compress/HPACKDecoder.h" +#include "proxygen/lib/http/codec/compress/HPACKEncoder.h" +#include "proxygen/lib/http/codec/compress/HeaderCodec.h" + +#include +#include +#include + +namespace folly { namespace io { +class Cursor; +}} + +namespace proxygen { + +class HPACKHeader; + +/* + * Current version of the wire protocol. When we're making changes to the wire + * protocol we need to change this version so that old clients will not be + * able to negotiate it anymore. + */ +const uint8_t kHPACKMajorVersion = 0; // while is still in draft +const uint8_t kHPACKMinorVersion = 5; // based on HPACK draft 5 + +class HPACKCodec : public HeaderCodec { + public: + explicit HPACKCodec(TransportDirection direction); + virtual ~HPACKCodec() {} + + std::unique_ptr encode( + std::vector& headers) noexcept override; + + Result + decode(folly::io::Cursor& cursor, uint32_t length) noexcept override; + + protected: + std::unique_ptr encoder_; + std::unique_ptr decoder_; + + private: + std::vector decodedHeaders_; +}; + +} diff --git a/proxygen/lib/http/codec/compress/HPACKConstants.h b/proxygen/lib/http/codec/compress/HPACKConstants.h new file mode 100644 index 0000000000..c0d6fc3e02 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKConstants.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +namespace HPACK { + +const uint32_t kTableSize = 4096; + +const uint8_t NBIT_MASKS[9] = { + 0xFF, // 11111111, not used + 0xFE, // 11111110 + 0xFC, // 11111100 + 0xF8, // 11111000 + 0xF0, // 11110000 + 0xE0, // 11100000 + 0xC0, // 11000000 + 0x80, // 10000000 + 0x00 // 00000000 +}; + +enum HeaderEncoding : uint8_t { + LITERAL_NO_INDEXING = 0x40, // 2-bit + LITERAL_INCR_INDEXING = 0x00, // 2-bit + INDEXED = 0x80 // 1-bit +}; + +enum LiteralEncoding : uint8_t { + PLAIN = 0x00, + HUFFMAN = 0x80 +}; + +enum MessageType : uint8_t { + REQ = 0, + RESP = 1 +}; + +} + +} diff --git a/proxygen/lib/http/codec/compress/HPACKContext.cpp b/proxygen/lib/http/codec/compress/HPACKContext.cpp new file mode 100644 index 0000000000..b4a10d71ca --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKContext.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKContext.h" + +#include + +using std::string; + +namespace proxygen { + +HPACKContext::HPACKContext(HPACK::MessageType msgType, uint32_t tableSize) : + table_(tableSize), msgType_(msgType) { +} + +uint32_t HPACKContext::getIndex(const HPACKHeader& header) const { + uint32_t index = table_.getIndex(header); + if (index) { + return index; + } + index = StaticHeaderTable::get().getIndex(header); + if (index) { + return table_.size() + index; + } + return 0; +} + +uint32_t HPACKContext::nameIndex(const string& name) const { + uint32_t index = table_.nameIndex(name); + if (index) { + return index; + } + index = StaticHeaderTable::get().nameIndex(name); + if (index) { + return table_.size() + index; + } + return 0; +} + +bool HPACKContext::isStatic(uint32_t index) const { + return + index > table_.size() + && index <= table_.size() + StaticHeaderTable::get().size(); +} + +const HPACKHeader& HPACKContext::getHeader(uint32_t index) { + if (isStatic(index)) { + return StaticHeaderTable::get()[index - table_.size()]; + } + return table_[index]; +} + +} diff --git a/proxygen/lib/http/codec/compress/HPACKContext.h b/proxygen/lib/http/codec/compress/HPACKContext.h new file mode 100644 index 0000000000..c7102e1307 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKContext.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKConstants.h" +#include "proxygen/lib/http/codec/compress/HeaderTable.h" +#include "proxygen/lib/http/codec/compress/StaticHeaderTable.h" + +namespace proxygen { + +class HPACKContext { + public: + HPACKContext(HPACK::MessageType msgType, + uint32_t tableSize); + virtual ~HPACKContext() {} + + /** + * get the index of the given header by looking into both dynamic and static + * header table + * + * @return 0 if cannot be found + */ + uint32_t getIndex(const HPACKHeader& header) const; + + /** + * index of a header entry with the given name from dynamic or static table + * + * @return 0 if name not found + */ + uint32_t nameIndex(const std::string& name) const; + + /** + * @return true if the given index points to a static header entry + */ + bool isStatic(uint32_t index) const; + + /** + * @return header at the given index by composing dynamic and static tables + */ + const HPACKHeader& getHeader(uint32_t index); + + const HeaderTable& getTable() const { + return table_; + } + + protected: + HeaderTable table_; + HPACK::MessageType msgType_; +}; + +} diff --git a/proxygen/lib/http/codec/compress/HPACKDecodeBuffer.cpp b/proxygen/lib/http/codec/compress/HPACKDecodeBuffer.cpp new file mode 100644 index 0000000000..747b39928b --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKDecodeBuffer.cpp @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKDecodeBuffer.h" + +#include "proxygen/lib/http/codec/compress/Huffman.h" + +#include + +using folly::IOBuf; +using std::unique_ptr; + +namespace proxygen { + +bool HPACKDecodeBuffer::empty() { + return remainingBytes_ == 0; +} + +uint8_t HPACKDecodeBuffer::next() { + CHECK(remainingBytes_ > 0); + // in case we are the end of an IOBuf, peek() will move to the next one + uint8_t byte = peek(); + cursor_.skip(1); + remainingBytes_--; + + return byte; +} + +uint8_t HPACKDecodeBuffer::peek() { + CHECK(remainingBytes_ > 0); + if (cursor_.length() == 0) { + cursor_.peek(); + } + return *cursor_.data(); +} + +bool HPACKDecodeBuffer::decodeLiteral(std::string& literal) { + literal.clear(); + if (remainingBytes_ == 0) { + return false; + } + auto byte = peek(); + bool huffman = byte & HPACK::LiteralEncoding::HUFFMAN; + // extract the size + uint32_t size; + if (!decodeInteger(7, size)) { + return false; + } + if (size > remainingBytes_) { + return false; + } + const uint8_t* data; + unique_ptr tmpbuf; + // handle the case where the buffer spans multiple buffers + if (cursor_.length() >= size) { + data = cursor_.data(); + cursor_.skip(size); + } else { + // temporary buffer to pull the chunks together + tmpbuf = IOBuf::create(size); + // pull() will move the cursor + cursor_.pull(tmpbuf->writableData(), size); + data = tmpbuf->data(); + } + if (huffman) { + huffman::decode(msgType_, data, size, literal); + } else { + literal.append((const char *)data, size); + } + remainingBytes_ -= size; + return true; +} + +bool HPACKDecodeBuffer::decodeInteger(uint8_t nbit, uint32_t& integer) { + if (remainingBytes_ == 0) { + return false; + } + uint8_t byte = next(); + uint8_t mask = ~HPACK::NBIT_MASKS[nbit] & 0xFF; + // remove the first (8 - nbit) bits + byte = byte & mask; + integer = byte; + if (byte != mask) { + // the value fit in one byte + return true; + } + uint32_t f = 1; + do { + if (remainingBytes_ == 0) { + return false; + } + byte = next(); + integer += (byte & 127) * f; + f = f << 7; + } while (byte & 128); + return true; +} + +} diff --git a/proxygen/lib/http/codec/compress/HPACKDecodeBuffer.h b/proxygen/lib/http/codec/compress/HPACKDecodeBuffer.h new file mode 100644 index 0000000000..6d9c31a9b0 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKDecodeBuffer.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKConstants.h" + +#include +#include + +namespace proxygen { + +class HPACKDecodeBuffer { + public: + explicit HPACKDecodeBuffer(HPACK::MessageType msgType, + folly::io::Cursor& cursorVal, + uint32_t totalBytes) + : msgType_(msgType), + cursor_(cursorVal), + totalBytes_(totalBytes), + remainingBytes_(totalBytes) {} + + ~HPACKDecodeBuffer() {} + + void reset(folly::io::Cursor& cursorVal) { + reset(cursorVal, cursorVal.totalLength()); + } + + void reset(folly::io::Cursor& cursorVal, + uint32_t totalBytes) { + cursor_ = cursorVal; + totalBytes_ = totalBytes; + remainingBytes_ = totalBytes; + } + + uint32_t consumedBytes() const { + return totalBytes_ - remainingBytes_; + } + + const folly::io::Cursor& cursor() const { + return cursor_; + } + + /** + * @returns true if there are no more bytes to decode. Calling this method + * might move the cursor from the current IOBuf to the next one + */ + bool empty(); + + /** + * extracts one byte from the buffer and advances the cursor + */ + uint8_t next(); + + /** + * just peeks at the next available byte without moving the cursor + */ + uint8_t peek(); + + /** + * decode an integer from the current position, given a nbit prefix + * that basically needs to be ignored + */ + bool decodeInteger(uint8_t nbit, uint32_t& integer); + + /** + * decode a literal starting from the current position + */ + bool decodeLiteral(std::string& literal); + +private: + HPACK::MessageType msgType_; + folly::io::Cursor& cursor_; + uint32_t totalBytes_; + uint32_t remainingBytes_; +}; + +} diff --git a/proxygen/lib/http/codec/compress/HPACKDecoder.cpp b/proxygen/lib/http/codec/compress/HPACKDecoder.cpp new file mode 100644 index 0000000000..c76c20b815 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKDecoder.cpp @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKDecoder.h" + +#include +#include + +using folly::IOBuf; +using folly::io::Cursor; +using std::list; +using std::string; +using std::unique_ptr; +using std::vector; + +namespace proxygen { + +unique_ptr HPACKDecoder::decode(const IOBuf* buffer) { + auto headers = folly::make_unique(); + Cursor cursor(buffer); + uint32_t totalBytes = buffer ? cursor.totalLength() : 0; + decode(cursor, totalBytes, *headers); + // release ownership of the set of headers + return std::move(headers); +} + +uint32_t HPACKDecoder::decode(Cursor& cursor, + uint32_t totalBytes, + headers_t& headers) { + HPACKDecodeBuffer dbuf(msgType_, cursor, totalBytes); + while (!hasError() && !dbuf.empty()) { + decodeHeader(dbuf, headers); + } + emitRefset(headers); + return dbuf.consumedBytes(); +} + +void HPACKDecoder::emitRefset(headers_t& emitted) { + // emit the reference set + std::sort(emitted.begin(), emitted.end()); + list refset = table_.referenceSet(); + // remove the refset entries that have already been emitted + list::iterator refit = refset.begin(); + while (refit != refset.end()) { + const HPACKHeader& header = table_[*refit]; + if (std::binary_search(emitted.begin(), emitted.end(), header)) { + refit = refset.erase(refit); + } else { + refit++; + } + } + // try to avoid multiple resizing of the headers vector + emitted.reserve(emitted.size() + refset.size()); + for (const auto& index : refset) { + emit(table_[index], emitted); + } +} + +void HPACKDecoder::decodeLiteralHeader(HPACKDecodeBuffer& dbuf, + headers_t& emitted) { + uint8_t byte = dbuf.peek(); + bool indexing = !(byte & HPACK::HeaderEncoding::LITERAL_NO_INDEXING); + HPACKHeader header; + // check for indexed name + const uint8_t indexMask = 0x3F; // 0011 1111 + if (byte & indexMask) { + uint32_t index; + if (!dbuf.decodeInteger(6, index)) { + LOG(ERROR) << "buffer overflow decoding index"; + err_ = Error::BUFFER_OVERFLOW; + return; + } + // validate the index + if (!isValid(index)) { + LOG(ERROR) << "received invalid index: " << index; + err_ = Error::INVALID_INDEX; + return; + } + header.name = getHeader(index).name; + } else { + // skip current byte + dbuf.next(); + if (!dbuf.decodeLiteral(header.name)) { + LOG(ERROR) << "buffer overflow decoding header name"; + err_ = Error::BUFFER_OVERFLOW; + return; + } + } + // value + if (!dbuf.decodeLiteral(header.value)) { + LOG(ERROR) << "buffer overflow decoding header value"; + err_ = Error::BUFFER_OVERFLOW; + return; + } + + emit(header, emitted); + + if (indexing && table_.add(header)) { + // only add it to the refset if the header fit in the table + table_.addReference(1); + } +} + +void HPACKDecoder::decodeIndexedHeader(HPACKDecodeBuffer& dbuf, + headers_t& emitted) { + uint32_t index; + if (!dbuf.decodeInteger(7, index)) { + LOG(ERROR) << "buffer overflow decoding index"; + err_ = Error::BUFFER_OVERFLOW; + return; + } + if (index == 0) { + table_.clearReferenceSet(); + return; + } + // validate the index + if (!isValid(index)) { + LOG(ERROR) << "received invalid index: " << index; + err_ = Error::INVALID_INDEX; + return; + } + // a static index cannot be part of the reference set + if (isStatic(index)) { + auto& header = StaticHeaderTable::get()[index - table_.size()]; + emit(header, emitted); + if (table_.add(header)) { + table_.addReference(1); + } + } else if (table_.inReferenceSet(index)) { + // index remove operation + table_.removeReference(index); + } else { + auto& header = table_[index]; + emit(header, emitted); + table_.addReference(index); + } +} + +bool HPACKDecoder::isValid(uint32_t index) { + if (index <= table_.size()) { + return table_.isValid(index); + } + return StaticHeaderTable::get().isValid(index - table_.size()); +} + +void HPACKDecoder::decodeHeader(HPACKDecodeBuffer& dbuf, headers_t& emitted) { + uint8_t byte = dbuf.peek(); + if (byte & HPACK::HeaderEncoding::INDEXED) { + decodeIndexedHeader(dbuf, emitted); + } else { + // LITERAL_NO_INDEXING or LITERAL_INCR_INDEXING + decodeLiteralHeader(dbuf, emitted); + } +} + +void HPACKDecoder::emit(const HPACKHeader& header, + headers_t& emitted) { + emitted.push_back(header); +} + +} diff --git a/proxygen/lib/http/codec/compress/HPACKDecoder.h b/proxygen/lib/http/codec/compress/HPACKDecoder.h new file mode 100644 index 0000000000..2ffb7f0a9e --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKDecoder.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKContext.h" +#include "proxygen/lib/http/codec/compress/HPACKDecodeBuffer.h" +#include "proxygen/lib/http/codec/compress/HPACKHeader.h" + +#include +#include +#include +#include +#include + +namespace proxygen { + +class HPACKDecoder : public HPACKContext { + public: + enum Error : uint8_t { + NONE = 0, + INVALID_INDEX = 1, + INVALID_HUFFMAN_CODE = 2, + INVALID_ENCODING = 3, + BUFFER_OVERFLOW = 4, + }; + + explicit HPACKDecoder(HPACK::MessageType msgType, + uint32_t tableSize=HPACK::kTableSize) + : HPACKContext(msgType, tableSize) {} + + typedef std::vector headers_t; + + /** + * given a Cursor and a total amount of bytes we can consume from it, + * decode headers into the given vector. + */ + uint32_t decode(folly::io::Cursor& cursor, + uint32_t totalBytes, + headers_t& headers); + /** + * given a compressed header block as an IOBuf chain, decode all the + * headers and return them. This is just a convenience wrapper around + * the API above. + */ + std::unique_ptr decode(const folly::IOBuf* buffer); + + Error getError() const { + return err_; + } + + bool hasError() const { + return err_ != Error::NONE; + } + + private: + bool isValid(uint32_t index); + + void emitRefset(headers_t& emitted); + + void emit(const HPACKHeader& header, headers_t& emitted); + + void decodeIndexedHeader(HPACKDecodeBuffer& dbuf, headers_t& emitted); + + void decodeLiteralHeader(HPACKDecodeBuffer& dbuf, headers_t& emitted); + + void decodeHeader(HPACKDecodeBuffer& dbuf, headers_t& emitted); + + Error err_{Error::NONE}; +}; + +} diff --git a/proxygen/lib/http/codec/compress/HPACKEncodeBuffer.cpp b/proxygen/lib/http/codec/compress/HPACKEncodeBuffer.cpp new file mode 100644 index 0000000000..8a511e9864 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKEncodeBuffer.cpp @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKEncodeBuffer.h" + +#include "proxygen/lib/http/codec/compress/HPACKConstants.h" +#include "proxygen/lib/http/codec/compress/Logging.h" + +#include +#include + +using folly::IOBuf; +using std::string; +using std::unique_ptr; + +namespace proxygen { + +HPACKEncodeBuffer::HPACKEncodeBuffer( + uint32_t growthSize, + HPACK::MessageType msgType, + bool huffman) : + growthSize_(growthSize), + buf_(&bufQueue_, growthSize), + msgType_(msgType), + huffman_(huffman) { +} + +void HPACKEncodeBuffer::addHeadroom(uint32_t headroom) { + // we expect that this function is called before any encoding happens + CHECK(bufQueue_.front() == nullptr); + // create a custom IOBuf and add it to the queue + unique_ptr buf = IOBuf::create(std::max(headroom, growthSize_)); + buf->advance(headroom); + bufQueue_.append(std::move(buf)); +} + +void HPACKEncodeBuffer::append(uint8_t byte) { + buf_.ensure(1); + uint8_t* data = buf_.writableData(); + *data = byte; + buf_.append(1); +} + +uint32_t HPACKEncodeBuffer::encodeInteger(uint32_t value, uint8_t prefix, + uint8_t nbit) { + CHECK(nbit > 0 && nbit <= 8); + uint32_t count = 0; + uint8_t prefix_mask = HPACK::NBIT_MASKS[nbit]; + uint8_t mask = ~prefix_mask & 0xFF; + + // write the first byte + uint8_t byte = prefix & prefix_mask; + if (value < mask) { + // fits in the first byte + byte = byte | (mask & value); + append(byte); + return 1; + } + + byte |= mask; + value -= mask; + ++count; + append(byte); + // variable length encoding + while (value >= 128) { + byte = 128 | (127 & value); + append(byte); + value = value >> 7; + ++count; + } + // last byte, which should always fit on 1 byte + append(value); + ++count; + return count; +} + +uint32_t HPACKEncodeBuffer::encodeHuffman(const std::string& literal) { + uint32_t size = huffman::getSize(literal, msgType_); + // add the length + uint32_t count = encodeInteger(size, HPACK::LiteralEncoding::HUFFMAN, 7); + // ensure we have enough bytes before performing the encoding + buf_.ensure(size); + count += huffman::encode(literal, msgType_, buf_); + return count; +} + +uint32_t HPACKEncodeBuffer::encodeLiteral(const std::string& literal) { + if (huffman_) { + return encodeHuffman(literal); + } + // otherwise use simple layout + uint32_t count = + encodeInteger(literal.size(), HPACK::LiteralEncoding::PLAIN, 7); + // copy the entire string + buf_.ensure(literal.size()); + memcpy(buf_.writableData(), literal.c_str(), literal.size()); + buf_.append(literal.size()); + count += literal.size(); + return count; +} + +string HPACKEncodeBuffer::toBin(uint8_t bytesPerLine) { + return dumpBin(bufQueue_.front(), bytesPerLine); +} + +} diff --git a/proxygen/lib/http/codec/compress/HPACKEncodeBuffer.h b/proxygen/lib/http/codec/compress/HPACKEncodeBuffer.h new file mode 100644 index 0000000000..3a95ce840b --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKEncodeBuffer.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKConstants.h" +#include "proxygen/lib/http/codec/compress/Huffman.h" + +#include +#include + +namespace proxygen { + +class HPACKEncodeBuffer { + + public: + explicit HPACKEncodeBuffer( + uint32_t growthSize, + HPACK::MessageType msgType = HPACK::MessageType::REQ, + bool huffman = false); + ~HPACKEncodeBuffer() {} + + /** + * transfer ownership of the underlying IOBuf's + */ + std::unique_ptr release() { + return bufQueue_.move(); + } + + void clear() { + bufQueue_.clear(); + } + + /** + * Add headroom at the beginning of the IOBufQueue + * Meant to be called before encoding anything. + */ + void addHeadroom(uint32_t bytes); + + /** + * Encode the integer value using variable-length layout and the given prefix + * that spans nbit bits. + * The prefix is given as 1-byte value (not need for shifting) used only for + * the first byte. It starts from MSB. + * + * For example for integer=3, prefix=0x80, nbit=6: + * + * MSB LSB + * X X 0 0 0 0 1 1 (value) + * 1 0 X X X X X X (prefix) + * 1 0 0 0 0 0 1 1 (encoded value) + * + * @return how many bytes were used to encode the value + */ + uint32_t encodeInteger(uint32_t value, uint8_t prefix, uint8_t nbit); + + /** + * encodes a string, either header name or header value + * + * @return bytes used for encoding + */ + uint32_t encodeLiteral(const std::string& literal); + + /** + * encodes a string using huffman encoding + */ + uint32_t encodeHuffman(const std::string& literal); + + /** + * prints the content of an IOBuf in binary format. Useful for debugging. + */ + std::string toBin(uint8_t bytesPerLine=8); + + private: + + /** + * append one byte at the end of buffer ensuring we always have enough space + */ + void append(uint8_t byte); + + uint32_t growthSize_; + folly::IOBufQueue bufQueue_; + folly::io::QueueAppender buf_; + HPACK::MessageType msgType_; + bool huffman_; +}; + +} diff --git a/proxygen/lib/http/codec/compress/HPACKEncoder.cpp b/proxygen/lib/http/codec/compress/HPACKEncoder.cpp new file mode 100644 index 0000000000..d5c24cbce6 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKEncoder.cpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKEncoder.h" + +#include +#include +#include + +using folly::IOBuf; +using std::list; +using std::unique_ptr; +using std::unordered_set; +using std::vector; + +namespace proxygen { + +unique_ptr HPACKEncoder::encode(const vector& headers, + uint32_t headroom) { + table_.clearSkippedReferences(); + if (headroom) { + buffer_.addHeadroom(headroom); + } + encodeDelta(headers); + for (const auto& header : headers) { + if (willBeAdded(header)) { + encodeEvictedReferences(header); + } + encodeHeader(header); + } + return buffer_.release(); +} + +bool HPACKEncoder::willBeAdded(const HPACKHeader& header) { + auto index = getIndex(header); + return isStatic(index) || (index == 0 && header.isIndexable()); +} + +void HPACKEncoder::encodeEvictedReferences(const HPACKHeader& header) { + uint32_t index = table_.size(); + uint32_t bytes = table_.bytes(); + // the header will be added to the header table + while (index > 0 && (bytes + header.bytes() > table_.capacity())) { + // double encode only if the element is in the reference set + if (table_.isSkippedReference(index)) { + // 1. this will remove the entry from the refset + encodeAsIndex(index); + // 2. this will add the same entry to the refset and emit it + encodeAsIndex(index); + } + bytes -= table_[index].bytes(); + index--; + } +} + +void HPACKEncoder::encodeDelta(const vector& headers) { + // compute the difference between what's in reference set and what's in the + // reference set + + list refset = table_.referenceSet(); + // what's in the headers list and in the reference set - O(N) + vector toEncode; + toEncode.reserve(headers.size()); + for (const auto& header : headers) { + uint32_t index = table_.getIndex(header); + if (index > 0 && table_.inReferenceSet(index)) { + toEncode.push_back(index); + } + } + // what's in the reference set and not in the headers list - O(NlogN) + std::sort(toEncode.begin(), toEncode.end()); + vector toRemove; + toRemove.reserve(refset.size()); + for (auto index : refset) { + if (!std::binary_search(toEncode.begin(), toEncode.end(), index)) { + toRemove.push_back(index); + } + } + + if (!toRemove.empty()) { + // if we need to remove more than we keep in the refset, it's better to + // empty the refset entirely + if (refset.size() - toRemove.size() < toRemove.size()) { + clearReferenceSet(); + } else { + for (auto index : toRemove) { + encodeAsIndex(index); + table_.removeReference(index); + } + } + } +} + +void HPACKEncoder::encodeAsLiteral(const HPACKHeader& header) { + bool indexing = header.isIndexable(); + uint8_t prefix = indexing ? + HPACK::HeaderEncoding::LITERAL_INCR_INDEXING : + HPACK::HeaderEncoding::LITERAL_NO_INDEXING; + // name + uint32_t index = nameIndex(header.name); + if (index) { + buffer_.encodeInteger(index, prefix, 6); + } else { + buffer_.encodeInteger(0, prefix, 6); + buffer_.encodeLiteral(header.name); + } + // value + buffer_.encodeLiteral(header.value); + // indexed ones need to get added to the header table + if (indexing) { + if (table_.add(header)) { + table_.addReference(1); + } + } +} + +void HPACKEncoder::encodeAsIndex(uint32_t index) { + buffer_.encodeInteger(index, HPACK::HeaderEncoding::INDEXED, 7); +} + +void HPACKEncoder::clearReferenceSet() { + encodeAsIndex(0); + table_.clearReferenceSet(); +} + +void HPACKEncoder::encodeHeader(const HPACKHeader& header) { + uint32_t index = getIndex(header); + if (index) { + // firstly check if it's part of the static table + if (isStatic(index)) { + encodeAsIndex(index); + // insert the static header in the dynamic header table + // to take advantage of the delta compression + if (table_.add(StaticHeaderTable::get()[index - table_.size()])) { + table_.addReference(1); + } + } else if (!table_.inReferenceSet(index)) { + table_.addReference(index); + encodeAsIndex(index); + } else { + // there's nothing to encode, but keep a record for it in case of eviction + table_.addSkippedReference(index); + } + } else { + encodeAsLiteral(header); + } +} + +} diff --git a/proxygen/lib/http/codec/compress/HPACKEncoder.h b/proxygen/lib/http/codec/compress/HPACKEncoder.h new file mode 100644 index 0000000000..69c255ee26 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKEncoder.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKConstants.h" +#include "proxygen/lib/http/codec/compress/HPACKContext.h" +#include "proxygen/lib/http/codec/compress/HPACKEncodeBuffer.h" +#include "proxygen/lib/http/codec/compress/HeaderTable.h" + +#include +#include +#include + +namespace proxygen { + +class HPACKEncoder : public HPACKContext { + + public: + HPACKEncoder(HPACK::MessageType msgType, + bool huffman, + uint32_t tableSize=HPACK::kTableSize) : + HPACKContext(msgType, tableSize), + huffman_(huffman), + buffer_(kBufferGrowth, msgType, huffman) { + } + + /** + * Size of a new IOBuf which is added to the chain + */ + static const uint32_t kBufferGrowth = 4096; + + /** + * Encode the given headers and return the buffer + */ + std::unique_ptr encode(const std::vector& headers, + uint32_t headroom = 0); + + private: + void encodeHeader(const HPACKHeader& header); + + void encodeAsLiteral(const HPACKHeader& header); + + void encodeAsIndex(uint32_t index); + + void addHeader(const HPACKHeader& header); + + void encodeDelta(const std::vector& headers); + + /** + * Figures out in advance if the header is going to evict headers + * that are part of the reference set and it will encode them using + * double indexing technique: first one will remove the index from the + * refset and the second one will add it again to the refset, emitting it + */ + void encodeEvictedReferences(const HPACKHeader& header); + + /** + * Returns true if the given header will be added to the header table + */ + bool willBeAdded(const HPACKHeader& header); + + void clearReferenceSet(); + + bool huffman_; + HPACKEncodeBuffer buffer_; +}; + +} diff --git a/proxygen/lib/http/codec/compress/HPACKHeader.cpp b/proxygen/lib/http/codec/compress/HPACKHeader.cpp new file mode 100644 index 0000000000..dbefbd086b --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKHeader.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKHeader.h" + +namespace proxygen { + +bool HPACKHeader::isIndexable() const { + if (name == ":path") { + // no URL params + if (value.find('=') != std::string::npos) { + return false; + } + if (value.find("jpg") != std::string::npos) { + return false; + } + } else if (name == "content-length" || + name == "if-modified-since" || + name == "last-modified") { + return false; + } + return true; +} + +std::ostream& operator<<(std::ostream& os, const HPACKHeader& h) { + os << h.name << ": " << h.value; + return os; +} + +} diff --git a/proxygen/lib/http/codec/compress/HPACKHeader.h b/proxygen/lib/http/codec/compress/HPACKHeader.h new file mode 100644 index 0000000000..c341b4c412 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HPACKHeader.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +class HPACKHeader { + public: + + HPACKHeader() {} + + HPACKHeader(const std::string& name_, + const std::string& value_): + name(name_), value(value_) {} + + ~HPACKHeader() {} + + /** + * size in bytes of the header entry, as defined in the HPACK spec + */ + uint32_t bytes() const { + return 32 + name.size() + value.size(); + } + + bool operator==(const HPACKHeader& other) const { + return name == other.name && value == other.value; + } + + bool operator<(const HPACKHeader& other) const { + bool eqname = (name == other.name); + if (!eqname) { + return name < other.name; + } + return value < other.value; + } + + bool operator>(const HPACKHeader& other) const { + bool eqname = (name == other.name); + if (!eqname) { + return name > other.name; + } + return value > other.value; + } + + /** + * Some header entries don't have a value, see StaticHeaderTable + */ + bool hasValue() const { + return !value.empty(); + } + + /** + * Decide if we will add the current header into the header table + * + * This is a way to blacklist some headers that have variable values and are + * not efficient to index, like a :path with URL params, content-length or + * headers that contain timestamps: if-modified-since, last-modified. + * This is not standard for HPACK and it's part of some heuristics used by + * the encoder to improve the use of the header table. + * + * @return true if we should index the header + */ + bool isIndexable() const; + + std::string name; + std::string value; +}; + +std::ostream& operator<<(std::ostream& os, const HPACKHeader& h); + +} diff --git a/proxygen/lib/http/codec/compress/Header.h b/proxygen/lib/http/codec/compress/Header.h new file mode 100644 index 0000000000..9fa19a835f --- /dev/null +++ b/proxygen/lib/http/codec/compress/Header.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPHeaders.h" + +#include + +namespace proxygen { namespace compress { + +/** + * Helper structure used when serializing the uncompressed + * representation of a header name/value list. + */ +struct Header { + HTTPHeaderCode code{HTTP_HEADER_OTHER}; + const std::string* name; + const std::string* value; + + Header(const std::string& n, + const std::string& v) + : name(&n), value(&v) {} + + Header(HTTPHeaderCode c, + const std::string& v) + : code(c), name(HTTPCommonHeaders::getPointerToHeaderName(c)), value(&v) {} + + Header(HTTPHeaderCode c, + const std::string& n, + const std::string& v) + : code(c), name(&n), value(&v) {} + + bool operator<(const Header& h) const { + return (code < h.code) || + ((code == h.code) && (*name < *h.name)); + } +}; + +}} diff --git a/proxygen/lib/http/codec/compress/HeaderCodec.h b/proxygen/lib/http/codec/compress/HeaderCodec.h new file mode 100644 index 0000000000..4a4f861a9e --- /dev/null +++ b/proxygen/lib/http/codec/compress/HeaderCodec.h @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/codec/compress/Header.h" +#include "proxygen/lib/http/codec/compress/HeaderPiece.h" +#include "proxygen/lib/utils/Result.h" + +#include +#include + +namespace folly { +class IOBuf; +} + +namespace folly { namespace io { +class Cursor; +}} + +namespace proxygen { + +enum class HeaderDecodeError : uint8_t { + NONE = 0, + BAD_ENCODING = 1, + HEADERS_TOO_LARGE = 2, + INFLATE_DICTIONARY = 3, + EMPTY_HEADER_NAME = 4, + EMPTY_HEADER_VALUE = 5, + INVALID_HEADER_VALUE = 6, +}; + +struct HeaderDecodeResult { + compress::HeaderPieceList& headers; + uint32_t bytesConsumed; +}; + +class HeaderCodec { + public: + const uint32_t kMaxUncompressed = 80 * 1024; + + enum class Type : uint8_t { + GZIP = 0, + HPACK = 1, + }; + + class Stats { + public: + Stats() {} + virtual ~Stats() {} + + virtual void recordEncode(Type type, HTTPHeaderSize& size) = 0; + virtual void recordDecode(Type type, HTTPHeaderSize& size) = 0; + virtual void recordDecodeError(Type type) = 0; + }; + + HeaderCodec() {} + virtual ~HeaderCodec() {} + + /** + * Encode the given headers and return an IOBuf chain. + * + * The list of headers might be mutated during the encode, like order + * of the elements might change. + */ + virtual std::unique_ptr encode( + std::vector& headers) noexcept = 0; + + /** + * Decode headers given a Cursor and an amount of bytes to consume. + * + * @return Either the error that occurred while parsing the headers or + * the decoded header list. A header decode error should be considered + * fatal and no more bytes may be parsed from the cursor. + */ + virtual Result + decode(folly::io::Cursor& cursor, uint32_t length) noexcept = 0; + + /** + * compressed and uncompressed size of the last encode + */ + const HTTPHeaderSize& getEncodedSize() { + return encodedSize_; + } + + /** + * same as above, but for decode + */ + const HTTPHeaderSize& getDecodedSize() { + return decodedSize_; + } + + /** + * amount of space to reserve as a headroom in the encode buffer + */ + void setEncodeHeadroom(uint32_t headroom) { + encodeHeadroom_ = headroom; + } + + void setMaxUncompressed(uint32_t maxUncompressed) { + maxUncompressed_ = maxUncompressed; + } + + /** + * set the stats object + */ + void setStats(Stats* stats) { + stats_ = stats; + } + + protected: + + compress::HeaderPieceList outHeaders_; + uint32_t encodeHeadroom_{0}; + HTTPHeaderSize encodedSize_; + HTTPHeaderSize decodedSize_; + uint32_t maxUncompressed_{kMaxUncompressed}; + Stats* stats_{nullptr}; +}; + +} diff --git a/proxygen/lib/http/codec/compress/HeaderPiece.h b/proxygen/lib/http/codec/compress/HeaderPiece.h new file mode 100644 index 0000000000..76336d2149 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HeaderPiece.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { namespace compress { + +/** + * in-place representation of a header name or value + */ +class HeaderPiece { + public: + /** + * Construct a view around the data + */ + HeaderPiece(const char* inData, + uint32_t inLen, + bool inOwner, + bool inMultiValued) + : str(inData, inLen), + owner(inOwner), + multiValued(inMultiValued) {} + + HeaderPiece(HeaderPiece&& goner) noexcept + : str(goner.str), + owner(goner.owner), + multiValued(goner.multiValued){ + goner.owner = false; + } + + ~HeaderPiece() { + if (owner) { + CHECK(str.data() != nullptr); + delete[] str.data(); + } + } + + bool isMultiValued() const { + return multiValued; + } + + // should be const, but for one use in GzipHeaderCodec + folly::StringPiece str; + + private: + bool owner; + bool multiValued; +}; + +typedef std::deque HeaderPieceList; + +}} diff --git a/proxygen/lib/http/codec/compress/HeaderTable.cpp b/proxygen/lib/http/codec/compress/HeaderTable.cpp new file mode 100644 index 0000000000..4fa24f4655 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HeaderTable.cpp @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HeaderTable.h" + +#include + +using std::list; +using std::pair; +using std::string; + +namespace proxygen { + +void HeaderTable::init(uint32_t capacityVal) { + bytes_ = 0; + size_ = 0; + head_ = 0; + capacity_ = capacityVal; + // at a minimum an entry will take 32 bytes + length_ = (capacityVal >> 5) + 1; + table_.assign(length_, HPACKHeader()); + names_.clear(); +} + +bool HeaderTable::add(const HPACKHeader& header) { + // handle size overflow + if (bytes_ + header.bytes() > capacity_) { + evict(header.bytes()); + } + // this means the header entry is larger than our table + if (bytes_ + header.bytes() > capacity_) { + return false; + } + if (size_ > 0) { + head_ = next(head_); + } + table_[head_] = header; + // index name + names_[header.name].push_back(head_); + bytes_ += header.bytes(); + ++size_; + return true; +} + +uint32_t HeaderTable::getIndex(const HPACKHeader& header) const { + auto it = names_.find(header.name); + if (it == names_.end()) { + return 0; + } + for (auto i : it->second) { + if (table_[i].value == header.value) { + return toExternal(i); + } + } + return 0; +} + +bool HeaderTable::hasName(const std::string& name) { + return names_.find(name) != names_.end(); +} + +uint32_t HeaderTable::nameIndex(const std::string& name) const { + auto it = names_.find(name); + if (it == names_.end()) { + return 0; + } + return toExternal(it->second.back()); +} + +const HPACKHeader& HeaderTable::operator[](uint32_t i) const { + CHECK(isValid(i)); + return table_[toInternal(i)]; +} + +bool HeaderTable::inReferenceSet(uint32_t index) const { + return refset_.find(toInternal(index)) != refset_.end(); +} + +bool HeaderTable::isSkippedReference(uint32_t index) const { + return skippedRefs_.find(toInternal(index)) != skippedRefs_.end(); +} + +void HeaderTable::clearSkippedReferences() { + skippedRefs_.clear(); +} + +void HeaderTable::addSkippedReference(uint32_t index) { + skippedRefs_.insert(toInternal(index)); +} + +void HeaderTable::addReference(uint32_t index) { + refset_.insert(toInternal(index)); +} + +void HeaderTable::removeReference(uint32_t index) { + refset_.erase(toInternal(index)); +} + +void HeaderTable::clearReferenceSet() { + refset_.clear(); +} + +list HeaderTable::referenceSet() const { + list external; + for (auto& i : refset_) { + external.push_back(toExternal(i)); + } + // seems like the compiler will avoid the copy here + return external; +} + +void HeaderTable::removeLast() { + auto t = tail(); + refset_.erase(t); + skippedRefs_.erase(t); + // remove the first element from the names index + auto names_it = names_.find(table_[t].name); + DCHECK(names_it != names_.end()); + list &ilist = names_it->second; + DCHECK(ilist.front() ==t); + ilist.pop_front(); + // remove the name if there are no indices associated with it + if (ilist.empty()) { + names_.erase(names_it); + } + bytes_ -= table_[t].bytes(); + --size_; +} + +uint32_t HeaderTable::evict(uint32_t needed) { + uint32_t evicted = 0; + while (size_ > 0 && (bytes_ + needed > capacity_)) { + removeLast(); + ++evicted; + } + return evicted; +} + +bool HeaderTable::isValid(uint32_t index) const { + return 0 < index && index <= size_; +} + +uint32_t HeaderTable::next(uint32_t i) const { + return (i + 1) % length_; +} + +uint32_t HeaderTable::tail() const { + return (head_ + length_ - size_ + 1) % length_; +} + +uint32_t HeaderTable::toExternal(uint32_t internalIndex) const { + return toExternal(head_, length_, internalIndex); +} + +uint32_t HeaderTable::toExternal(uint32_t head, uint32_t length, + uint32_t internalIndex) { + return ((head + length - internalIndex) % length) + 1; +} + +uint32_t HeaderTable::toInternal(uint32_t externalIndex) const { + return toInternal(head_, length_, externalIndex); +} + +uint32_t HeaderTable::toInternal(uint32_t head, uint32_t length, + uint32_t externalIndex) { + // remove the offset + --externalIndex; + return (head + length - externalIndex) % length; +} + +bool HeaderTable::operator==(const HeaderTable& other) const { + if (size() != other.size()) { + return false; + } + if (bytes() != other.bytes()) { + return false; + } + list refset = referenceSet(); + refset.sort(); + list otherRefset = other.referenceSet(); + otherRefset.sort(); + if (refset != otherRefset) { + return false; + } + return true; +} + +std::ostream& operator<<(std::ostream& os, const HeaderTable& table) { + os << std::endl; + for (size_t i = 1; i <= table.size(); i++) { + const HPACKHeader& h = table[i]; + os << '[' << i << "] (s=" << h.bytes() << ") " + << h.name << ": " << h.value << std::endl; + } + os << "reference set: ["; + for (const auto& index : table.referenceSet()) { + os << index << ", "; + } + os << "]" << std::endl; + os << "total size: " << table.bytes() << std::endl; + return os; +} + +} diff --git a/proxygen/lib/http/codec/compress/HeaderTable.h b/proxygen/lib/http/codec/compress/HeaderTable.h new file mode 100644 index 0000000000..4b68eba0f0 --- /dev/null +++ b/proxygen/lib/http/codec/compress/HeaderTable.h @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKHeader.h" + +#include +#include +#include +#include +#include + +namespace proxygen { + +/** + * Data structure for maintaining indexed headers, based on a fixed-length ring + * with FIFO semantics. Externally it acts as an array. + */ + +class HeaderTable { + public: + + typedef std::unordered_map> names_map; + + explicit HeaderTable(uint32_t capacityVal) { + init(capacityVal); + } + HeaderTable() {} + + ~HeaderTable() {} + + /** + * Initialize with a given capacity. + */ + void init(uint32_t capacityVal); + + /** + * Add the header entry at the beginning of the table (index=1) and add the + * index to the reference set. + * + * @return true if it was able to add the entry + */ + bool add(const HPACKHeader& header); + + /** + * Get the index of the given header, if found. + * + * @return 0 in case the header is not found + */ + uint32_t getIndex(const HPACKHeader& header) const; + + /** + * Get the table entry at the given index. + * + * @return the header entry + */ + const HPACKHeader& operator[](uint32_t index) const; + + /** + * Checks if an external index is valid + */ + bool isValid(uint32_t index) const; + + /** + * @return true if there is at least one header with the given name + */ + bool hasName(const std::string& name); + + /** + * @return the map holding the indexed names + * + * Note: this contains references to internal indices, so it's useful only + * for testing or instrumentation. + */ + const names_map& names() const { + return names_; + } + + /** + * Get any index of a header that has the given name. From all the + * headers with the given name we pick the last one added to the header + * table, but the way we pick the header can be arbitrary. + */ + uint32_t nameIndex(const std::string& name) const; + + /** + * Clear new references set + */ + void clearSkippedReferences(); + + /** + * Tests whether the given index is a new reference. + */ + bool isSkippedReference(uint32_t index) const; + + /** + * Keep record of the given entry as a skipped reference. + */ + void addSkippedReference(uint32_t index); + + /** + * Check if a given index is part of the reference set. + */ + bool inReferenceSet(uint32_t index) const; + + /** + * Add index to the reference set. + */ + void addReference(uint32_t index); + + /** + * Remove index from the reference set. + */ + void removeReference(uint32_t index); + + /** + * Create a list with all the indices that are in the reference set. The + * caller will have ownership on the returned list. + */ + std::list referenceSet() const; + + /** + * Remove all indices from the reference set. + */ + void clearReferenceSet(); + + /** + * Table capacity, or maximum number of bytes we can hold. + */ + uint32_t capacity() const { + return capacity_; + } + + /** + * @return number of entries + */ + uint32_t size() const { + return size_; + } + + /** + * @return size in bytes, the sum of the size of all entries + */ + uint32_t bytes() const { + return bytes_; + } + + /** + * @return how many entries we have in the table + */ + uint32_t length() const { + return length_; + } + + bool operator==(const HeaderTable& other) const; + + /** + * Static versions of the methods that translate indices. + */ + static uint32_t toExternal(uint32_t head, uint32_t length, + uint32_t internalIndex); + + static uint32_t toInternal(uint32_t head, uint32_t length, + uint32_t externalIndex); + + private: + HeaderTable& operator=(const HeaderTable&); // non-copyable + + /** + * Removes one header entry from the beginning of the header table. + */ + void removeLast(); + + /** + * Evict entries to make space for the needed amount of bytes. + */ + uint32_t evict(uint32_t needed); + + /** + * Move the index to the right. + */ + uint32_t next(uint32_t i) const; + + /** + * Get the index of the tail element of the table. + */ + uint32_t tail() const; + + /** + * Translate internal index to external one, including a static version. + */ + uint32_t toExternal(uint32_t internalIndex) const; + + /** + * Translate external index to internal one. + */ + uint32_t toInternal(uint32_t externalIndex) const; + + uint32_t capacity_{0}; + uint32_t bytes_{0}; // size in bytes of the current entries + std::vector table_; + + uint32_t size_{0}; // how many entries we have in the table + uint32_t length_{0}; // number of entries in table_ + uint32_t head_{0}; // points to the first element of the ring + + names_map names_; + std::unordered_set refset_; + std::unordered_set skippedRefs_; +}; + +std::ostream& operator<<(std::ostream& os, const HeaderTable& table); + +} diff --git a/proxygen/lib/http/codec/compress/Huffman.cpp b/proxygen/lib/http/codec/compress/Huffman.cpp new file mode 100644 index 0000000000..170f2ac4a2 --- /dev/null +++ b/proxygen/lib/http/codec/compress/Huffman.cpp @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/Huffman.h" + +#include + +using folly::IOBuf; +using std::string; + +namespace proxygen { namespace huffman { + +HuffTree::HuffTree(const uint32_t* codes, const uint8_t* bits) : + codes_(codes), bits_(bits) { + buildTable(); +} + +HuffTree::HuffTree(const HuffTree& tree) : + codes_(tree.codes_), bits_(tree.bits_) { + buildTable(); +} + +bool HuffTree::decode(const uint8_t* buf, uint32_t size, string& literal) + const { + const SuperHuffNode* snode = &table_[0]; + uint32_t w = 0; + uint32_t wbits = 0; + uint32_t i = 0; + while (i < size || wbits > 0) { + // decide if we need to load more bits using an 8-bit chunk + if (i < size && wbits < 8) { + w = (w << 8) | buf[i]; + wbits += 8; + i++; + } + // key is used for performing the indexed lookup + uint32_t key; + if (wbits >= 8) { + key = w >> (wbits - 8); + } else { + // this the case we're at the end of the buffer + uint8_t xbits = 8 - wbits; + w = (w << xbits) | ((1 << xbits) - 1); + key = w; + wbits = 8; + } + // perform the indexed lookup + const HuffNode& node = snode->index[key]; + if (node.isLeaf()) { + // final node, we can emit the character + literal.push_back(node.ch); + wbits -= node.bits; + snode = &table_[0]; + } else { + // this is a branch, so we just need to move one level + wbits -= 8; + snode = &table_[node.superNode]; + } + // remove what we've just used + w = w & ((1 << wbits) - 1); + } + return true; +} + +/** + * insert a new character into the tree, identified by an unique code, + * a number of bits to represent it. The code is aligned at LSB. + */ +void HuffTree::insert(uint32_t code, uint8_t bits, uint8_t ch) { + SuperHuffNode* snode = &table_[0]; + uint32_t mask = 0xFF << (bits - 8); + while (bits > 8) { + uint32_t x = (code & mask) >> (bits - 8); + // mark this node as branch + if (snode->index[x].superNode == 0) { + nodes_++; + HuffNode& node = snode->index[x]; + node.superNode = nodes_; + } + snode = &table_[snode->index[x].superNode]; + bits -= 8; + code = code & ~mask; + mask = mask >> 8; + } + // fill the node with all the suffixes + fillIndex(*snode, code, bits, ch, bits); +} + +const uint32_t* HuffTree::codesTable() const { + return codes_; +} + +const uint8_t* HuffTree::bitsTable() const { + return bits_; +} + +/** + * recursive function for generating subtrees + */ +void HuffTree::fillIndex(SuperHuffNode& snode, uint32_t code, uint8_t bits, + uint8_t ch, uint8_t level) { + if (level == 8) { + snode.index[code].ch = ch; + snode.index[code].bits = bits; + return; + } + // generate the bit at the current level + code = code << 1; + for (uint8_t bit = 0; bit <= 1; bit++) { + fillIndex(snode, code | bit, bits, ch, level + 1); + } +} + +/** + * initializes and builds the huffman tree + */ +void HuffTree::buildTable() { + // create the indexed table + for (uint32_t i = 0; i < 256; i++) { + insert(codes_[i], bits_[i], i); + } +} + +/** + * static tables for Huffman encoding specific to HTTP requests and responses: + *http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-05#appendix-C + */ + +// use static functions to instantiate the trees to problems related to +// initialization order +const HuffTree& reqHuffTree() { + static const uint32_t requestCodes[256] = { + 0x7ffffba, 0x7ffffbb, 0x7ffffbc, 0x7ffffbd, 0x7ffffbe, 0x7ffffbf, 0x7ffffc0, + 0x7ffffc1, 0x7ffffc2, 0x7ffffc3, 0x7ffffc4, 0x7ffffc5, 0x7ffffc6, 0x7ffffc7, + 0x7ffffc8, 0x7ffffc9, 0x7ffffca, 0x7ffffcb, 0x7ffffcc, 0x7ffffcd, 0x7ffffce, + 0x7ffffcf, 0x7ffffd0, 0x7ffffd1, 0x7ffffd2, 0x7ffffd3, 0x7ffffd4, 0x7ffffd5, + 0x7ffffd6, 0x7ffffd7, 0x7ffffd8, 0x7ffffd9, 0xe8, 0xffc, 0x3ffa, 0x7ffc, + 0x7ffd, 0x24, 0x6e, 0x7ffe, 0x7fa, 0x7fb, 0x3fa, 0x7fc, 0xe9, 0x25, 0x4, + 0x0, 0x5, 0x6, 0x7, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1ec, 0xea, + 0x3fffe, 0x2d, 0x1fffc, 0x1ed, 0x3ffb, 0x6f, 0xeb, 0xec, 0xed, 0xee, 0x70, + 0x1ee, 0x1ef, 0x1f0, 0x1f1, 0x3fb, 0x1f2, 0xef, 0x1f3, 0x1f4, 0x1f5, 0x1f6, + 0x1f7, 0xf0, 0xf1, 0x1f8, 0x1f9, 0x1fa, 0x1fb, 0x1fc, 0x3fc, 0x3ffc, + 0x7ffffda, 0x1ffc, 0x3ffd, 0x2e, 0x7fffe, 0x8, 0x2f, 0x9, 0x30, 0x1, 0x31, + 0x32, 0x33, 0xa, 0x71, 0x72, 0xb, 0x34, 0xc, 0xd, 0xe, 0xf2, 0xf, 0x10, + 0x11, 0x35, 0x73, 0x36, 0xf3, 0xf4, 0xf5, 0x1fffd, 0x7fd, 0x1fffe, 0xffd, + 0x7ffffdb, 0x7ffffdc, 0x7ffffdd, 0x7ffffde, 0x7ffffdf, 0x7ffffe0, 0x7ffffe1, + 0x7ffffe2, 0x7ffffe3, 0x7ffffe4, 0x7ffffe5, 0x7ffffe6, 0x7ffffe7, 0x7ffffe8, + 0x7ffffe9, 0x7ffffea, 0x7ffffeb, 0x7ffffec, 0x7ffffed, 0x7ffffee, 0x7ffffef, + 0x7fffff0, 0x7fffff1, 0x7fffff2, 0x7fffff3, 0x7fffff4, 0x7fffff5, 0x7fffff6, + 0x7fffff7, 0x7fffff8, 0x7fffff9, 0x7fffffa, 0x7fffffb, 0x7fffffc, 0x7fffffd, + 0x7fffffe, 0x7ffffff, 0x3ffff80, 0x3ffff81, 0x3ffff82, 0x3ffff83, 0x3ffff84, + 0x3ffff85, 0x3ffff86, 0x3ffff87, 0x3ffff88, 0x3ffff89, 0x3ffff8a, 0x3ffff8b, + 0x3ffff8c, 0x3ffff8d, 0x3ffff8e, 0x3ffff8f, 0x3ffff90, 0x3ffff91, 0x3ffff92, + 0x3ffff93, 0x3ffff94, 0x3ffff95, 0x3ffff96, 0x3ffff97, 0x3ffff98, 0x3ffff99, + 0x3ffff9a, 0x3ffff9b, 0x3ffff9c, 0x3ffff9d, 0x3ffff9e, 0x3ffff9f, 0x3ffffa0, + 0x3ffffa1, 0x3ffffa2, 0x3ffffa3, 0x3ffffa4, 0x3ffffa5, 0x3ffffa6, 0x3ffffa7, + 0x3ffffa8, 0x3ffffa9, 0x3ffffaa, 0x3ffffab, 0x3ffffac, 0x3ffffad, 0x3ffffae, + 0x3ffffaf, 0x3ffffb0, 0x3ffffb1, 0x3ffffb2, 0x3ffffb3, 0x3ffffb4, 0x3ffffb5, + 0x3ffffb6, 0x3ffffb7, 0x3ffffb8, 0x3ffffb9, 0x3ffffba, 0x3ffffbb, 0x3ffffbc, + 0x3ffffbd, 0x3ffffbe, 0x3ffffbf, 0x3ffffc0, 0x3ffffc1, 0x3ffffc2, 0x3ffffc3, + 0x3ffffc4, 0x3ffffc5, 0x3ffffc6, 0x3ffffc7, 0x3ffffc8, 0x3ffffc9, 0x3ffffca, + 0x3ffffcb, 0x3ffffcc, 0x3ffffcd, 0x3ffffce, 0x3ffffcf, 0x3ffffd0, 0x3ffffd1, + 0x3ffffd2, 0x3ffffd3, 0x3ffffd4, 0x3ffffd5, 0x3ffffd6, 0x3ffffd7, 0x3ffffd8, + 0x3ffffd9, 0x3ffffda, 0x3ffffdb + }; + + static const uint8_t requestBits[256] = { + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 12, 14, 15, 15, 6, + 7, 15, 11, 11, 10, 11, 8, 6, 5, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 9, 8, 18, + 6, 17, 9, 14, 7, 8, 8, 8, 8, 7, 9, 9, 9, 9, 10, 9, 8, 9, 9, 9, 9, 9, 8, 8, + 9, 9, 9, 9, 9, 10, 14, 27, 13, 14, 6, 19, 5, 6, 5, 6, 4, 6, 6, 6, 5, 7, 7, + 5, 6, 5, 5, 5, 8, 5, 5, 5, 6, 7, 6, 8, 8, 8, 17, 11, 17, 12, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26 + }; + + static const HuffTree huffTree(requestCodes, requestBits); + return huffTree; +} + +const HuffTree& respHuffTree() { + static const uint32_t responseCodes[256] = { + 0x1ffffbc, 0x1ffffbd, 0x1ffffbe, 0x1ffffbf, 0x1ffffc0, 0x1ffffc1, 0x1ffffc2, + 0x1ffffc3, 0x1ffffc4, 0x1ffffc5, 0x1ffffc6, 0x1ffffc7, 0x1ffffc8, 0x1ffffc9, + 0x1ffffca, 0x1ffffcb, 0x1ffffcc, 0x1ffffcd, 0x1ffffce, 0x1ffffcf, 0x1ffffd0, + 0x1ffffd1, 0x1ffffd2, 0x1ffffd3, 0x1ffffd4, 0x1ffffd5, 0x1ffffd6, 0x1ffffd7, + 0x1ffffd8, 0x1ffffd9, 0x1ffffda, 0x1ffffdb, 0x0, 0xffa, 0x6a, 0x1ffa, + 0x3ffc, 0x1ec, 0x3f8, 0x1ffb, 0x1ed, 0x1ee, 0xffb, 0x7fa, 0x22, 0x23, 0x24, + 0x6b, 0x1, 0x2, 0x3, 0x8, 0x9, 0xa, 0x25, 0x26, 0xb, 0xc, 0xd, 0x1ef, + 0xfffa, 0x6c, 0x1ffc, 0xffc, 0xfffb, 0x6d, 0xea, 0xeb, 0xec, 0xed, 0xee, + 0x27, 0x1f0, 0xef, 0xf0, 0x3f9, 0x1f1, 0x28, 0xf1, 0xf2, 0x1f2, 0x3fa, + 0x1f3, 0x29, 0xe, 0x1f4, 0x1f5, 0xf3, 0x3fb, 0x1f6, 0x3fc, 0x7fb, 0x1ffd, + 0x7fc, 0x7ffc, 0x1f7, 0x1fffe, 0xf, 0x6e, 0x2a, 0x2b, 0x10, 0x6f, 0x70, + 0x71, 0x2c, 0x1f8, 0x1f9, 0x72, 0x2d, 0x2e, 0x2f, 0x30, 0x1fa, 0x31, 0x32, + 0x33, 0x34, 0x73, 0xf4, 0x74, 0xf5, 0x1fb, 0xfffc, 0x3ffd, 0xfffd, 0xfffe, + 0x1ffffdc, 0x1ffffdd, 0x1ffffde, 0x1ffffdf, 0x1ffffe0, 0x1ffffe1, 0x1ffffe2, + 0x1ffffe3, 0x1ffffe4, 0x1ffffe5, 0x1ffffe6, 0x1ffffe7, 0x1ffffe8, 0x1ffffe9, + 0x1ffffea, 0x1ffffeb, 0x1ffffec, 0x1ffffed, 0x1ffffee, 0x1ffffef, 0x1fffff0, + 0x1fffff1, 0x1fffff2, 0x1fffff3, 0x1fffff4, 0x1fffff5, 0x1fffff6, 0x1fffff7, + 0x1fffff8, 0x1fffff9, 0x1fffffa, 0x1fffffb, 0x1fffffc, 0x1fffffd, 0x1fffffe, + 0x1ffffff, 0xffff80, 0xffff81, 0xffff82, 0xffff83, 0xffff84, 0xffff85, + 0xffff86, 0xffff87, 0xffff88, 0xffff89, 0xffff8a, 0xffff8b, 0xffff8c, + 0xffff8d, 0xffff8e, 0xffff8f, 0xffff90, 0xffff91, 0xffff92, 0xffff93, + 0xffff94, 0xffff95, 0xffff96, 0xffff97, 0xffff98, 0xffff99, 0xffff9a, + 0xffff9b, 0xffff9c, 0xffff9d, 0xffff9e, 0xffff9f, 0xffffa0, 0xffffa1, + 0xffffa2, 0xffffa3, 0xffffa4, 0xffffa5, 0xffffa6, 0xffffa7, 0xffffa8, + 0xffffa9, 0xffffaa, 0xffffab, 0xffffac, 0xffffad, 0xffffae, 0xffffaf, + 0xffffb0, 0xffffb1, 0xffffb2, 0xffffb3, 0xffffb4, 0xffffb5, 0xffffb6, + 0xffffb7, 0xffffb8, 0xffffb9, 0xffffba, 0xffffbb, 0xffffbc, 0xffffbd, + 0xffffbe, 0xffffbf, 0xffffc0, 0xffffc1, 0xffffc2, 0xffffc3, 0xffffc4, + 0xffffc5, 0xffffc6, 0xffffc7, 0xffffc8, 0xffffc9, 0xffffca, 0xffffcb, + 0xffffcc, 0xffffcd, 0xffffce, 0xffffcf, 0xffffd0, 0xffffd1, 0xffffd2, + 0xffffd3, 0xffffd4, 0xffffd5, 0xffffd6, 0xffffd7, 0xffffd8, 0xffffd9, + 0xffffda, 0xffffdb, 0xffffdc + }; + + static const uint8_t responseBits[256] = { + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 4, 12, 7, 13, 14, 9, 10, + 13, 9, 9, 12, 11, 6, 6, 6, 7, 4, 4, 4, 5, 5, 5, 6, 6, 5, 5, 5, 9, 16, 7, 13, + 12, 16, 7, 8, 8, 8, 8, 8, 6, 9, 8, 8, 10, 9, 6, 8, 8, 9, 10, 9, 6, 5, 9, 9, + 8, 10, 9, 10, 11, 13, 11, 15, 9, 17, 5, 7, 6, 6, 5, 7, 7, 7, 6, 9, 9, 7, 6, + 6, 6, 6, 9, 6, 6, 6, 6, 7, 8, 7, 8, 9, 16, 14, 16, 16, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 + }; + + static const HuffTree huffTree(responseCodes, responseBits); + return huffTree; +} + +void decode(HPACK::MessageType msgType, + const uint8_t* buf, + uint32_t size, + std::string& literal) { + if (msgType == HPACK::MessageType::REQ) { + reqHuffTree().decode(buf, size, literal); + } else { + respHuffTree().decode(buf, size, literal); + } +} + +uint32_t encode(const std::string& literal, + HPACK::MessageType msgType, + folly::io::QueueAppender& buf) { + uint32_t code; // the huffman code of a given character + uint8_t bits; // on how many bits code is represented + uint32_t w = 0; // 4-byte word used for packing bits and write it to memory + uint8_t wbits = 0; // how many bits we have in 'w' + uint32_t* wdata = (uint32_t*) buf.writableData(); + uint32_t totalBytes = 0; + for (size_t i = 0; i < literal.size(); i++) { + code = getCode(literal[i], msgType, &bits); + + if (wbits + bits < 32) { + w = (w << bits) | code; + wbits += bits; + } else { + uint8_t xbits = wbits + bits - 32; + w = (w << (bits - xbits)) | (code >> xbits); + // write the word into the buffer by converting to network order, which + // takes care of the endianness problems + *wdata = htonl(w); + ++wdata; + totalBytes += 4; + // carry for next batch + wbits = xbits; + w = code & ((1 << xbits) - 1); + } + } + // we might have some padding at the byte level + if (wbits & 0x7) { + // padding bits + uint8_t padbits = 8 - (wbits & 0x7); + w = (w << padbits) | ((1 << padbits) - 1); + + wbits += padbits; + } + // we need to write the leftover bytes, from 1 to 4 bytes + if (wbits > 0) { + uint8_t bytes = wbits >> 3; + // align the bits to the MSB + w = w << (32 - wbits); + // set the bytes in the network order and copy w[0], w[1]... + w = htonl(w); + // we need to use memcpy because we might write less than 4 bytes + memcpy(wdata, &w, bytes); + totalBytes += bytes; + } + // in this point we know if we corrupted memory or not + CHECK(totalBytes <= buf.length()); + if (totalBytes > 0) { + buf.append(totalBytes); + } + return totalBytes; +} + +// Note: maybe templatize the Huffman class based on message type + +uint32_t getCode(uint8_t ch, HPACK::MessageType msgType, uint8_t* bits) { + // Request + if (msgType == HPACK::MessageType::REQ) { + auto& huffTree = reqHuffTree(); + *bits = huffTree.bitsTable()[ch]; + return huffTree.codesTable()[ch]; + } + // Response + auto& huffTree = respHuffTree(); + *bits = huffTree.bitsTable()[ch]; + return huffTree.codesTable()[ch]; +} + +uint32_t getSize(const std::string& literal, HPACK::MessageType msgType) { + uint32_t totalBits = 0; + uint8_t bits; + for (size_t i = 0; i < literal.size(); i++) { + // we just need the number of bits + getCode(literal[i], msgType, &bits); + totalBits += bits; + } + uint32_t size = totalBits >> 3; + if (totalBits & 0x07) { + ++size; + } + return size; +} + +}} diff --git a/proxygen/lib/http/codec/compress/Huffman.h b/proxygen/lib/http/codec/compress/Huffman.h new file mode 100644 index 0000000000..2feb9deca0 --- /dev/null +++ b/proxygen/lib/http/codec/compress/Huffman.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKConstants.h" + +#include +#include +#include + +namespace proxygen { + +namespace huffman { + +/** + * node from the huffman tree + * + * A leaf has no index table, or index == nullptr + */ +struct HuffNode { + uint8_t ch{0}; // leafs hold characters + uint8_t bits{0}; // how many bits are used for representing ch + uint8_t superNode{0}; + + bool isLeaf() const { + return superNode == 0; + } +}; + +/** + * a super node from the condensed Huffman Tree representation with 8-bit level + */ +struct SuperHuffNode { + HuffNode index[256]; +}; + +/** + * Immutable Huffman tree used in the process of decoding. Traditionally the + * huffman tree is binary, but using that approach leads to major inefficiencies + * since it's using per-bit level processing and needs to perform several memory + * accesses and bit operations for every single bit. + * This implementation is using 8-bit level indexing and uses aggregated nodes + * that link up to 256 other nodes. The complexity for lookup is reduced from + * O(bits) to O(Bytes) which is 1 or 2 for most of the printable characters. + * The tradeoff of using this approach is using more memory and generating the + * tree is more laborious since we need to fill all the subtrees denoted by a + * character code, which is an unique prefix. + * + * Example + * + * bit stream: + * 00001111 1111010 + * 1. our lookup key is 00001111 which will point to character '/', since the + * entire subtree with prefix 0000 points to it. We know the subtree has just + * 4 bits, we remove just those from the current key. + * bit stream: + * 11111111 010 + * + * 2. key is 11111111 which points to a branch, so we go down one level + * bit stream: + * 010 + * + * 3. we don't have enough bits, so we use paddding and we get a key of + * 01011111, which points to '(' character, like any other node under the + * subtree '010'. + */ +class HuffTree { + public: + explicit HuffTree(const uint32_t* codes, const uint8_t* bits); + ~HuffTree() {} + bool decode(const uint8_t* buf, uint32_t size, std::string& literal) const; + const uint32_t* codesTable() const; + const uint8_t* bitsTable() const; + + private: + void fillIndex(SuperHuffNode& snode, uint32_t code, uint8_t bits, uint8_t ch, + uint8_t level); + void buildTable(); + void insert(uint32_t code, uint8_t bits, uint8_t ch); + + uint32_t nodes_{0}; + const uint32_t* codes_; + const uint8_t* bits_; + + protected: + explicit HuffTree(const HuffTree& tree); + SuperHuffNode table_[46]; +}; + +const HuffTree& reqHuffTree(); +const HuffTree& respHuffTree(); + +/** + * encode the given string using Huffman encoding + * + * @return number of bytes used for encoding + */ +uint32_t encode(const std::string& literal, + HPACK::MessageType msgType, + folly::io::QueueAppender& buf); + +/** + * decode a block of size bytes into the given literal + */ +void decode(HPACK::MessageType msgType, + const uint8_t* buf, + uint32_t size, + std::string& literal); + +/** + * Get the Huffman code for a given character and HTTP message type {req,resp} + * + * @return The value of the code represented as a 32-bit unsigned value and + * the number of bits needed to represent the value in a bit stream. + * The code is aligned to LSB. + * + * Example: + * 'e' will be encoded as 1 using 4 bits: 0001 + */ +uint32_t getCode(uint8_t ch, HPACK::MessageType msgType, + uint8_t* bits); + +/** + * Get the number of bytes we need to represent the given string using Huffman + * encoding + */ +uint32_t getSize(const std::string& literal, + HPACK::MessageType msgType); +} + +} diff --git a/proxygen/lib/http/codec/compress/Logging.cpp b/proxygen/lib/http/codec/compress/Logging.cpp new file mode 100644 index 0000000000..7308ff60b9 --- /dev/null +++ b/proxygen/lib/http/codec/compress/Logging.cpp @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/Logging.h" + +#include +#include +#include +#include +#include + +using std::ostream; +using std::string; +using std::stringstream; +using std::vector; + +namespace proxygen { + +ostream& operator<<(ostream& os, const folly::IOBuf* buf) { + const uint8_t* data = buf->data(); + char tmp[24]; + for (size_t i = 0; i < buf->length(); i++) { + sprintf(tmp, "%02x", data[i]); + os << tmp; + if ((i + 1) % 2 == 0) { + os << ' '; + } + if ((i + 1) % 16 == 0) { + os << std::endl; + } + } + return os; +} + +ostream& operator<<(ostream& os, const std::list* refset) { + os << std::endl << '['; + for (auto& ref : *refset) { + os << ref << ' '; + } + os << ']' << std::endl; + return os; +} + +std::ostream& operator<<(std::ostream& os, const std::vector& v) { + for (const auto &h : v) { + os << h.name << ": " << h.value << std::endl; + } + os << std::endl; + return os; +} + +void dumpBinToFile(const std::string& filename, const folly::IOBuf* buf) { + struct stat fstat; + bool exists = (stat(filename.c_str(), &fstat) == 0); + if (exists) { + // don't write anything if the file exists + return; + } + std::ofstream file(filename, std::ofstream::binary); + if (!file.is_open()) { + LOG(ERROR) << "cannot open file " << filename; + return; + } + if (!buf) { + file.close(); + return; + } + const folly::IOBuf* first = buf; + do { + file.write((const char *)buf->data(), buf->length()); + buf = buf->next(); + } while (buf != first); + file.close(); + LOG(INFO) << "wrote chain " << dumpChain(buf) << " to " << filename; +} + +string dumpChain(const folly::IOBuf* buf) { + stringstream out; + auto b = buf; + do { + out << "iobuf of size " << b->length() + << " tailroom " << b->tailroom(); + b = b->next(); + } while (b != buf); + return out.str(); +} + +string dumpBin(const folly::IOBuf* buf, uint8_t bytesPerLine) { + string out; + const folly::IOBuf* first = buf; + if (!buf) { + return out; + } + do { + const uint8_t* data = buf->data(); + for (size_t i = 0; i < buf->length(); i++) { + for (int b = 7; b >= 0; b--) { + out += data[i] & 1 << b ? '1' : '0'; + } + out += ' '; + out += isprint(data[i]) ? data[i] : ' '; + if ((i + 1) % bytesPerLine == 0) { + out += '\n'; + } else { + out += ' '; + } + } + out += '\n'; + buf = buf->next(); + } while (buf != first); + return out; +} + +string printDelta(const vector &v1, + const vector &v2) { + stringstream out; + // similar with merge operation + size_t i = 0; + size_t j = 0; + out << std::endl; + while (i < v1.size() && j < v2.size()) { + if (v1[i] < v2[j]) { + if (i > 0 && v1[i - 1] == v1[i]) { + out << " duplicate " << v1[i] << std::endl; + } else { + out << " + " << v1[i] << std::endl; + } + i++; + } else if (v1[i] > v2[j]) { + out << " - " << v2[j] << std::endl; + j++; + } else { + i++; + j++; + } + } + while (i < v1.size()) { + out << " + " << v1[i]; + if (i > 0 && v1[i - 1] == v1[i]) { + out << " (duplicate)"; + } + out << std::endl; + i++; + } + while (j < v2.size()) { + out << " - " << v2[j] << std::endl; + j++; + } + return out.str(); +} + +} diff --git a/proxygen/lib/http/codec/compress/Logging.h b/proxygen/lib/http/codec/compress/Logging.h new file mode 100644 index 0000000000..06601eb110 --- /dev/null +++ b/proxygen/lib/http/codec/compress/Logging.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HeaderTable.h" + +#include +#include +#include +#include + +namespace proxygen { + +std::ostream& operator<<(std::ostream& os, const folly::IOBuf* buf); + +std::ostream& operator<<(std::ostream& os, const std::list* refset); + +std::string dumpChain(const folly::IOBuf* buf); + +std::ostream& operator<<(std::ostream& os, const std::vector &v); + +std::string dumpBin(const folly::IOBuf* buf, uint8_t bytes_per_line=8); + +void dumpBinToFile(const std::string& filename, const folly::IOBuf* buf); + +/** + * print the difference between 2 sorted list of headers + */ +std::string printDelta(const std::vector &v1, + const std::vector &v2); + +} diff --git a/proxygen/lib/http/codec/compress/StaticHeaderTable.cpp b/proxygen/lib/http/codec/compress/StaticHeaderTable.cpp new file mode 100644 index 0000000000..fffcd87af4 --- /dev/null +++ b/proxygen/lib/http/codec/compress/StaticHeaderTable.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/StaticHeaderTable.h" + +#include +#include + +using std::list; + +namespace proxygen { + +namespace { + +// instantiate the static table +StaticHeaderTable table; + +} + +StaticHeaderTable::StaticHeaderTable() : HeaderTable() { + static const char* STATIC_TABLE[][2] { + {":authority", ""}, + {":method", "GET"}, + {":method", "POST"}, + {":path", "/"}, + {":path", "/index.html"}, + {":scheme", "http"}, + {":scheme", "https"}, + {":status", "200"}, + {":status", "500"}, + {":status", "404"}, + {":status", "403"}, + {":status", "400"}, + {":status", "401"}, + {"accept-charset", ""}, + {"accept-encoding", ""}, + {"accept-language", ""}, + {"accept-ranges", ""}, + {"accept", ""}, + {"access-control-allow-origin", ""}, + {"age", ""}, + {"allow", ""}, + {"authorization", ""}, + {"cache-control", ""}, + {"content-disposition", ""}, + {"content-encoding", ""}, + {"content-language", ""}, + {"content-length", ""}, + {"content-location", ""}, + {"content-range", ""}, + {"content-type", ""}, + {"cookie", ""}, + {"date", ""}, + {"etag", ""}, + {"expect", ""}, + {"expires", ""}, + {"from", ""}, + {"host", ""}, + {"if-match", ""}, + {"if-modified-since", ""}, + {"if-none-match", ""}, + {"if-range", ""}, + {"if-unmodified-since", ""}, + {"last-modified", ""}, + {"link", ""}, + {"location", ""}, + {"max-forwards", ""}, + {"proxy-authenticate", ""}, + {"proxy-authorization", ""}, + {"range", ""}, + {"referer", ""}, + {"refresh", ""}, + {"retry-after", ""}, + {"server", ""}, + {"set-cookie", ""}, + {"strict-transport-security", ""}, + {"transfer-encoding", ""}, + {"user-agent", ""}, + {"vary", ""}, + {"via", ""}, + {"www-authenticate", ""} + }; + + // calculate the size + list hlist; + uint32_t byteCount = 0; + for (size_t i = 0; i < sizeof(STATIC_TABLE) / sizeof(STATIC_TABLE[0]); i++) { + hlist.push_back(HPACKHeader(STATIC_TABLE[i][0], STATIC_TABLE[i][1])); + byteCount += hlist.back().bytes(); + } + // initialize with a capacity that will exactly fit the static headers + init(byteCount); + hlist.reverse(); + for (auto& header : hlist) { + add(header); + } + // the static table is not involved in the delta compression + clearReferenceSet(); +} + +const HeaderTable& StaticHeaderTable::get() { + return table; +} + +} diff --git a/proxygen/lib/http/codec/compress/StaticHeaderTable.h b/proxygen/lib/http/codec/compress/StaticHeaderTable.h new file mode 100644 index 0000000000..54a0b826f9 --- /dev/null +++ b/proxygen/lib/http/codec/compress/StaticHeaderTable.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HeaderTable.h" + +namespace proxygen { + +class StaticHeaderTable : public HeaderTable { + + public: + StaticHeaderTable(); + + static const HeaderTable& get(); +}; + +} diff --git a/proxygen/lib/http/codec/compress/test/HPACKBufferTests.cpp b/proxygen/lib/http/codec/compress/test/HPACKBufferTests.cpp new file mode 100644 index 0000000000..f3856ed704 --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HPACKBufferTests.cpp @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKDecodeBuffer.h" +#include "proxygen/lib/http/codec/compress/HPACKEncodeBuffer.h" + +#include +#include +#include +#include + +using namespace folly::io; +using namespace folly; +using namespace proxygen; +using namespace std; +using namespace testing; + +class HPACKBufferTests : public testing::Test { + public: + /* + * most of the tests need a tiny slice of data, so we're setting up + * a queue with one IOBuf of 512 bytes in it + */ + HPACKBufferTests() : encoder_(512), + decoder_(HPACK::MessageType::REQ, cursor_, 0) { + } + + protected: + // extracts the data from the encoder and setup pointers to it + void releaseData() { + releaseData(encoder_); + } + + void releaseData(HPACKEncodeBuffer& encoder) { + buf_ = encoder.release(); + if (buf_) { + data_ = buf_->data(); + } + } + + void resetDecoder() { + resetDecoder(buf_.get()); + } + + void resetDecoder(IOBuf* buf) { + cursor_.reset(buf); + decoder_.reset(cursor_); + } + + Cursor cursor_{nullptr}; + HPACKEncodeBuffer encoder_; + HPACKDecodeBuffer decoder_; + std::unique_ptr buf_{nullptr}; + const uint8_t* data_{nullptr}; +}; + +TEST_F(HPACKBufferTests, encode_integer) { + uint32_t size; + // all these fit in one byte + EXPECT_EQ(encoder_.encodeInteger(7, 192, 8), 1); + // this one fits perfectly, but needs an additional 0 byte + EXPECT_EQ(encoder_.encodeInteger(7, 192, 3), 2); + EXPECT_EQ(encoder_.encodeInteger(255, 0, 8), 2); + releaseData(); + EXPECT_EQ(buf_->length(), 5); + EXPECT_EQ(data_[0], 7); // 00000111 + EXPECT_EQ(data_[1], 199); // 11000111 + EXPECT_EQ(data_[2], 0); + EXPECT_EQ(data_[3], 255); // 11111111 + EXPECT_EQ(data_[4], 0); + + // multiple byte span + EXPECT_EQ(encoder_.encodeInteger(7, 192, 2), 2); + releaseData(); + EXPECT_EQ(buf_->length(), 2); + EXPECT_EQ(data_[0], 195); // 11000011 + EXPECT_EQ(data_[1], 4); // 00000100 + + // the one from the spec - 1337 + EXPECT_EQ(encoder_.encodeInteger(1337, 0, 5), 3); + releaseData(); + EXPECT_EQ(buf_->length(), 3); + EXPECT_EQ(data_[0], 31); // 00011111 + EXPECT_EQ(data_[1], 154); // 10011010 + EXPECT_EQ(data_[2], 10); // 00001010 +} + +TEST_F(HPACKBufferTests, encode_plain_literal) { + string literal("accept-encoding"); + EXPECT_EQ(encoder_.encodeLiteral(literal), 16); + releaseData(); + EXPECT_EQ(buf_->length(), 16); + EXPECT_EQ(data_[0], 15); + for (size_t i = 0; i < literal.size(); i++) { + EXPECT_EQ(data_[i + 1], literal[i]); + } +} + +TEST_F(HPACKBufferTests, encode_huffman_literal) { + string accept("accept-encoding"); + HPACKEncodeBuffer encoder(512, HPACK::MessageType::REQ, true); + uint32_t size = encoder.encodeLiteral(accept); + EXPECT_EQ(size, 11); + releaseData(encoder); + EXPECT_EQ(buf_->length(), 11); + EXPECT_EQ(data_[0], 138); // 128(huffman bit) + 10(length) + EXPECT_EQ(data_[10], 47); +} + +TEST_F(HPACKBufferTests, decode_single_byte) { + buf_ = IOBuf::create(512); + uint8_t* wdata = buf_->writableData(); + buf_->append(1); + // 6-bit prefix + *wdata = 67; + resetDecoder(); + uint32_t integer; + CHECK_EQ(decoder_.decodeInteger(7, integer), true); + CHECK_EQ(integer, 67); + + resetDecoder(); + + CHECK_EQ(decoder_.decodeInteger(6, integer), true); + CHECK_EQ(integer, 3); + + // set a bit in the prefix - it should not affect the decoded value + *wdata = 195; // 195 = 128 + 67 + resetDecoder(); + CHECK_EQ(decoder_.decodeInteger(7, integer), true); + CHECK_EQ(integer, 67); + + // 8-bit prefix - the entire byte + resetDecoder(); + CHECK_EQ(decoder_.decodeInteger(8, integer), true); + CHECK_EQ(integer, 195); +} + +TEST_F(HPACKBufferTests, decode_multi_byte) { + buf_ = IOBuf::create(512); + uint8_t* wdata = buf_->writableData(); + // edge case - max value in a 2-bit space + buf_->append(2); + wdata[0] = 67; + wdata[1] = 0; + resetDecoder(); + uint32_t integer; + CHECK_EQ(decoder_.decodeInteger(2, integer), true); + CHECK_EQ(integer, 3); + CHECK_EQ(decoder_.cursor().length(), 0); + // edge case - encode 130 = 127 + 3 on 2-bit prefix + wdata[0] = 3; + wdata[1] = 127; + resetDecoder(); + CHECK_EQ(decoder_.decodeInteger(2, integer), true); + CHECK_EQ(integer, 130); + CHECK_EQ(decoder_.cursor().length(), 0); + // edge case - encode 131 = 128 + 3 + buf_->append(1); + wdata[0] = 3; + wdata[1] = 128; + wdata[2] = 1; + resetDecoder(); + CHECK_EQ(decoder_.decodeInteger(2, integer), true); + CHECK_EQ(integer, 131); + CHECK_EQ(decoder_.cursor().length(), 0); + // encode the value from the RFC example - 1337 + wdata[0] = 31; + wdata[1] = 154; + wdata[2] = 10; + resetDecoder(); + CHECK_EQ(decoder_.decodeInteger(5, integer), true); + CHECK_EQ(integer, 1337); + CHECK_EQ(decoder_.cursor().length(), 0); +} + +TEST_F(HPACKBufferTests, decode_integer_error) { + buf_ = IOBuf::create(128); + resetDecoder(); + // empty buffer + uint32_t integer; + CHECK_EQ(decoder_.decodeInteger(5, integer), false); + + // incomplete buffer + buf_->append(2); + uint8_t* wdata = buf_->writableData(); + wdata[0] = 31; + wdata[1] = 154; + // wdata[2] = 10 missing + CHECK_EQ(decoder_.decodeInteger(5, integer), false); +} + +TEST_F(HPACKBufferTests, decode_literal_error) { + buf_ = IOBuf::create(128); + + uint8_t* wdata = buf_->writableData(); + buf_->append(3); + resetDecoder(); + wdata[0] = 255; // size + wdata[1] = 'a'; + wdata[2] = 'b'; + string literal; + CHECK_EQ(decoder_.decodeLiteral(literal), false); + + resetDecoder(); + // error decoding the size of the literal + wdata[0] = 0xFF; + wdata[1] = 0x80; + wdata[2] = 0x80; + EXPECT_EQ(decoder_.decodeLiteral(literal), false); +} + +TEST_F(HPACKBufferTests, decode_literal_multi_buffer) { + auto buf1 = IOBuf::create(128); + auto buf2 = IOBuf::create(128); + // encode the size + // buf2 will not be entirely filled, to keep space for encoding the size + // without overflowing + uint32_t size = buf1->capacity() + buf2->capacity() - 10; + releaseData(); + uint32_t sizeLen = encoder_.encodeInteger(size, 0, 7); + releaseData(); + // copy the encoding of the size at the beginning + memcpy(buf1->writableData(), buf_->data(), sizeLen); + for (size_t i = sizeLen; i < buf1->capacity(); i++) { + buf1->writableData()[i] = 'x'; + } + buf1->append(buf1->capacity()); + for (size_t i = 0; i < buf2->capacity() - 10 + sizeLen; i++) { + buf2->writableData()[i] = 'y'; + } + buf2->append(buf2->capacity() - 10 + sizeLen); + buf1->appendChain(std::move(buf2)); + // decode + resetDecoder(buf1.get()); + string literal; + EXPECT_EQ(decoder_.decodeLiteral(literal), true); + EXPECT_EQ(literal.size(), size); + EXPECT_EQ(literal[0], 'x'); + EXPECT_EQ(literal[literal.size() - 1], 'y'); +} + +TEST_F(HPACKBufferTests, decode_huffman_literal_multi_buffer) { + // "gzip" fits perfectly in a 3 bytes block + uint8_t gzip[3] = {203, 213, 78}; + auto buf1 = IOBuf::create(128); + auto buf2 = IOBuf::create(128); + // total size + uint32_t size = buf1->capacity() + buf2->capacity() - 10; + // it needs to fit a multiple of 3 blocks + size -= (size % 3); + // just in case we have some bytes left encoded + releaseData(); + uint32_t sizeLen = encoder_.encodeInteger(size, 128, 7); + // extract the encoded size + releaseData(); + memcpy(buf1->writableData(), buf_->data(), sizeLen); + // huffman index + uint32_t hi = 0; + for (size_t i = sizeLen; i < buf1->capacity(); i++) { + buf1->writableData()[i] = gzip[hi]; + hi = (hi + 1) % 3; + } + buf1->append(buf1->capacity()); + for (size_t i = 0; i < buf2->capacity() - 10 + sizeLen; i++) { + buf2->writableData()[i] = gzip[hi]; + hi = (hi + 1) % 3; + } + buf2->append(buf2->capacity() - 10 + sizeLen); + buf1->appendChain(std::move(buf2)); + // decode + resetDecoder(buf1.get()); + string literal; + EXPECT_EQ(decoder_.decodeLiteral(literal), true); + EXPECT_EQ(literal.size(), 4 * (size / 3)); + EXPECT_EQ(literal.find("gzip"), 0); + EXPECT_EQ(literal.rfind("gzip"), literal.size() - 4); +} + +TEST_F(HPACKBufferTests, decode_plain_literal) { + buf_ = IOBuf::create(512); + std::string gzip("gzip"); + std::string literal; + uint8_t* wdata = buf_->writableData(); + + buf_->append(1 + gzip.size()); + wdata[0] = gzip.size(); + memcpy(wdata + 1, gzip.c_str(), gzip.size()); + + resetDecoder(); + decoder_.decodeLiteral(literal); + CHECK_EQ(literal, gzip); +} + +TEST_F(HPACKBufferTests, integer_encode_decode) { + HPACKEncodeBuffer encoder(512); + // first encode + uint32_t value = 145333; + encoder.encodeInteger(value, 128, 5); + releaseData(encoder); + resetDecoder(); + EXPECT_EQ(decoder_.cursor().length(), 4); + // now decode + uint32_t integer; + EXPECT_EQ(decoder_.decodeInteger(5, integer), true); + EXPECT_EQ(integer, value); + EXPECT_EQ(decoder_.cursor().length(), 0); + + // corner case + value = 63; + encoder.encodeInteger(value, 64, 6); + releaseData(encoder); + resetDecoder(); + EXPECT_EQ(decoder_.decodeInteger(6, integer), true); + EXPECT_EQ(integer, value); +} + +/** + * making sure we're calling peek() before deferencing the first byte + * to figure out if it's a huffman encoding or not + */ +TEST_F(HPACKBufferTests, empty_iobuf_literal) { + // construct an IOBuf chain made of 1 empty chain and a literal + unique_ptr first = IOBuf::create(128); + // set a trap by setting first byte to 128, which signals Huffman encoding + first->writableData()[0] = 0x80; + + HPACKEncodeBuffer encoder(128); // no huffman + string literal("randomheadervalue"); + encoder.encodeLiteral(literal); + first->appendChain(encoder.release()); + + uint32_t size = first->next()->length(); + Cursor cursor(first.get()); + HPACKDecodeBuffer decoder(HPACK::MessageType::REQ, cursor, size); + string decoded; + decoder.decodeLiteral(decoded); + + EXPECT_EQ(literal, decoded); +} diff --git a/proxygen/lib/http/codec/compress/test/HPACKCodecTests.cpp b/proxygen/lib/http/codec/compress/test/HPACKCodecTests.cpp new file mode 100644 index 0000000000..7863c1e3d8 --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HPACKCodecTests.cpp @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKCodec.h" +#include "proxygen/lib/http/codec/compress/Header.h" +#include "proxygen/lib/http/codec/compress/HeaderCodec.h" + +#include +#include +#include +#include +#include +#include + +using namespace folly::io; +using namespace folly; +using namespace proxygen::compress; +using namespace proxygen; +using namespace std; +using namespace testing; + +bool isLowercase(StringPiece str) { + for (auto ch : str) { + if (isalpha(ch) && !islower(ch)) { + return false; + } + } + return true; +} + +class TestHeaderCodecStats : public HeaderCodec::Stats { + + public: + void recordEncode(HeaderCodec::Type type, HTTPHeaderSize& size) override { + EXPECT_EQ(type, HeaderCodec::Type::HPACK); + encodes++; + encodedBytesCompr += size.compressed; + encodedBytesUncompr += size.uncompressed; + } + + void recordDecode(HeaderCodec::Type type, HTTPHeaderSize& size) override { + EXPECT_EQ(type, HeaderCodec::Type::HPACK); + decodes++; + decodedBytesCompr += size.compressed; + decodedBytesUncompr += size.uncompressed; + } + + void recordDecodeError(HeaderCodec::Type type) override { + EXPECT_EQ(type, HeaderCodec::Type::HPACK); + errors++; + } + + void reset() { + encodes = 0; + decodes = 0; + encodedBytesCompr = 0; + encodedBytesUncompr = 0; + decodedBytesCompr = 0; + decodedBytesUncompr = 0; + errors = 0; + } + + uint32_t encodes{0}; + uint32_t encodedBytesCompr{0}; + uint32_t encodedBytesUncompr{0}; + uint32_t decodes{0}; + uint32_t decodedBytesCompr{0}; + uint32_t decodedBytesUncompr{0}; + uint32_t errors{0}; +}; + +class HPACKCodecTests : public testing::Test { + + protected: + vector
headersFromArray(vector>& a) { + vector
headers; + for (auto& ha : a) { + headers.push_back(Header(ha[0], ha[1])); + } + return std::move(headers); + } + + HPACKCodec client{TransportDirection::UPSTREAM}; + HPACKCodec server{TransportDirection::DOWNSTREAM}; +}; + +TEST_F(HPACKCodecTests, request) { + vector> headers = { + {":path", "/index.php"}, + {":host", "www.facebook.com"}, + {"accept-encoding", "gzip"} + }; + vector
req = headersFromArray(headers); + + for (int i = 0; i < 3; i++) { + unique_ptr encodedReq = client.encode(req); + Cursor cursor(encodedReq.get()); + uint32_t len = 0; + if (encodedReq) { + len = encodedReq->computeChainDataLength(); + } + auto result = server.decode(cursor, len); + EXPECT_TRUE(result.isOk()); + EXPECT_EQ(result.ok().headers.size(), 6); + } +} + +TEST_F(HPACKCodecTests, response) { + vector> headers = { + {"content-length", "80"}, + {"content-encoding", "gzip"}, + {"x-fb-debug", "sdfgrwer"} + }; + vector
req = headersFromArray(headers); + + for (int i = 0; i < 3; i++) { + unique_ptr encodedReq = server.encode(req); + Cursor cursor(encodedReq.get()); + uint32_t len = 0; + if (encodedReq) { + len = encodedReq->computeChainDataLength(); + } + auto result = client.decode(cursor, len); + EXPECT_TRUE(result.isOk()); + EXPECT_EQ(result.ok().headers.size(), 6); + } +} + +TEST_F(HPACKCodecTests, headroom) { + vector> headers = { + {":path", "/index.php"}, + {":host", "www.facebook.com"}, + {"accept-encoding", "gzip"} + }; + vector
req = headersFromArray(headers); + + uint32_t headroom = 20; + client.setEncodeHeadroom(headroom); + for (int i = 0; i < 3; i++) { + unique_ptr encodedReq = client.encode(req); + EXPECT_EQ(encodedReq->headroom(), headroom); + Cursor cursor(encodedReq.get()); + uint32_t len = 0; + if (encodedReq) { + len = encodedReq->computeChainDataLength(); + } + auto result = server.decode(cursor, len); + EXPECT_TRUE(result.isOk()); + EXPECT_EQ(result.ok().headers.size(), 6); + } +} + +/** + * makes sure that the encoder will lowercase the header names + */ +TEST_F(HPACKCodecTests, lowercasing_header_names) { + vector> headers = { + {"Content-Length", "80"}, + {"Content-Encoding", "gzip"}, + {"X-FB-Debug", "bleah"} + }; + vector
req = headersFromArray(headers); + + unique_ptr encodedReq = server.encode(req); + Cursor cursor(encodedReq.get()); + uint32_t len = 0; + if (encodedReq) { + len = encodedReq->computeChainDataLength(); + } + auto result = client.decode(cursor, len); + EXPECT_TRUE(result.isOk()); + auto& decoded = result.ok().headers; + CHECK_EQ(decoded.size(), 6); + for (int i = 0; i < 6; i+=2) { + EXPECT_TRUE(isLowercase(decoded[i].str)); + } +} + +/** + * make sure we mark multi-valued headers appropriately, + * as expected by the SPDY codec. + */ +TEST_F(HPACKCodecTests, multivalue_headers) { + vector> headers = { + {"Content-Length", "80"}, + {"Content-Encoding", "gzip"}, + {"X-FB-Dup", "bleah"}, + {"X-FB-Dup", "hahaha"} + }; + vector
req = headersFromArray(headers); + + unique_ptr encodedReq = server.encode(req); + Cursor cursor(encodedReq.get()); + uint32_t len = 0; + if (encodedReq) { + len = encodedReq->computeChainDataLength(); + } + auto result = client.decode(cursor, len); + EXPECT_TRUE(result.isOk()); + auto& decoded = result.ok().headers; + CHECK_EQ(decoded.size(), 8); + for (int i = 0; i < 6; i+=2) { + if (decoded[i].str == "x-fb-dup") { + EXPECT_TRUE(decoded[i].isMultiValued()); + } + } +} + +/** + * test that we're propagating the error correctly in the decoder + */ +TEST_F(HPACKCodecTests, decode_error) { + vector> headers = { + {"Content-Length", "80"} + }; + vector
req = headersFromArray(headers); + + unique_ptr encodedReq = server.encode(req); + // mangle the buffer to force an error + uint32_t len = encodedReq->computeChainDataLength(); + encodedReq->writableData()[0] = 0xFF; + Cursor cursor(encodedReq.get()); + + TestHeaderCodecStats stats; + client.setStats(&stats); + auto result = client.decode(cursor, len); + // this means there was an error + EXPECT_TRUE(result.isError()); + EXPECT_EQ(result.error(), HeaderDecodeError::BAD_ENCODING); + EXPECT_EQ(stats.errors, 1); + client.setStats(nullptr); +} + +/** + * testing that we're calling the stats callbacks appropriately + */ +TEST_F(HPACKCodecTests, header_codec_stats) { + vector> headers = { + {"Content-Length", "80"}, + {"Content-Encoding", "gzip"}, + {"X-FB-Debug", "eirtijvdgtccffkutnbttcgbfieghgev"} + }; + vector
resp = headersFromArray(headers); + + TestHeaderCodecStats stats; + // encode + server.setStats(&stats); + unique_ptr encodedResp = server.encode(resp); + EXPECT_EQ(stats.encodes, 1); + EXPECT_EQ(stats.decodes, 0); + EXPECT_EQ(stats.errors, 0); + EXPECT_TRUE(stats.encodedBytesCompr > 0); + EXPECT_TRUE(stats.encodedBytesUncompr > 0); + EXPECT_EQ(stats.decodedBytesCompr, 0); + EXPECT_EQ(stats.decodedBytesUncompr, 0); + server.setStats(nullptr); + + // decode + Cursor cursor(encodedResp.get()); + uint32_t len = 0; + if (encodedResp) { + len = encodedResp->computeChainDataLength(); + } + stats.reset(); + client.setStats(&stats); + auto result = client.decode(cursor, len); + EXPECT_TRUE(result.isOk()); + auto& decoded = result.ok().headers; + CHECK_EQ(decoded.size(), 3 * 2); + EXPECT_EQ(stats.decodes, 1); + EXPECT_EQ(stats.encodes, 0); + EXPECT_TRUE(stats.decodedBytesCompr > 0); + EXPECT_TRUE(stats.decodedBytesUncompr > 0); + EXPECT_EQ(stats.encodedBytesCompr, 0); + EXPECT_EQ(stats.encodedBytesUncompr, 0); + client.setStats(nullptr); +} diff --git a/proxygen/lib/http/codec/compress/test/HPACKContextTests.cpp b/proxygen/lib/http/codec/compress/test/HPACKContextTests.cpp new file mode 100644 index 0000000000..54321857c1 --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HPACKContextTests.cpp @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKContext.h" +#include "proxygen/lib/http/codec/compress/HPACKDecoder.h" +#include "proxygen/lib/http/codec/compress/HPACKEncoder.h" +#include "proxygen/lib/http/codec/compress/Logging.h" + +#include +#include +#include +#include + +using namespace folly; +using namespace proxygen; +using namespace std; +using namespace testing; + +class HPACKContextTests : public testing::Test { +}; + +class TestContext : public HPACKContext { + + public: + TestContext(HPACK::MessageType msgType, + uint32_t tableSize) : HPACKContext(msgType, tableSize) {} + + void add(const HPACKHeader& header) { + table_.add(header); + } + +}; + +TEST_F(HPACKContextTests, get_index) { + HPACKContext context(HPACK::MessageType::REQ, HPACK::kTableSize); + HPACKHeader method(":method", "POST"); + + // this will get it from the static table + CHECK_EQ(context.getIndex(method), 3); +} + +TEST_F(HPACKContextTests, is_static) { + TestContext context(HPACK::MessageType::REQ, HPACK::kTableSize); + // add 10 headers to the table + for (int i = 1; i <= 10; i++) { + HPACKHeader header("name" + folly::to(i), + "value" + folly::to(i)); + context.add(header); + } + EXPECT_EQ(context.getTable().size(), 10); + + EXPECT_EQ(context.isStatic(1), false); + EXPECT_EQ(context.isStatic(10), false); + EXPECT_EQ(context.isStatic(40), true); + EXPECT_EQ(context.isStatic(60), true); + EXPECT_EQ(context.isStatic(69), true); +} + +TEST_F(HPACKContextTests, static_table) { + auto& table = StaticHeaderTable::get(); + const HPACKHeader& first = table[1]; + const HPACKHeader& methodPost = table[3]; + const HPACKHeader& last = table[table.size()]; + // there are 60 entries in the spec + CHECK_EQ(table.size(), 60); + CHECK_EQ(table[3], HPACKHeader(":method", "POST")); + CHECK_EQ(table[1].name, ":authority"); + CHECK_EQ(table[table.size()].name, "www-authenticate"); +} + +TEST_F(HPACKContextTests, static_index) { + TestContext context(HPACK::MessageType::REQ, HPACK::kTableSize); + HPACKHeader authority(":authority", ""); + EXPECT_EQ(context.getHeader(1), authority); + + HPACKHeader post(":method", "POST"); + EXPECT_EQ(context.getHeader(3), post); + + HPACKHeader contentLength("content-length", ""); + EXPECT_EQ(context.getHeader(27), contentLength); +} + +TEST_F(HPACKContextTests, encoder_multiple_values) { + HPACKEncoder encoder(HPACK::MessageType::RESP, true); + vector req; + req.push_back(HPACKHeader("accept-encoding", "gzip")); + req.push_back(HPACKHeader("accept-encoding", "sdch,gzip")); + // with the first encode both headers should be in the reference set + unique_ptr encoded = encoder.encode(req); + EXPECT_TRUE(encoded->length() > 0); + EXPECT_EQ(encoder.getTable().size(), 2); + // sending the same request again should lead to an empty encode buffer + EXPECT_EQ(encoder.encode(req), nullptr); +} + +TEST_F(HPACKContextTests, decoder_large_header) { + // with this size basically the table will not be able to store any entry + uint32_t size = 32; + HPACKHeader header; + HPACKEncoder encoder(HPACK::MessageType::REQ, true, size); + HPACKDecoder decoder(HPACK::MessageType::REQ, size); + vector headers; + headers.push_back(HPACKHeader(":path", "verylargeheader")); + // add a static entry + headers.push_back(HPACKHeader(":method", "GET")); + auto buf = encoder.encode(headers); + auto decoded = decoder.decode(buf.get()); + EXPECT_EQ(encoder.getTable().size(), 0); + EXPECT_EQ(encoder.getTable().referenceSet().size(), 0); + EXPECT_EQ(decoder.getTable().size(), 0); + EXPECT_EQ(decoder.getTable().referenceSet().size(), 0); +} + +/** + * testing invalid memory access in the decoder; it has to always call peek() + */ +TEST_F(HPACKContextTests, decoder_invalid_peek) { + HPACKEncoder encoder(HPACK::MessageType::REQ, true); + HPACKDecoder decoder(HPACK::MessageType::REQ); + vector headers; + headers.push_back(HPACKHeader("x-fb-debug", "test")); + + unique_ptr encoded = encoder.encode(headers); + unique_ptr first = IOBuf::create(128); + // set a trap for indexed header and don't call append + first->writableData()[0] = HPACK::HeaderEncoding::INDEXED; + + first->appendChain(std::move(encoded)); + auto decoded = decoder.decode(first.get()); + + EXPECT_FALSE(decoder.hasError()); + EXPECT_EQ(*decoded, headers); +} + +/** + * similar with the one above, but slightly different code paths + */ +TEST_F(HPACKContextTests, decoder_invalid_literal_peek) { + HPACKEncoder encoder(HPACK::MessageType::REQ, true); + HPACKDecoder decoder(HPACK::MessageType::REQ); + vector headers; + headers.push_back(HPACKHeader("x-fb-random", "bla")); + unique_ptr encoded = encoder.encode(headers); + + unique_ptr first = IOBuf::create(128); + first->writableData()[0] = 0x3F; + + first->appendChain(std::move(encoded)); + auto decoded = decoder.decode(first.get()); + + EXPECT_FALSE(decoder.hasError()); + EXPECT_EQ(*decoded, headers); +} + +/** + * testing various error cases in HPACKDecoder::decodeLiterHeader() + */ +void checkError(const IOBuf* buf, const HPACKDecoder::Error err) { + HPACKDecoder decoder(HPACK::MessageType::REQ); + auto decoded = decoder.decode(buf); + EXPECT_TRUE(decoder.hasError()); + EXPECT_EQ(decoder.getError(), err); +} + +TEST_F(HPACKContextTests, decode_errors) { + unique_ptr buf = IOBuf::create(128); + + // 1. simulate an error decoding the index for an indexed header name + // we try to encode index 65 + buf->writableData()[0] = 0x3F; + buf->append(1); // intentionally omit the second byte + checkError(buf.get(), HPACKDecoder::Error::BUFFER_OVERFLOW); + + // 2. invalid index for indexed header name + buf->writableData()[1] = 0xFF; + buf->writableData()[2] = 0x7F; + buf->append(2); + checkError(buf.get(), HPACKDecoder::Error::INVALID_INDEX); + + // 3. buffer overflow when decoding literal header name + buf->writableData()[0] = 0x00; // this will activate the non-indexed branch + checkError(buf.get(), HPACKDecoder::Error::BUFFER_OVERFLOW); + + // 4. buffer overflow when decoding a header value + // size for header name size and the actual header name + buf->writableData()[1] = 0x01; + buf->writableData()[2] = 'h'; + checkError(buf.get(), HPACKDecoder::Error::BUFFER_OVERFLOW); + + // 5. buffer overflow decoding the index of an indexed header + buf->writableData()[0] = 0xFF; // first bit is 1 to mark indexed header + buf->writableData()[1] = 0x80; // first bit is 1 to continue the + // variable-length encoding + buf->writableData()[2] = 0x80; + checkError(buf.get(), HPACKDecoder::Error::BUFFER_OVERFLOW); +} diff --git a/proxygen/lib/http/codec/compress/test/HPACKHeaderTests.cpp b/proxygen/lib/http/codec/compress/test/HPACKHeaderTests.cpp new file mode 100644 index 0000000000..e84fc3aabf --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HPACKHeaderTests.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKHeader.h" + +#include +#include +#include +#include + +using namespace proxygen; +using namespace std; +using namespace testing; + +class HPACKHeaderTests : public testing::Test { +}; + +TEST_F(HPACKHeaderTests, size) { + HPACKHeader h(":path", "/"); + EXPECT_EQ(h.bytes(), 32 + 5 + 1); +} + +TEST_F(HPACKHeaderTests, operators) { + HPACKHeader h0(":path", "/"); + HPACKHeader h1(":path", "/"); + HPACKHeader h2(":path", "/index.php"); + HPACKHeader h3("x-fb-debug", "test"); + // == + EXPECT_TRUE(h0 == h1); + EXPECT_FALSE(h1 == h2); + // < + EXPECT_FALSE(h1 < h1); + EXPECT_TRUE(h1 < h2); + EXPECT_TRUE(h1 < h3); + // > + EXPECT_FALSE(h2 > h2); + EXPECT_TRUE(h3 > h2); + EXPECT_TRUE(h2 > h1); + + stringstream out; + out << h1; + EXPECT_EQ(out.str(), ":path: /"); +} + +TEST_F(HPACKHeaderTests, has_value) { + HPACKHeader h1(":path", ""); + HPACKHeader h2(":path", "/"); + EXPECT_FALSE(h1.hasValue()); + EXPECT_TRUE(h2.hasValue()); +} + +TEST_F(HPACKHeaderTests, is_indexable) { + HPACKHeader path(":path", "index.php?q=42"); + EXPECT_FALSE(path.isIndexable()); + HPACKHeader cdn(":path", "/hprofile-ak-prn1/49496_6024432_1026115112_n.jpg"); + EXPECT_FALSE(cdn.isIndexable()); + HPACKHeader clen("content-length", "512"); + EXPECT_FALSE(clen.isIndexable()); +} diff --git a/proxygen/lib/http/codec/compress/test/HTTPArchive.cpp b/proxygen/lib/http/codec/compress/test/HTTPArchive.cpp new file mode 100644 index 0000000000..d786101b17 --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HTTPArchive.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/test/HTTPArchive.h" + +#include +#include +#include +#include +#include +#include +#include + +using folly::IOBuf; +using std::ifstream; +using std::ios; +using std::string; +using std::unique_ptr; +using std::vector; + +namespace proxygen { + +unique_ptr HTTPArchive::fromFile(const string& filename) { + unique_ptr har = folly::make_unique(); + // read the contents of the file + ifstream file(filename); + if (!file.is_open()) { + LOG(ERROR) << "could not open file '" << filename << "'"; + return nullptr; + } + file.seekg(0, ios::end); + int64_t size = file.tellg(); + if (size < 0) { + LOG(ERROR) << "failed to fetch the position at the end of the file"; + return nullptr; + } + file.seekg(0, ios::beg); + unique_ptr buffer = IOBuf::create(size + 1); + file.read((char *)buffer->writableData(), size); + buffer->writableData()[size] = 0; + buffer->append(size + 1); + if (!file) { + LOG(ERROR) << "error occurred, was able to read only " + << file.gcount() << " bytes out of " << size; + return nullptr; + } + + folly::dynamic jsonObj = folly::parseJson((const char *)buffer->data()); + auto entries = jsonObj["log"]["entries"]; + vector msg; + // go over all the transactions + for (size_t i = 0; i < entries.size(); i++) { + extractHeaders(entries[i]["request"], msg); + if (!msg.empty()) { + har->requests.push_back(msg); + } + extractHeaders(entries[i]["response"], msg); + if (!msg.empty()) { + har->responses.push_back(msg); + } + } + + return std::move(har); +} + +void HTTPArchive::extractHeaders(folly::dynamic& obj, + vector &msg) { + msg.clear(); + auto& headersObj = obj["headers"]; + for (size_t i = 0; i < headersObj.size(); i++) { + string name = headersObj[i]["name"].asString().toStdString(); + std::transform(name.begin(), name.end(), name.begin(), ::tolower); + msg.push_back( + HPACKHeader( + name, + headersObj[i]["value"].asString().toStdString()) + ); + } +} + +uint32_t HTTPArchive::getSize(const vector &headers) { + uint32_t size = 0; + + for (const auto header : headers) { + size += header.name.size() + header.value.size() + 2; + } + return size; +} + +} diff --git a/proxygen/lib/http/codec/compress/test/HTTPArchive.h b/proxygen/lib/http/codec/compress/test/HTTPArchive.h new file mode 100644 index 0000000000..e9dd4d23eb --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HTTPArchive.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKHeader.h" + +#include +#include +#include +#include +#include + +namespace proxygen { + +class HTTPArchive { + public: + + std::vector> requests; + std::vector> responses; + + static std::unique_ptr fromFile(const std::string& filename); + + // helper function for extracting a list of headers from a json array + static void extractHeaders(folly::dynamic& obj, + std::vector& msg); + + static uint32_t getSize(const std::vector& headers); +}; + +} diff --git a/proxygen/lib/http/codec/compress/test/HeaderPieceTests.cpp b/proxygen/lib/http/codec/compress/test/HeaderPieceTests.cpp new file mode 100644 index 0000000000..8311fbfd7d --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HeaderPieceTests.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HeaderPiece.h" + +#include +#include + +using namespace proxygen::compress; +using namespace testing; + +class HeaderPieceTests : public testing::Test {}; + +TEST_F(HeaderPieceTests, basic) { + HeaderPiece *hp; + + // creating non-owner piece with null pointer + hp = new HeaderPiece(nullptr, 0, false, true); + EXPECT_TRUE(hp->isMultiValued()); + // destructing this should be fine, since will not try to release the memory + delete hp; + + char *buf = new char[16]; + hp = new HeaderPiece(buf, 16, true, true); + EXPECT_EQ(hp->str.data(), buf); + EXPECT_EQ(hp->str.size(), 16); + // this should release the mem + delete hp; +} diff --git a/proxygen/lib/http/codec/compress/test/HeaderTableTests.cpp b/proxygen/lib/http/codec/compress/test/HeaderTableTests.cpp new file mode 100644 index 0000000000..a999ff48ad --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HeaderTableTests.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HeaderTable.h" +#include "proxygen/lib/http/codec/compress/Logging.h" +#include "proxygen/lib/http/codec/compress/StaticHeaderTable.h" + +#include +#include +#include +#include + +using namespace std; +using namespace testing; + +namespace proxygen { + +class HeaderTableTests : public testing::Test { + protected: + void xcheck(uint32_t internal, uint32_t external) { + EXPECT_EQ(HeaderTable::toExternal(head_, length_, internal), external); + EXPECT_EQ(HeaderTable::toInternal(head_, length_, external), internal); + } + + uint32_t head_{0}; + uint32_t length_{0}; +}; + +TEST_F(HeaderTableTests, index_translation) { + // simple cases + length_ = 10; + head_ = 5; + xcheck(0, 6); + xcheck(3, 3); + xcheck(5, 1); + + // wrap + head_ = 1; + xcheck(0, 2); + xcheck(8, 4); + xcheck(5, 7); +} + +TEST_F(HeaderTableTests, add) { + HeaderTable table(4096); + table.add(HPACKHeader("accept-encoding", "gzip")); + table.add(HPACKHeader("accept-encoding", "gzip")); + table.add(HPACKHeader("accept-encoding", "gzip")); + EXPECT_EQ(table.names().size(), 1); + EXPECT_EQ(table.hasName("accept-encoding"), true); + auto it = table.names().find("accept-encoding"); + EXPECT_EQ(it->second.size(), 3); + EXPECT_EQ(table.nameIndex("accept-encoding"), 1); +} + +TEST_F(HeaderTableTests, evict) { + HPACKHeader accept("accept-encoding", "gzip"); + HPACKHeader accept2("accept-encoding", "----"); // same size, different header + HPACKHeader accept3("accept-encoding", "third"); // size is larger with 1 byte + uint32_t max = 10; + uint32_t capacity = accept.bytes() * max; + HeaderTable table(capacity); + // fill the table + for (size_t i = 0; i < max; i++) { + EXPECT_EQ(table.add(accept), true); + } + EXPECT_EQ(table.size(), max); + EXPECT_EQ(table.add(accept2), true); + // evict the first one + EXPECT_EQ(table[1], accept2); + auto ilist = table.names().find("accept-encoding")->second; + EXPECT_EQ(ilist.size(), max); + // evict all the 'accept' headers + for (size_t i = 0; i < max - 1; i++) { + EXPECT_EQ(table.add(accept2), true); + } + EXPECT_EQ(table.size(), max); + EXPECT_EQ(table[max], accept2); + EXPECT_EQ(table.names().size(), 1); + // add an entry that will cause 2 evictions + EXPECT_EQ(table.add(accept3), true); + EXPECT_EQ(table[1], accept3); + EXPECT_EQ(table.size(), max - 1); + + // add a super huge header + string bigvalue; + bigvalue.append(capacity, 'x'); + HPACKHeader bigheader("user-agent", bigvalue); + EXPECT_EQ(table.add(bigheader), false); + EXPECT_EQ(table.size(), 0); + EXPECT_EQ(table.names().size(), 0); +} + +TEST_F(HeaderTableTests, comparison) { + uint32_t capacity = 128; + HeaderTable t1(capacity); + HeaderTable t2(capacity); + + HPACKHeader h1("Content-Encoding", "gzip"); + HPACKHeader h2("Content-Encoding", "deflate"); + // different in number of elements + t1.add(h1); + EXPECT_FALSE(t1 == t2); + // different in size (bytes) + t2.add(h2); + EXPECT_FALSE(t1 == t2); + + // make them the same + t1.add(h2); + t2.add(h1); + EXPECT_TRUE(t1 == t2); + + // make them mismatch on refset + t1.addReference(1); + EXPECT_FALSE(t1 == t2); +} + +TEST_F(HeaderTableTests, print) { + stringstream out; + HeaderTable t(128); + t.add(HPACKHeader("Accept-Encoding", "gzip")); + t.addReference(1); + out << t; + EXPECT_EQ(out.str(), + "\n[1] (s=51) Accept-Encoding: gzip\nreference set: [1, ]\ntotal size: 51\n"); +} + +} diff --git a/proxygen/lib/http/codec/compress/test/HuffmanTests.cpp b/proxygen/lib/http/codec/compress/test/HuffmanTests.cpp new file mode 100644 index 0000000000..5727ba67da --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/HuffmanTests.cpp @@ -0,0 +1,312 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/Huffman.h" +#include "proxygen/lib/http/codec/compress/Logging.h" + +#include +#include +#include +#include + +using namespace folly::io; +using namespace folly; +using namespace proxygen::huffman; +using namespace proxygen; +using namespace std; +using namespace testing; + +class HuffmanTests : public testing::Test { +}; + +TEST_F(HuffmanTests, codes) { + uint32_t code; + uint8_t bits; + // check 'e' for both requests and responses + code = huffman::getCode('e', HPACK::MessageType::REQ, &bits); + EXPECT_EQ(code, 0x01); + EXPECT_EQ(bits, 4); + code = huffman::getCode('e', HPACK::MessageType::RESP, &bits); + EXPECT_EQ(code, 0x10); + EXPECT_EQ(bits, 5); + // some extreme cases + code = huffman::getCode(0, HPACK::MessageType::REQ, &bits); + EXPECT_EQ(code, 0x7ffffba); + EXPECT_EQ(bits, 27); + code = huffman::getCode(255, HPACK::MessageType::REQ, &bits); + EXPECT_EQ(code, 0x3ffffdb); + EXPECT_EQ(bits, 26); + + code = huffman::getCode(0, HPACK::MessageType::RESP, &bits); + EXPECT_EQ(code, 0x1ffffbc); + EXPECT_EQ(bits, 25); + code = huffman::getCode(255, HPACK::MessageType::RESP, &bits); + EXPECT_EQ(code, 0xffffdc); + EXPECT_EQ(bits, 24); +} + +TEST_F(HuffmanTests, size) { + uint32_t size; + string onebyte("/e"); + size = huffman::getSize(onebyte, HPACK::MessageType::REQ); + EXPECT_EQ(size, 1); + + string accept("accept-encoding"); + size = huffman::getSize(accept, HPACK::MessageType::REQ); + EXPECT_EQ(size, 10); + size = huffman::getSize(accept, HPACK::MessageType::RESP); + EXPECT_EQ(size, 11); +} + +TEST_F(HuffmanTests, encode) { + uint32_t size; + // this is going to fit perfectly into 3 bytes + string gzip("gzip"); + IOBufQueue bufQueue; + QueueAppender appender(&bufQueue, 512); + // force the allocation + appender.ensure(512); + + size = huffman::encode(gzip, HPACK::MessageType::REQ, appender); + EXPECT_EQ(size, 3); + const IOBuf* buf = bufQueue.front(); + const uint8_t* data = buf->data(); + EXPECT_EQ(data[0], 203); // 11001011 + EXPECT_EQ(data[1], 213); // 11010101 + EXPECT_EQ(data[2], 78); // 01001110 + + // size must equal with the actual encoding + string accept("accept-encoding"); + size = huffman::getSize(accept, HPACK::MessageType::REQ); + uint32_t encodedSize = huffman::encode(accept, HPACK::MessageType::REQ, + appender); + EXPECT_EQ(size, encodedSize); +} + +TEST_F(HuffmanTests, decode) { + uint8_t buffer[3]; + // simple test with one byte + buffer[0] = 1; // 0000 0001 + string literal; + huffman::decode(HPACK::MessageType::REQ, buffer, 1, literal); + CHECK_EQ(literal, "/e"); + + // simple test with "gzip" + buffer[0] = 203; + buffer[1] = 213; + buffer[2] = 78; + literal.clear(); + huffman::decode(HPACK::MessageType::REQ, buffer, 3, literal); + EXPECT_EQ(literal, "gzip"); + + // something with padding + buffer[0] = 200; + buffer[1] = 127; + literal.clear(); + huffman::decode(HPACK::MessageType::REQ, buffer, 2, literal); + EXPECT_EQ(literal, "ge"); +} + +/* + * non-printable characters, that use 3 levels + */ +TEST_F(HuffmanTests, non_printable_decode) { + // character code 1 and 46 (.) that have 27 + 5 = 32 bits + uint8_t buffer1[4] = { + 0xFF, 0xFF, 0xF7, 0x64 + }; + string literal; + huffman::decode(HPACK::MessageType::REQ, buffer1, 4, literal); + EXPECT_EQ(literal.size(), 2); + EXPECT_EQ((uint8_t)literal[0], 1); + EXPECT_EQ((uint8_t)literal[1], 46); + + // two weird characters and padding + // 1 and 240 will have 27 + 26 = 53 bits + 3 bits padding + uint8_t buffer2[7] = { + 0xFF, 0xFF, 0xF7, 0x7F, 0xFF, 0xFE, 0x67 + }; + literal.clear(); + huffman::decode(HPACK::MessageType::REQ, buffer2, 7, literal); + EXPECT_EQ(literal.size(), 2); + EXPECT_EQ((uint8_t) literal[0], 1); + EXPECT_EQ((uint8_t) literal[1], 240); +} + +TEST_F(HuffmanTests, example_com) { + // interesting case of one bit with value 0 in the last byte + IOBufQueue bufQueue; + QueueAppender appender(&bufQueue, 512); + appender.ensure(512); + + string example("www.example.com"); + uint32_t size = huffman::getSize(example, HPACK::MessageType::REQ); + EXPECT_EQ(size, 11); + uint32_t encodedSize = + huffman::encode(example, HPACK::MessageType::REQ, appender); + EXPECT_EQ(size, encodedSize); + + string decoded; + huffman::decode(HPACK::MessageType::REQ, + bufQueue.front()->data(), size, decoded); + CHECK_EQ(example, decoded); +} + +TEST_F(HuffmanTests, user_agent) { + string user_agent( + "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_4 like Mac OS X) AppleWebKit/537.51" + ".1 (KHTML, like Gecko) Mobile/11B554a [FBAN/FBIOS;FBAV/6.7;FBBV/566055;FBD" + "V/iPhone5,1;FBMD/iPhone;FBSN/iPhone OS;FBSV/7.0.4;FBSS/2; FBCR/AT&T;FBID/p" + "hone;FBLC/en_US;FBOP/5]"); + for (int i = 0; i < 2; i++) { + IOBufQueue bufQueue; + QueueAppender appender(&bufQueue, 512); + appender.ensure(512); + HPACK::MessageType msgType = (HPACK::MessageType)i; + uint32_t size = huffman::getSize(user_agent, msgType); + uint32_t encodedSize = huffman::encode(user_agent, msgType, appender); + EXPECT_EQ(size, encodedSize); + + string decoded; + huffman::decode(msgType, bufQueue.front()->data(), size, decoded); + CHECK_EQ(user_agent, decoded); + } +} + +/* + * this test is verifying the CHECK for length at the end of huffman::encode() + */ +TEST_F(HuffmanTests, fit_in_buffer) { + IOBufQueue bufQueue; + QueueAppender appender(&bufQueue, 128); + + // call with an empty string + string literal(""); + huffman::encode(literal, HPACK::MessageType::REQ, appender); + + // allow just 1 byte + appender.ensure(128); + appender.append(appender.length() - 1); + literal = "g"; + huffman::encode(literal, HPACK::MessageType::REQ, appender); + CHECK_EQ(appender.length(), 0); +} + +/* + * sanity checks of each node in decode tree performed by a depth first search + * + * allSnodes is an array of up to 46 SuperHuffNode's, the 46 is hardcoded + * in creation + * nodeIndex is the current SuperHuffNode being visited + * depth is the depth of the current SuperHuffNode being visited + * fullCode remembers the code from parent HuffNodes + * eosCode stores the code for End-Of-String characters which the tables + * do not store + * eosCodeBits stores the number of bits for the End-Of-String character + * codes + */ +uint32_t treeDfs( + const SuperHuffNode *allSnodes, + const uint32_t &snodeIndex, + const uint32_t &depth, + const uint32_t &fullCode, + const uint32_t &eosCode, + const uint32_t &eosCodeBits) { + + EXPECT_TRUE(depth < 4); + + unordered_set leaves; + uint32_t subtreeLeafCount = 0; + + for (uint32_t i = 0; i < 256; i++) { + const HuffNode& node = allSnodes[snodeIndex].index[i]; + + uint32_t newFullCode = fullCode ^ (i << (24 - 8 * depth)); + uint32_t eosCodeDepth = (eosCodeBits - 1) / 8; + + if(eosCodeDepth == depth + && (newFullCode >> (32 - eosCodeBits)) == eosCode) { + + // this condition corresponds to an EOS code + // this should be a leaf that doesn't have supernode or bits set + + EXPECT_TRUE(node.isLeaf()); + EXPECT_TRUE(node.superNode == 0); + EXPECT_TRUE(node.bits == 0); + } else if (node.isLeaf()) { + + // this condition is a normal leaf + // this should have bits set but superNode should be 0 + + EXPECT_TRUE(node.superNode == 0); + EXPECT_TRUE(node.bits > 0); + + EXPECT_TRUE(node.bits <= 8); + + // used to count unique leaves at this node + leaves.insert(node.ch); + } else { + + // this condition is a branching node + // this should have the superNode set but not bits should be set + + EXPECT_TRUE(node.superNode > 0); + EXPECT_TRUE(node.bits == 0); + + EXPECT_TRUE(node.superNode < 46); + + // keep track of leaf counts for this subtree + subtreeLeafCount += treeDfs( + allSnodes, + node.superNode, + depth + 1, + newFullCode, + eosCode, + eosCodeBits); + } + } + return subtreeLeafCount + leaves.size(); +} + +/** + * Class used in testing to expose the internal tables for requests + * and responses + */ +class TestingHuffTree : public HuffTree { + public: + + explicit TestingHuffTree(const HuffTree& tree) : HuffTree(tree) {} + + const SuperHuffNode* getInternalTable() { + return table_; + } + + static TestingHuffTree getReqHuffTree() { + TestingHuffTree reqTree(reqHuffTree()); + return reqTree; + } + + static TestingHuffTree getRespHuffTree() { + TestingHuffTree respTree(respHuffTree()); + return respTree; + } + +}; + +TEST_F(HuffmanTests, sanity_checks) { + TestingHuffTree reqTree = TestingHuffTree::getReqHuffTree(); + const SuperHuffNode* allSnodesReq = reqTree.getInternalTable(); + uint32_t totalReqChars = treeDfs(allSnodesReq, 0, 0, 0, 0x3ffffdc, 26); + EXPECT_EQ(totalReqChars, 256); + + TestingHuffTree respTree = TestingHuffTree::getRespHuffTree(); + const SuperHuffNode* allSnodesResp = respTree.getInternalTable(); + uint32_t totalRespChars = treeDfs(allSnodesResp, 0, 0, 0, 0xffffdd, 24); + EXPECT_EQ(totalRespChars, 256); +} diff --git a/proxygen/lib/http/codec/compress/test/LoggingTests.cpp b/proxygen/lib/http/codec/compress/test/LoggingTests.cpp new file mode 100644 index 0000000000..bd728f12f6 --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/LoggingTests.cpp @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKEncodeBuffer.h" +#include "proxygen/lib/http/codec/compress/HPACKHeader.h" +#include "proxygen/lib/http/codec/compress/Logging.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace folly; +using namespace proxygen; +using namespace std; +using namespace testing; + +class LoggingTests : public testing::Test { +}; + +TEST_F(LoggingTests, hex_iobuf) { + unique_ptr buf = IOBuf::create(128); + stringstream out; + out << buf.get(); + EXPECT_EQ(out.str(), ""); + + uint8_t* data = buf->writableData(); + data[0] = 0x0C; + data[1] = 0xFF; + data[2] = 0x00; + data[3] = 0x10; + buf->append(4); + out << buf.get(); + EXPECT_EQ(out.str(), "0cff 0010 "); + + // some linewrap + for (int i = 0; i < 16; i++) { + data[4 + i] = 0xFE; + } + buf->append(16); + out.str(""); + out << buf.get(); + EXPECT_EQ(out.str(), "0cff 0010 fefe fefe fefe fefe fefe fefe \nfefe fefe "); +} + +TEST_F(LoggingTests, refset) { + list refset; + refset.push_back(3); + refset.push_back(5); + stringstream out; + out << &refset; + EXPECT_EQ(out.str(), "\n[3 5 ]\n"); +} + +TEST_F(LoggingTests, dump_header_vector) { + vector headers; + headers.push_back(HPACKHeader(":path", "index.html")); + headers.push_back(HPACKHeader("content-type", "gzip")); + stringstream out; + out << headers; + EXPECT_EQ(out.str(), ":path: index.html\ncontent-type: gzip\n\n"); +} + +TEST_F(LoggingTests, dump_chain) { + unique_ptr b1 = IOBuf::create(128); + EXPECT_TRUE(dumpChain(b1.get()) != ""); +} + +TEST_F(LoggingTests, dump_bin) { + // null IOBuf + EXPECT_EQ(dumpBin(nullptr, 1), ""); + + unique_ptr b1 = IOBuf::create(128); + b1->writableData()[0] = 0x33; + b1->writableData()[1] = 0x77; + b1->append(2); + unique_ptr b2 = IOBuf::create(128); + b2->writableData()[0] = 0xFF; + b1->appendChain(std::move(b2)); + EXPECT_EQ(dumpBin(b1.get(), 1), "00110011 3\n01110111 w\n\n\n"); + EXPECT_EQ(dumpBin(b1.get(), 2), "00110011 3 01110111 w\n\n\n"); + // test with an HPACKEncodeBuffer + HPACKEncodeBuffer buf(128); + buf.encodeLiteral("test"); + EXPECT_EQ(buf.toBin(), + "00000100 01110100 t 01100101 e 01110011 s 01110100 t \n"); +} + +TEST_F(LoggingTests, print_delta) { + vector v1; + v1.push_back(HPACKHeader(":path", "/")); + v1.push_back(HPACKHeader(":host", "www.facebook.com")); + vector v2; + + // empty v1 or v2 + EXPECT_EQ(printDelta(v1, v2), "\n + :path: /\n + :host: www.facebook.com\n"); + EXPECT_EQ(printDelta(v2, v1), "\n - :path: /\n - :host: www.facebook.com\n"); + + // skip first header from v1 + v2.push_back(HPACKHeader(":path", "/")); + EXPECT_EQ(printDelta(v1, v2), "\n + :host: www.facebook.com\n"); + + v2.push_back(HPACKHeader(":path", "/")); + EXPECT_EQ(printDelta(v2, v1), + "\n - :host: www.facebook.com\n + :path: / (duplicate)\n"); + + v2.pop_back(); + v1.clear(); + v1.push_back(HPACKHeader(":a", "b")); + v1.push_back(HPACKHeader(":a", "b")); + v1.push_back(HPACKHeader(":path", "/")); + EXPECT_EQ(printDelta(v1, v2), "\n + :a: b\n duplicate :a: b\n"); +} + +TEST_F(LoggingTests, dump_bin_to_file) { + struct stat fstat; + string tmpfile("/tmp/test.bin"); + + unlink(tmpfile.c_str()); + unique_ptr buf = IOBuf::create(128); + // the content doesn't matter + buf->append(2); + dumpBinToFile(tmpfile, buf.get()); + EXPECT_EQ(stat(tmpfile.c_str(), &fstat), 0); + + // check if it's going to overwrite the existing file + buf->append(4); + dumpBinToFile(tmpfile, buf.get()); + EXPECT_EQ(stat(tmpfile.c_str(), &fstat), 0); + EXPECT_EQ(fstat.st_size, 2); + unlink(tmpfile.c_str()); + + // null iobuf + dumpBinToFile(tmpfile, nullptr); + // unable to open file + dumpBinToFile("/proc/test", nullptr); +} diff --git a/proxygen/lib/http/codec/compress/test/RFCExamplesTests.cpp b/proxygen/lib/http/codec/compress/test/RFCExamplesTests.cpp new file mode 100644 index 0000000000..9d0f4672ab --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/RFCExamplesTests.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/HPACKDecoder.h" +#include "proxygen/lib/http/codec/compress/HPACKEncoder.h" +#include "proxygen/lib/http/codec/compress/Logging.h" +#include "proxygen/lib/http/codec/compress/test/TestUtil.h" + +#include +#include +#include +#include +#include + +using namespace folly; +using namespace proxygen; +using namespace std; +using namespace testing; + +class RFCExamplesTests : public testing::Test { + public: + RFCExamplesTests() { + req1.push_back(HPACKHeader(":method", "GET")); + req1.push_back(HPACKHeader(":scheme", "http")); + req1.push_back(HPACKHeader(":path", "/")); + req1.push_back(HPACKHeader(":authority", "www.example.com")); + + req2.push_back(HPACKHeader(":method", "GET")); + req2.push_back(HPACKHeader(":scheme", "http")); + req2.push_back(HPACKHeader(":path", "/")); + req2.push_back(HPACKHeader(":authority", "www.example.com")); + req2.push_back(HPACKHeader("cache-control", "no-cache")); + + req3.push_back(HPACKHeader(":method", "GET")); + req3.push_back(HPACKHeader(":scheme", "https")); + req3.push_back(HPACKHeader(":path", "/index.html")); + req3.push_back(HPACKHeader(":authority", "www.example.com")); + req3.push_back(HPACKHeader("custom-key", "custom-value")); + + resp1.push_back(HPACKHeader(":status", "302")); + resp1.push_back(HPACKHeader("cache-control", "private")); + resp1.push_back(HPACKHeader("date", "Mon, 21 Oct 2013 20:13:21 GMT")); + resp1.push_back(HPACKHeader("location", "https://www.example.com")); + + resp2.push_back(HPACKHeader(":status", "200")); + resp2.push_back(HPACKHeader("cache-control", "private")); + resp2.push_back(HPACKHeader("date", "Mon, 21 Oct 2013 20:13:21 GMT")); + resp2.push_back(HPACKHeader("location", "https://www.example.com")); + + resp3.push_back(HPACKHeader(":status", "200")); + resp3.push_back(HPACKHeader("cache-control", "private")); + resp3.push_back(HPACKHeader("date", "Mon, 21 Oct 2013 20:13:22 GMT")); + resp3.push_back(HPACKHeader("location", "https://www.example.com")); + resp3.push_back(HPACKHeader("content-encoding", "gzip")); + resp3.push_back( + HPACKHeader( + "set-cookie", + "foo=ASDJKHQKBZXOQWEOPIUAXQWEOIU; max-age=3600; version=1") + ); + } + + protected: + vector req1; + vector req2; + vector req3; + vector resp1; + vector resp2; + vector resp3; +}; + +TEST_F(RFCExamplesTests, rfc_example_e2_request_no_huffman) { + HPACKEncoder encoder(HPACK::MessageType::REQ, false); + HPACKDecoder decoder(HPACK::MessageType::REQ); + // first request + unique_ptr encoded = hpack::encodeDecode(req1, encoder, decoder); + EXPECT_EQ(encoded->length(), 20); + EXPECT_EQ(encoder.getTable().bytes(), 180); + EXPECT_EQ(encoder.getTable().size(), 4); + auto refset = encoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 4); + + // second request + encoded = hpack::encodeDecode(req2, encoder, decoder); + EXPECT_EQ(encoded->length(), 10); + EXPECT_EQ(encoder.getTable().bytes(), 233); + EXPECT_EQ(encoder.getTable().size(), 5); + refset = encoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 5); + + // third request + encoded = hpack::encodeDecode(req3, encoder, decoder); + EXPECT_EQ(encoded->length(), 30); + EXPECT_EQ(encoder.getTable().bytes(), 379); + EXPECT_EQ(encoder.getTable().size(), 8); + refset = encoder.getTable().referenceSet(); + EXPECT_EQ(decoder.getTable().referenceSet(), refset); +} + +TEST_F(RFCExamplesTests, rfc_example_e3_request_with_huffman) { + HPACKEncoder encoder(HPACK::MessageType::REQ, true); + HPACKDecoder decoder(HPACK::MessageType::REQ); + + // first + unique_ptr encoded = hpack::encodeDecode(req1, encoder, decoder); + EXPECT_EQ(encoded->length(), 16); + EXPECT_EQ(encoder.getTable().bytes(), 180); + EXPECT_EQ(encoder.getTable().size(), 4); + // verify the last byte + EXPECT_EQ(encoded->data()[15], 0x7F); + EXPECT_EQ(decoder.getTable().bytes(), 180); + EXPECT_EQ(decoder.getTable().size(), 4); + auto refset = decoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 4); + + // second + encoded = hpack::encodeDecode(req2, encoder, decoder); + EXPECT_EQ(encoded->length(), 8); + EXPECT_EQ(encoder.getTable().bytes(), 233); + EXPECT_EQ(encoder.getTable().size(), 5); + refset = decoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 5); + + // third + encoded = hpack::encodeDecode(req3, encoder, decoder); + EXPECT_EQ(encoded->length(), 25); + EXPECT_EQ(encoder.getTable().bytes(), 379); + EXPECT_EQ(encoder.getTable().size(), 8); + refset = decoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 5); +} + +TEST_F(RFCExamplesTests, rfc_example_e4_response_no_huffman) { + // this test does some evictions + uint32_t tableSize = 256; + HPACKEncoder encoder(HPACK::MessageType::RESP, false, tableSize); + HPACKDecoder decoder(HPACK::MessageType::RESP, tableSize); + + // first + unique_ptr encoded = hpack::encodeDecode(resp1, encoder, decoder); + EXPECT_EQ(encoded->length(), 70); + EXPECT_EQ(encoder.getTable().bytes(), 222); + EXPECT_EQ(encoder.getTable().size(), 4); + auto refset = encoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 4); + + // second + encoded = hpack::encodeDecode(resp2, encoder, decoder); + EXPECT_EQ(encoded->length(), 2); + EXPECT_EQ(encoded->data()[0], 0x84); + EXPECT_EQ(encoded->data()[1], 0x8c); + EXPECT_EQ(encoder.getTable().bytes(), 222); + EXPECT_EQ(encoder.getTable().size(), 4); + refset = encoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 4); + + // third + encoded = hpack::encodeDecode(resp3, encoder, decoder); + EXPECT_EQ(encoded->length(), 102); + EXPECT_EQ(encoded->data()[0], 0x83); + // this sequence of two identical bytes is the reference eviction + EXPECT_EQ(encoded->data()[1], 0x84); + EXPECT_EQ(encoded->data()[2], 0x84); + // last byte + EXPECT_EQ(encoded->data()[101], 0x31); + + EXPECT_EQ(encoder.getTable().size(), 3); + EXPECT_EQ(encoder.getTable().bytes(), 215); +} + +TEST_F(RFCExamplesTests, rfc_example_e5_response_with_huffman) { + uint32_t tableSize = 256; + HPACKEncoder encoder(HPACK::MessageType::RESP, true, tableSize); + HPACKDecoder decoder(HPACK::MessageType::RESP, tableSize); + + // first + unique_ptr encoded = hpack::encodeDecode(resp1, encoder, decoder); + EXPECT_EQ(encoded->length(), 53); + EXPECT_EQ(encoded->data()[52], 0xFF); + EXPECT_EQ(encoder.getTable().size(), 4); + EXPECT_EQ(encoder.getTable().bytes(), 222); + auto refset = encoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 4); + + // second + encoded = hpack::encodeDecode(resp2, encoder, decoder); + EXPECT_EQ(encoded->length(), 2); + EXPECT_EQ(encoded->data()[0], 0x84); + EXPECT_EQ(encoded->data()[1], 0x8c); + EXPECT_EQ(encoder.getTable().bytes(), 222); + EXPECT_EQ(encoder.getTable().size(), 4); + refset = encoder.getTable().referenceSet(); + EXPECT_EQ(refset.size(), 4); + + // third + encoded = hpack::encodeDecode(resp3, encoder, decoder); + EXPECT_EQ(encoded->length(), 86); + EXPECT_EQ(encoded->data()[0], 0x83); + // this sequence of two identical bytes is the reference eviction + EXPECT_EQ(encoded->data()[1], 0x84); + EXPECT_EQ(encoded->data()[2], 0x84); + // last byte + EXPECT_EQ(encoded->data()[85], 0x2F); + + EXPECT_EQ(encoder.getTable().size(), 3); + EXPECT_EQ(encoder.getTable().bytes(), 215); +} diff --git a/proxygen/lib/http/codec/compress/test/TestUtil.cpp b/proxygen/lib/http/codec/compress/test/TestUtil.cpp new file mode 100644 index 0000000000..a78cbdf3cd --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/TestUtil.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/compress/test/TestUtil.h" + +#include "proxygen/lib/http/codec/compress/Logging.h" + +#include +#include +#include + +using folly::IOBuf; +using std::ofstream; +using std::string; +using std::unique_ptr; +using std::vector; + +namespace proxygen { namespace hpack { + +void dumpToFile(const string& filename, const IOBuf* buf) { + ofstream outfile(filename, ofstream::binary); + if (buf) { + const IOBuf* p = buf; + do { + outfile.write((const char *)p->data(), p->length()); + p = p->next(); + } while (p->next() != buf); + } + outfile.close(); +} + +unique_ptr encodeDecode( + vector headers, + HPACKEncoder& encoder, + HPACKDecoder& decoder) { + unique_ptr encoded = encoder.encode(headers); + auto decodedHeaders = decoder.decode(encoded.get()); + CHECK(!decoder.hasError()); + + EXPECT_EQ(headers.size(), decodedHeaders->size()); + CHECK(headers.size() == decodedHeaders->size()); + sort(decodedHeaders->begin(), decodedHeaders->end()); + sort(headers.begin(), headers.end()); + if (headers.size() != decodedHeaders->size()) { + printDelta(*decodedHeaders, headers); + } + EXPECT_EQ(headers, *decodedHeaders); + // header tables should look the same + CHECK(encoder.getTable() == decoder.getTable()); + EXPECT_EQ(encoder.getTable(), decoder.getTable()); + + return std::move(encoded); +} + +}} diff --git a/proxygen/lib/http/codec/compress/test/TestUtil.h b/proxygen/lib/http/codec/compress/test/TestUtil.h new file mode 100644 index 0000000000..d604a1bfc0 --- /dev/null +++ b/proxygen/lib/http/codec/compress/test/TestUtil.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/compress/HPACKDecoder.h" +#include "proxygen/lib/http/codec/compress/HPACKEncoder.h" + +#include +#include +#include + +namespace proxygen { namespace hpack { + +void dumpToFile(const std::string& filename, const folly::IOBuf* buf); + +std::unique_ptr encodeDecode( + std::vector headers, + HPACKEncoder& encoder, + HPACKDecoder& decoder); + +}} diff --git a/proxygen/lib/http/codec/test/FilterTests.cpp b/proxygen/lib/http/codec/test/FilterTests.cpp new file mode 100644 index 0000000000..35923018a2 --- /dev/null +++ b/proxygen/lib/http/codec/test/FilterTests.cpp @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/http/codec/FlowControlFilter.h" +#include "proxygen/lib/http/codec/HTTPChecks.h" +#include "proxygen/lib/http/codec/SPDYConstants.h" +#include "proxygen/lib/http/codec/test/MockHTTPCodec.h" +#include "proxygen/lib/http/codec/test/TestUtils.h" + +#include +#include +#include + +using namespace proxygen; +using namespace std; +using namespace testing; + +class MockFlowControlCallback: public FlowControlFilter::Callback { + public: + MOCK_METHOD0(onConnectionSendWindowOpen, void()); +}; + +class FilterTest : public testing::Test { + public: + FilterTest(): + codec_(new MockHTTPCodec()), + chain_(unique_ptr(codec_)) { + EXPECT_CALL(*codec_, setCallback(_)) + .WillRepeatedly(SaveArg<0>(&callbackStart_)); + chain_.setCallback(&callback_); + } + protected: + + MockHTTPCodec* codec_; + HTTPCodec::Callback* callbackStart_; + HTTPCodecFilterChain chain_; + MockHTTPCodecCallback callback_; + folly::IOBufQueue writeBuf_{folly::IOBufQueue::cacheChainLength()}; +}; + +class HTTPChecksTest: public FilterTest { + public: + void SetUp() override { + chain_.add(); + } +}; + +template +class FlowControlFilterTest: public FilterTest { + public: + void SetUp() override { + if (initSize > spdy::kInitialWindow) { + // If the initial size is bigger than the default, a window update + // will immediately be generated + EXPECT_CALL(*codec_, generateWindowUpdate(_, 0, initSize - + spdy::kInitialWindow)) + .WillOnce(InvokeWithoutArgs([this] () { + writeBuf_.append(makeBuf(10)); + return 10; + })); + } + EXPECT_CALL(*codec_, generateBody(_, _, _, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + std::shared_ptr chain, + bool eom) { + auto len = chain->computeChainDataLength() + 4; + writeBuf.append(makeBuf(len)); + return len; + })); + EXPECT_CALL(*codec_, isReusable()).WillRepeatedly(Return(true)); + + // Construct flow control filter with capacity of 0, which will be + // overridden to 65536, which is the minimum + filter_ = new FlowControlFilter(flowCallback_, writeBuf_, codec_, initSize); + chain_.addFilters(std::unique_ptr(filter_)); + } + StrictMock flowCallback_; + FlowControlFilter* filter_; + int recvWindow_{initSize}; +}; + +typedef FlowControlFilterTest<0> DefaultFlowControl; +typedef FlowControlFilterTest<1000000> BigWindow; + +MATCHER(IsFlowException, "") { + return arg->hasCodecStatusCode() && + arg->getCodecStatusCode() == ErrorCode::FLOW_CONTROL_ERROR && + !arg->hasHttpStatusCode() && + !arg->hasProxygenError(); +} + +TEST_F(DefaultFlowControl, flow_control_construct) { + // Constructing the filter with a low capacity defaults to spdy's + // initial capacity, so no window update should have been generated in + // the constructor + InSequence enforceSequence; + ASSERT_EQ(writeBuf_.chainLength(), 0); + + // Our send window is limited to spdy::kInitialWindow + chain_->generateBody(writeBuf_, 1, + makeBuf(spdy::kInitialWindow - 1), false); + + // the window isn't full yet, so getting a window update shouldn't give a + // callback informing us that it is open again + callbackStart_->onWindowUpdate(0, 1); + + // Now fill the window (2 more bytes) + chain_->generateBody(writeBuf_, 1, makeBuf(2), false); + // get the callback informing the window is open once we get a window update + EXPECT_CALL(flowCallback_, onConnectionSendWindowOpen()); + callbackStart_->onWindowUpdate(0, 1); + + // Overflowing the window is fatal. Write 2 bytes (only 1 byte left in window) + EXPECT_DEATH_NO_CORE(chain_->generateBody(writeBuf_, 1, makeBuf(2), false), + ".*"); +} + +TEST_F(DefaultFlowControl, send_update) { + // Make sure we send a window update when the window decreases below half + InSequence enforceSequence; + EXPECT_CALL(callback_, onBody(_, _)) + .WillRepeatedly(Return()); + + // Have half the window outstanding + callbackStart_->onBody(1, makeBuf(spdy::kInitialWindow / 2 + 1)); + filter_->ingressBytesProcessed(writeBuf_, spdy::kInitialWindow / 2); + + // It should wait until the "+1" is ack'd to generate the coallesced update + EXPECT_CALL(*codec_, + generateWindowUpdate(_, 0, spdy::kInitialWindow / 2 + 1)); + filter_->ingressBytesProcessed(writeBuf_, 1); +} + +TEST_F(BigWindow, recv_too_much) { + // Constructing the filter with a large capacity causes a WINDOW_UPDATE + // for stream zero to be generated + ASSERT_GT(writeBuf_.chainLength(), 0); + + InSequence enforceSequence; + EXPECT_CALL(callback_, onBody(_, _)); + EXPECT_CALL(callback_, onError(0, IsFlowException(), _)); + + // Receive the max amount advertised + callbackStart_->onBody(1, makeBuf(recvWindow_)); + ASSERT_TRUE(chain_->isReusable()); + // Receive 1 byte too much + callbackStart_->onBody(1, makeBuf(1)); + ASSERT_FALSE(chain_->isReusable()); +} + +TEST_F(BigWindow, remote_increase) { + // The remote side sends us a window update for stream=0, increasing our + // available window + InSequence enforceSequence; + + ASSERT_EQ(filter_->getAvailableSend(), spdy::kInitialWindow); + callbackStart_->onWindowUpdate(0, 10); + ASSERT_EQ(filter_->getAvailableSend(), spdy::kInitialWindow + 10); + + chain_->generateBody(writeBuf_, 1, + makeBuf(spdy::kInitialWindow + 10), false); + ASSERT_EQ(filter_->getAvailableSend(), 0); + + // Now the remote side sends a HUGE update (just barely legal) + // Since the window was full, this generates a callback from the filter + // telling us the window is no longer full. + EXPECT_CALL(flowCallback_, onConnectionSendWindowOpen()); + callbackStart_->onWindowUpdate(0, std::numeric_limits::max()); + ASSERT_EQ(filter_->getAvailableSend(), std::numeric_limits::max()); + + // Now overflow it by 1 + EXPECT_CALL(callback_, onError(0, IsFlowException(), _)); + callbackStart_->onWindowUpdate(0, 1); + ASSERT_FALSE(chain_->isReusable()); +} + +TEST_F(HTTPChecksTest, send_trace_body_death) { + // It is NOT allowed to send a TRACE with a body. + + HTTPMessage msg = getPostRequest(); + msg.setMethod("TRACE"); + + EXPECT_DEATH_NO_CORE(chain_->generateHeader(writeBuf_, 0, msg, 0), ".*"); +} + +TEST_F(HTTPChecksTest, send_get_body) { + // It is allowed to send a GET with a content-length. It is up to the + // server to ignore it. + + EXPECT_CALL(*codec_, generateHeader(_, _, _, _, _)); + + HTTPMessage msg = getPostRequest(); + msg.setMethod("GET"); + + chain_->generateHeader(writeBuf_, 0, msg, 0); +} + +TEST_F(HTTPChecksTest, recv_trace_body) { + // In proxygen, we deal with receiving a TRACE with a body by 400'ing it + + EXPECT_CALL(callback_, onError(_, _, _)) + .WillOnce(Invoke([] (HTTPCodec::StreamID, + std::shared_ptr exc, + bool newTxn) { + ASSERT_TRUE(newTxn); + ASSERT_EQ(exc->getHttpStatusCode(), 400); + })); + + auto msg = makePostRequest(); + msg->setMethod("TRACE"); + + callbackStart_->onHeadersComplete(0, std::move(msg)); +} diff --git a/proxygen/lib/http/codec/test/HTTP1xCodecTest.cpp b/proxygen/lib/http/codec/test/HTTP1xCodecTest.cpp new file mode 100644 index 0000000000..2df26f83a5 --- /dev/null +++ b/proxygen/lib/http/codec/test/HTTP1xCodecTest.cpp @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/http/codec/HTTP1xCodec.h" + +#include + +using namespace proxygen; +using namespace std; + +class HTTP1xCodecCallback : public HTTPCodec::Callback { + public: + HTTP1xCodecCallback() {} + + void onMessageBegin(HTTPCodec::StreamID stream, HTTPMessage* msg) {} + void onPushMessageBegin(HTTPCodec::StreamID stream, + HTTPCodec::StreamID assocStream, + HTTPMessage* msg) {} + void onHeadersComplete(HTTPCodec::StreamID stream, + std::unique_ptr msg) { + headersComplete++; + headerSize = msg->getIngressHeaderSize(); + } + void onBody(HTTPCodec::StreamID stream, + std::unique_ptr chain) {} + void onChunkHeader(HTTPCodec::StreamID stream, size_t length) {} + void onChunkComplete(HTTPCodec::StreamID stream) {} + void onTrailersComplete(HTTPCodec::StreamID stream, + std::unique_ptr trailers) {} + void onMessageComplete(HTTPCodec::StreamID stream, bool upgrade) {} + void onError(HTTPCodec::StreamID stream, + const HTTPException& error, bool newTxn) { + LOG(ERROR) << "parse error"; + } + + uint32_t headersComplete{0}; + HTTPHeaderSize headerSize; +}; + +unique_ptr getSimpleRequestData() { + string req("GET /yeah HTTP/1.1\nHost: www.facebook.com\n\n"); + auto buffer = folly::IOBuf::copyBuffer(req); + return std::move(buffer); +} + +TEST(HTTP1xCodecTest, TestSimpleHeaders) { + HTTP1xCodec codec(TransportDirection::DOWNSTREAM); + HTTP1xCodecCallback callbacks; + codec.setCallback(&callbacks); + auto buffer = getSimpleRequestData(); + codec.onIngress(*buffer); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(buffer->length(), callbacks.headerSize.uncompressed); + EXPECT_EQ(callbacks.headerSize.compressed, 0); +} + +unique_ptr getChunkedRequest1st() { + string req("GET /aha HTTP/1.1\n"); + auto buffer = folly::IOBuf::copyBuffer(req); + return std::move(buffer); +} + +unique_ptr getChunkedRequest2nd() { + string req("Host: m.facebook.com\nAccept-Encoding: meflate\n\n"); + auto buffer = folly::IOBuf::copyBuffer(req); + return std::move(buffer); +} + +TEST(HTTP1xCodecTest, TestChunkedHeaders) { + HTTP1xCodec codec(TransportDirection::DOWNSTREAM); + HTTP1xCodecCallback callbacks; + codec.setCallback(&callbacks); + // test a sequence of requests to make sure we're resetting the size counter + for (int i = 0; i < 3; i++) { + callbacks.headersComplete = 0; + auto buffer1 = getChunkedRequest1st(); + codec.onIngress(*buffer1); + EXPECT_EQ(callbacks.headersComplete, 0); + + auto buffer2 = getChunkedRequest2nd(); + codec.onIngress(*buffer2); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.headerSize.uncompressed, + buffer1->length() + buffer2->length()); + } +} + +TEST(HTTP1xCodecTest, TestChunkedUpstream) { + HTTP1xCodec codec(TransportDirection::UPSTREAM); + + auto txnID = codec.createStream(); + + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.setURL("https://www.facebook.com/"); + msg.getHeaders().set("Host", "www.facebook.com"); + msg.getHeaders().set("Transfer-Encoding", "chunked"); + msg.setIsChunked(true); + + HTTPHeaderSize size; + + folly::IOBufQueue buf(folly::IOBufQueue::cacheChainLength()); + codec.generateHeader(buf, txnID, msg, 0, &size); + auto headerFromBuf = buf.split(buf.chainLength()); + + string req1("Hello"); + auto body1 = folly::IOBuf::copyBuffer(req1); + + string req2("World"); + auto body2 = folly::IOBuf::copyBuffer(req2); + + codec.generateBody(buf, txnID, std::move(body1), false); + + auto bodyFromBuf = buf.split(buf.chainLength()); + ASSERT_EQ("5\r\nHello\r\n", bodyFromBuf->moveToFbString()); + + codec.generateBody(buf, txnID, std::move(body2), true); + LOG(WARNING) << "len chain" << buf.chainLength(); + + auto eomFromBuf = buf.split(buf.chainLength()); + ASSERT_EQ("5\r\nWorld\r\n0\r\n\r\n", eomFromBuf->moveToFbString()); +} diff --git a/proxygen/lib/http/codec/test/Makefile.am b/proxygen/lib/http/codec/test/Makefile.am new file mode 100644 index 0000000000..7d2d951b18 --- /dev/null +++ b/proxygen/lib/http/codec/test/Makefile.am @@ -0,0 +1,21 @@ +SUBDIRS = . + +check_LTLIBRARIES = libcodectestutils.la +libcodectestutils_la_SOURCES = TestUtils.cpp + +libcodectestutilsdir = $(includedir)/proxygen/lib/http/codec/test +nobase_libcodectestutils_HEADERS = TestUtils.h + +check_PROGRAMS = CodecTests +CodecTests_SOURCES = \ + FilterTests.cpp \ + SPDYCodecTest.cpp \ + HTTP1xCodecTest.cpp + +CodecTests_LDADD = \ + ../../libproxygenhttp.la \ + ../../../utils/libutils.la \ + libcodectestutils.la \ + ../../../test/libtestmain.la + +TESTS = CodecTests diff --git a/proxygen/lib/http/codec/test/MockHTTPCodec.h b/proxygen/lib/http/codec/test/MockHTTPCodec.h new file mode 100644 index 0000000000..22b047cf6c --- /dev/null +++ b/proxygen/lib/http/codec/test/MockHTTPCodec.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/HTTPCodec.h" + +#include + +namespace proxygen { + +class MockHTTPCodec: public HTTPCodec { + public: + MOCK_CONST_METHOD0(getProtocol, CodecProtocol()); + MOCK_CONST_METHOD0(getTransportDirection, TransportDirection()); + MOCK_CONST_METHOD0(supportsStreamFlowControl, bool()); + MOCK_CONST_METHOD0(supportsSessionFlowControl, bool()); + MOCK_METHOD0(createStream, HTTPCodec::StreamID()); + MOCK_METHOD1(setCallback, void(Callback*)); + MOCK_CONST_METHOD0(isBusy, bool()); + MOCK_CONST_METHOD0(hasPartialTransaction, bool()); + MOCK_METHOD1(setParserPaused, void(bool)); + MOCK_METHOD1(onIngress, size_t(const folly::IOBuf&)); + MOCK_METHOD0(onIngressEOF, void()); + MOCK_CONST_METHOD0(isReusable, bool()); + MOCK_CONST_METHOD0(isWaitingToDrain, bool()); + MOCK_CONST_METHOD0(closeOnEgressComplete, bool()); + MOCK_CONST_METHOD0(supportsParallelRequests, bool()); + MOCK_CONST_METHOD0(supportsPushTransactions, bool()); + MOCK_METHOD5(generateHeader, void(folly::IOBufQueue&, + HTTPCodec::StreamID, + const HTTPMessage&, + HTTPCodec::StreamID, + HTTPHeaderSize*)); + MOCK_METHOD4(generateBody, size_t(folly::IOBufQueue&, + HTTPCodec::StreamID, + std::shared_ptr, + bool)); + size_t generateBody(folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + std::unique_ptr chain, + bool eom) override { + return generateBody(writeBuf, + stream, + std::shared_ptr(chain.release()), + eom); + } + MOCK_METHOD3(generateChunkHeader, size_t(folly::IOBufQueue&, + HTTPCodec::StreamID, + size_t)); + MOCK_METHOD2(generateChunkTerminator, size_t(folly::IOBufQueue&, + HTTPCodec::StreamID)); + MOCK_METHOD3(generateTrailers, size_t(folly::IOBufQueue&, + HTTPCodec::StreamID, + const HTTPHeaders&)); + MOCK_METHOD2(generateEOM, size_t(folly::IOBufQueue&, + HTTPCodec::StreamID)); + MOCK_METHOD3(generateRstStream, size_t(folly::IOBufQueue&, + HTTPCodec::StreamID, + ErrorCode)); + MOCK_METHOD3(generateGoaway, size_t(folly::IOBufQueue&, + StreamID, + ErrorCode)); + MOCK_METHOD1(generatePingRequest, size_t(folly::IOBufQueue&)); + MOCK_METHOD2(generatePingReply, size_t(folly::IOBufQueue&, + uint64_t)); + MOCK_METHOD1(generateSettings, size_t(folly::IOBufQueue&)); + MOCK_METHOD3(generateWindowUpdate, size_t(folly::IOBufQueue&, + StreamID, + uint32_t)); + MOCK_METHOD0(getEgressSettings, HTTPSettings*()); + MOCK_CONST_METHOD0(getIngressSettings, const HTTPSettings*()); + MOCK_METHOD0(enableDoubleGoawayDrain, void()); + }; + +class MockHTTPCodecCallback: public HTTPCodec::Callback { + public: + MOCK_METHOD2(onMessageBegin, void(HTTPCodec::StreamID, HTTPMessage*)); + MOCK_METHOD3(onPushMessageBegin, void(HTTPCodec::StreamID, + HTTPCodec::StreamID, + HTTPMessage*)); + MOCK_METHOD2(onHeadersComplete, void(HTTPCodec::StreamID, + std::shared_ptr)); + void onHeadersComplete(HTTPCodec::StreamID stream, + std::unique_ptr msg) override { + onHeadersComplete(stream, std::shared_ptr(msg.release())); + } + MOCK_METHOD2(onBody, void(HTTPCodec::StreamID, + std::shared_ptr)); + void onBody(HTTPCodec::StreamID stream, + std::unique_ptr chain) override { + onBody(stream, std::shared_ptr(chain.release())); + } + MOCK_METHOD2(onChunkHeader, void(HTTPCodec::StreamID, size_t)); + MOCK_METHOD1(onChunkComplete, void(HTTPCodec::StreamID)); + MOCK_METHOD2(onTrailersComplete, void(HTTPCodec::StreamID, + std::shared_ptr)); + void onTrailersComplete(HTTPCodec::StreamID stream, + std::unique_ptr trailers) override { + onTrailersComplete(stream, + std::shared_ptr(trailers.release())); + } + MOCK_METHOD2(onMessageComplete, void(HTTPCodec::StreamID, bool)); + MOCK_METHOD3(onError, void(HTTPCodec::StreamID, + std::shared_ptr, bool)); + void onError(HTTPCodec::StreamID stream, + const HTTPException& exc, + bool newStream) override { + onError(stream, + std::shared_ptr(new HTTPException(exc)), + newStream); + } + MOCK_METHOD2(onAbort, void(HTTPCodec::StreamID, ErrorCode)); + MOCK_METHOD2(onGoaway, void(uint64_t, ErrorCode)); + MOCK_METHOD1(onPingRequest, void(uint64_t)); + MOCK_METHOD1(onPingReply, void(uint64_t)); + MOCK_METHOD2(onWindowUpdate, void(HTTPCodec::StreamID, uint32_t)); + MOCK_METHOD1(onSettings, void(const SettingsList&)); + MOCK_METHOD0(onSettingsAck, void()); + MOCK_CONST_METHOD0(numOutgoingStreams, uint32_t()); + MOCK_CONST_METHOD0(numIncomingStreams, uint32_t()); +}; + +} diff --git a/proxygen/lib/http/codec/test/SPDYCodecTest.cpp b/proxygen/lib/http/codec/test/SPDYCodecTest.cpp new file mode 100644 index 0000000000..47856c481b --- /dev/null +++ b/proxygen/lib/http/codec/test/SPDYCodecTest.cpp @@ -0,0 +1,1228 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/http/codec/SPDYCodec.h" +#include "proxygen/lib/http/codec/SPDYConstants.h" +#include "proxygen/lib/http/codec/SPDYVersionSettings.h" +#include "proxygen/lib/http/codec/test/TestUtils.h" + +#include +#include + +using namespace folly; +using namespace proxygen; +using namespace std; +using namespace testing; + +size_t parseSPDY(SPDYCodec* codec, const uint8_t* inputData, uint32_t length, + int32_t atOnce = 0) { + return parse(codec, inputData, length, atOnce); +} + +uint8_t shortSynStream[] = +{ 0x80, 0x02, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x04, // length must be >= 12 + 0x61, 0x62, 0x63, 0x64 +}; + +TEST(SPDYCodecTest, JunkSPDY) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY2); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = + parseSPDY(&codec, shortSynStream, sizeof(shortSynStream), -1); + EXPECT_EQ(unconsumed, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 0); +} + +uint8_t longNoop[] = +{ 0x80, 0x02, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00 +}; + +TEST(SPDYCodecTest, LongNoop) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY2); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = + parseSPDY(&codec, longNoop, sizeof(longNoop), -1); + EXPECT_EQ(unconsumed, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 0); +} + +uint8_t longPing[] = +{ 0x80, 0x02, 0x00, 0x06, + 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x00, + 0x00 +}; + +TEST(SPDYCodecTest, LongPing) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY2); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = + parseSPDY(&codec, longPing, sizeof(longPing), -1); + EXPECT_EQ(unconsumed, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 0); +} + +uint8_t badType[] = +{ 0x80, 0x02, 0x00, 0x0A, + 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x00, + 0x00 +}; + +TEST(SPDYCodecTest, BadType) { + // If an endpoint receives a control frame for a type it does not recognize, + // it must ignore the frame. + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY2); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = + parseSPDY(&codec, badType, sizeof(badType), -1); + EXPECT_EQ(unconsumed, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.streamErrors, 0); +} + +/** + * A request from firefox for facebook.com + */ +uint8_t synStream[] = +{ 0x80, 0x02, 0x00, 0x01, 0x01, 0x00, 0x02, 0x38, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x78, 0xbb, 0xdf, 0xa2, 0x51, 0xb2, + 0x74, 0x54, 0x5d, 0x6f, 0xda, 0x30, 0x14, 0xf5, + 0xd6, 0x8d, 0x7d, 0x68, 0xea, 0xfe, 0x82, 0x5f, + 0x2a, 0x6d, 0x1d, 0x0d, 0x89, 0x13, 0x3b, 0x4e, + 0xa3, 0x68, 0x0a, 0x60, 0xa3, 0x75, 0xd0, 0x0d, + 0xda, 0x42, 0xfb, 0x14, 0x85, 0x10, 0x0a, 0x85, + 0x24, 0x34, 0x5f, 0xcd, 0xfa, 0xeb, 0x77, 0x53, + 0x3a, 0xa6, 0xb5, 0xda, 0x83, 0xa3, 0xe8, 0xde, + 0x73, 0x6d, 0x1f, 0xdf, 0x7b, 0x0e, 0x7a, 0x87, + 0x1a, 0x11, 0xe8, 0x30, 0x99, 0xa1, 0xbd, 0x9e, + 0x38, 0x47, 0x8d, 0x0c, 0x5a, 0x17, 0x85, 0xe8, + 0xf5, 0x22, 0xcf, 0x37, 0x19, 0xda, 0xab, 0x31, + 0x2f, 0x5a, 0xe8, 0xcd, 0x63, 0x09, 0x7a, 0xfb, + 0x67, 0x27, 0xf4, 0xaa, 0x16, 0x17, 0xfa, 0x30, + 0x07, 0xbd, 0x4e, 0xe1, 0xf5, 0x94, 0x20, 0x89, + 0x50, 0x63, 0x3b, 0x7a, 0xe8, 0xeb, 0x8e, 0x6c, + 0xf3, 0x19, 0x87, 0x2f, 0xd5, 0xd3, 0x68, 0xb4, + 0xb6, 0x6f, 0x1d, 0x55, 0xb1, 0x9a, 0x87, 0xad, + 0xc3, 0x87, 0x3f, 0x8e, 0x3e, 0x3e, 0xd1, 0x37, + 0xda, 0x0f, 0xe3, 0xa3, 0x22, 0x6b, 0x86, 0xf1, + 0x03, 0x80, 0xa2, 0xc6, 0xb6, 0x63, 0x2f, 0xf7, + 0x61, 0xa8, 0x52, 0x67, 0x36, 0x15, 0xea, 0x79, + 0x99, 0x16, 0xf3, 0x8c, 0x26, 0xd1, 0x25, 0xbd, + 0x59, 0x8e, 0x16, 0xa9, 0x91, 0x9e, 0x47, 0x36, + 0x5e, 0x17, 0xce, 0xf5, 0x70, 0x7c, 0x31, 0x6d, + 0xf7, 0x46, 0xb9, 0x5f, 0x9c, 0x58, 0xc3, 0x6f, + 0x37, 0xbd, 0xa4, 0x7f, 0x25, 0x4a, 0xd7, 0xb5, + 0x71, 0xe0, 0xd5, 0x7a, 0x75, 0x18, 0xd7, 0x0c, + 0x93, 0x71, 0x6e, 0x40, 0x24, 0x8b, 0x1c, 0x62, + 0xe3, 0x2a, 0x73, 0xe8, 0x81, 0xee, 0x26, 0xbf, + 0x4a, 0x2f, 0xf6, 0xab, 0x76, 0x20, 0x82, 0x55, + 0xe2, 0x42, 0x80, 0xc0, 0xd2, 0x74, 0x5d, 0xa3, + 0x84, 0x9b, 0xdc, 0xb2, 0xf1, 0x3c, 0x75, 0xd4, + 0xfe, 0x70, 0x12, 0x67, 0xc3, 0xc1, 0xe9, 0x77, + 0x19, 0x8f, 0xaf, 0x57, 0xaa, 0xae, 0xb8, 0x93, + 0x71, 0xf7, 0x6c, 0x32, 0x4b, 0xa2, 0x7c, 0x9c, + 0xaf, 0x7e, 0x14, 0xc1, 0x26, 0x1c, 0xf1, 0x52, + 0x16, 0x83, 0xbc, 0xed, 0x2a, 0xed, 0x9f, 0xe4, + 0xaa, 0xbc, 0x53, 0x6e, 0x5c, 0x40, 0x5d, 0x2e, + 0xd4, 0x95, 0x28, 0x6d, 0x9c, 0x39, 0xae, 0xcf, + 0x4e, 0xce, 0x46, 0x13, 0xcb, 0x94, 0x62, 0x35, + 0xbc, 0x4f, 0x4a, 0x80, 0x99, 0x47, 0xb7, 0x47, + 0x36, 0xf6, 0x83, 0xdc, 0xd1, 0x74, 0xb0, 0x01, + 0x93, 0xe9, 0xdc, 0xb2, 0x88, 0x76, 0x40, 0x24, + 0x83, 0x3b, 0xa8, 0x36, 0xde, 0x38, 0xd4, 0x84, + 0x2f, 0x0c, 0x33, 0x48, 0x28, 0x74, 0xc4, 0x60, + 0x0b, 0x63, 0x06, 0x17, 0x35, 0x29, 0xe9, 0x92, + 0x1d, 0x2d, 0x97, 0x88, 0xba, 0xef, 0xa1, 0xec, + 0x66, 0x53, 0x22, 0xb7, 0xfb, 0x51, 0xc6, 0x2d, + 0x5d, 0x67, 0x22, 0x27, 0xd2, 0xa3, 0xd3, 0xee, + 0x12, 0xf0, 0x75, 0x99, 0xa7, 0xfb, 0x50, 0x04, + 0xbe, 0xa4, 0xea, 0x8c, 0x42, 0x5d, 0x2a, 0xb5, + 0x8e, 0x1b, 0xfc, 0x93, 0x06, 0xf3, 0x61, 0x0c, + 0xdc, 0xcc, 0x7c, 0x4c, 0x7b, 0x74, 0x26, 0xd6, + 0x11, 0xf9, 0x0f, 0xa2, 0x08, 0xf2, 0x67, 0x47, + 0xd6, 0x97, 0x5b, 0x27, 0xfe, 0x0c, 0xd2, 0xf9, + 0x9d, 0x34, 0x09, 0xd1, 0x75, 0x6e, 0x18, 0x4c, + 0xf8, 0xf9, 0x23, 0x92, 0x51, 0x58, 0x5c, 0xed, + 0xed, 0x28, 0xe9, 0x9a, 0xd1, 0x11, 0xc1, 0x42, + 0x76, 0x37, 0x1e, 0x9d, 0xef, 0x78, 0x49, 0xc6, + 0x49, 0xa7, 0x63, 0xe3, 0xbb, 0x19, 0x3c, 0x92, + 0xa5, 0x55, 0xcc, 0x62, 0xe8, 0xfd, 0x5f, 0x07, + 0x46, 0xa7, 0x83, 0xe4, 0x7e, 0xb9, 0x5e, 0xfb, + 0x2d, 0xaa, 0xa8, 0xf8, 0xd3, 0xa5, 0xa6, 0xd9, + 0xf8, 0x62, 0x5a, 0xc4, 0x79, 0x61, 0xe3, 0xfe, + 0x32, 0x2e, 0x2a, 0x5c, 0x71, 0xe6, 0x31, 0xe8, + 0x7a, 0x5a, 0x1e, 0x6b, 0x86, 0xa2, 0x7e, 0xc6, + 0xbd, 0x10, 0x1a, 0xdd, 0x02, 0xc7, 0x7e, 0x30, + 0x6f, 0x2c, 0xc1, 0x2b, 0xe6, 0x49, 0xd5, 0xaa, + 0x93, 0x8a, 0xf6, 0x1b, 0x00, 0x00, 0xff, 0xff +}; + +TEST(SPDYCodecTest, SynStreamBoundaries) { + for (int i = -1; i < int(sizeof(synStream)); i++) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY2); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = + parseSPDY(&codec, synStream, sizeof(synStream), i); + EXPECT_EQ(unconsumed, 0); + EXPECT_EQ(callbacks.messageBegin, 1); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.messageComplete, 1); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + } +} + +TEST(SPDYCodecTest, SetSettings) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY3); + HTTPSettings* settings = codec.getEgressSettings(); + // There are 2 settings by default. Turn on another setting + settings->setSetting(SettingsId::_SPDY_DOWNLOAD_BANDWIDTH, 10); + // This should no-op since this setting should be off by default + settings->unsetSetting(SettingsId::_SPDY_ROUND_TRIP_TIME); + EXPECT_EQ(settings->getSetting(SettingsId::_SPDY_ROUND_TRIP_TIME), nullptr); + EXPECT_EQ(settings->getSetting(SettingsId::_SPDY_CURRENT_CWND), nullptr); + EXPECT_EQ(settings->getSetting(SettingsId::MAX_CONCURRENT_STREAMS)->value, + spdy::kMaxConcurrentStreams); + EXPECT_EQ(settings->getSetting(SettingsId::INITIAL_WINDOW_SIZE)->value, + spdy::kInitialWindow); + EXPECT_EQ( + settings->getSetting(SettingsId::_SPDY_DOWNLOAD_BANDWIDTH)->value, 10); + // Turn off one of the defaults + settings->unsetSetting(SettingsId::MAX_CONCURRENT_STREAMS); + // Change the value of an existing default setting + settings->setSetting(SettingsId::INITIAL_WINDOW_SIZE, 123); + EXPECT_EQ(settings->getSetting(SettingsId::MAX_CONCURRENT_STREAMS), + nullptr); + EXPECT_EQ(settings->getSetting(SettingsId::INITIAL_WINDOW_SIZE)->value, + 123); + EXPECT_EQ(settings->getNumSettings(), 2); + // Change the value of a unset setting + settings->setSetting(SettingsId::MAX_CONCURRENT_STREAMS, 400); + EXPECT_EQ(settings->getSetting(SettingsId::MAX_CONCURRENT_STREAMS)->value, + 400); + EXPECT_EQ(settings->getNumSettings(), 3); +} + +TEST(SPDYCodecTest, FrameTooLarge) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY2); + codec.setMaxFrameLength(500); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = parseSPDY(&codec, synStream, sizeof(synStream), -1); + EXPECT_EQ(unconsumed, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 1); +} + +TEST(SPDYCodecTest, FrameUncompressedTooLarge) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY2); + codec.setMaxUncompressedHeaders(600); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = parseSPDY(&codec, synStream, sizeof(synStream), -1); + EXPECT_EQ(unconsumed, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 1); +} + +uint8_t spdy3UnknownCtlFrame[] = +{ 0x80, 0x03, 0x00, 0x0B, // ctl frame for spdy/3 (11 is not defined) + 0x00, 0x00, 0x00, 0x02, // len = 2 + 0xD4, 0x74 // The data +}; + +TEST(SPDYCodecTest, UnsupportedVersion) { + // Send a spdy/2 frame on a spdy/3 codec + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY3); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = parseSPDY(&codec, shortSynStream, + sizeof(shortSynStream), -1); + EXPECT_EQ(unconsumed, 0); + // Expect a GOAWAY with PROTOCOL_ERROR (because of unsupported version) + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.lastParseError->getCodecStatusCode(), + spdy::rstToErrorCode(spdy::RST_PROTOCOL_ERROR)); +} + +TEST(SPDYCodecTest, UnsupportedVersion2) { + // Send a spdy/3 frame on a spdy/2 codec + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY2); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = parseSPDY(&codec, spdy3UnknownCtlFrame, + sizeof(spdy3UnknownCtlFrame), -1); + EXPECT_EQ(unconsumed, 0); + // Expect a GOAWAY with PROTOCOL_ERROR (because of unsupported version) + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.lastParseError->getCodecStatusCode(), + spdy::rstToErrorCode(spdy::RST_PROTOCOL_ERROR)); +} + +TEST(SPDYCodecTest, UnsupportedFrameType) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY3); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + size_t unconsumed = parseSPDY(&codec, spdy3UnknownCtlFrame, + sizeof(spdy3UnknownCtlFrame), -1); + EXPECT_EQ(unconsumed, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + // Spec says unknown control frames must be ignored +} + +template +unique_ptr getSynStream(Codec& egressCodec, + uint32_t streamID, + const HTTPMessage& msg, + uint32_t assocStreamId = 0, + HTTPHeaderSize* size = nullptr) { + folly::IOBufQueue output(folly::IOBufQueue::cacheChainLength()); + egressCodec.generateHeader(output, streamID, msg, assocStreamId, size); + return output.move(); +} + +template +unique_ptr getSynStream(Codec& egressCodec, + uint32_t streamID) { + HTTPMessage msg; + msg.setMethod("GET"); + msg.getHeaders().set("HOST", "www.foo.com"); + msg.setURL("https://www.foo.com"); + return getSynStream(egressCodec, streamID, msg); +} + +// Runs the function with all combinations of spdy/2 and spdy/3 +template +void callFunction(F f, V version) { + Codec ingressCodec(TransportDirection::DOWNSTREAM, version); + Codec egressCodec(TransportDirection::UPSTREAM, version); + f(ingressCodec, egressCodec); +} + +#define copyForMe(F, C, V) callFunction(F, \ + SPDYVersion::V) + +#define permuteTest(f) copyForMe(f, SPDYCodec, SPDY2); \ + copyForMe(f, SPDYCodec, SPDY3); + +/** + * Returns a SPDY frame with the specified version + */ +unique_ptr getVersionedSpdyFrame(const uint8_t* bytes, + size_t len, + uint8_t version) { + auto frame = folly::IOBuf::copyBuffer(bytes, len); + uint8_t* data = frame->writableData(); + data[1] = version; /* Set the version */ + return std::move(frame); +} + +template +void maxTransactionHelper(Codec1& ingressCodec, Codec2& egressCodec, + uint32_t parallel) { + uint16_t expectedFailures = 0; + uint16_t synCount = 101; + ingressCodec.getEgressSettings()->setSetting( + SettingsId::MAX_CONCURRENT_STREAMS, parallel); + FakeHTTPCodecCallback callbacks; + ingressCodec.setCallback(&callbacks); + for (uint16_t i = 0; i < synCount; ++i) { + if (i >= parallel) { + ++expectedFailures; + } + auto toParse = getSynStream(egressCodec, 2*i + 1); + ingressCodec.onIngress(*toParse); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.streamErrors, expectedFailures); + } +} + +template +void doDefaultMaxTransactionTest(Codec1& ingressCodec, Codec2& egressCodec) { + maxTransactionHelper(ingressCodec, egressCodec, 50); +} + +template +void doNonDefaultMaxTransactionTest(Codec1& ingressCodec, Codec2& egressCodec) { + maxTransactionHelper(ingressCodec, egressCodec, 100); +} + +TEST(SPDYCodecTest, DefaultMaxTransactions) { + permuteTest(doDefaultMaxTransactionTest); +} + +TEST(SPDYCodecTest, NonDefaultMaxTransactions) { + permuteTest(doNonDefaultMaxTransactionTest); +} + +template +void doEmptyHeaderValueTest(Codec1& ingressCodec, Codec2& egressCodec) { + uint8_t version = ingressCodec.getVersion(); + bool emptyAllowed = version != 2; + FakeHTTPCodecCallback callbacks; + ingressCodec.setCallback(&callbacks); + HTTPMessage toSend; + toSend.setMethod("GET"); + toSend.setURL("http://www.foo.com"); + auto& headers = toSend.getHeaders(); + headers.set("Host", "www.foo.com"); + headers.set("Pragma", ""); + headers.set("X-Test1", "yup"); + HTTPHeaderSize size; + auto toParse = getSynStream(egressCodec, 1, toSend, 0, &size); + ingressCodec.onIngress(*toParse); + + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + ASSERT_NE(callbacks.msg.get(), nullptr); + const auto& parsed = callbacks.msg->getHeaders(); + EXPECT_EQ(parsed.exists("Pragma"), emptyAllowed); + EXPECT_EQ(parsed.exists("pragma"), emptyAllowed); + EXPECT_EQ(parsed.getSingleOrEmpty("X-Test1"), "yup"); + // All codecs add the accept-encoding header + EXPECT_EQ(parsed.exists("accept-encoding"), true); + // SPDY/2 subtracts the Host header, but it should infer it from the host:port + // portion of the requested url and present it in the headers + EXPECT_EQ(parsed.exists("host"), true); + EXPECT_EQ(callbacks.msg->getURL(), "http://www.foo.com"); + EXPECT_EQ(parsed.size(), emptyAllowed ? 4 : 3); + EXPECT_TRUE(size.uncompressed > 0); + EXPECT_TRUE(size.compressed > 0); +} + +TEST(SPDYCodecTest, EmptyHeaderValue) { + permuteTest(doEmptyHeaderValueTest); +} + +/** + * Tests a syn stream request containing the wrong version. This frame is + * generated by using the v2 synStream firefox frame specified above and + * changing the version field to 3. The dictionary used for the zlib compression + * will be different and the session will be rejected. + */ +TEST(SPDYCodecTest, SynStreamWrongVersion) { + SPDYCodec codec(TransportDirection::DOWNSTREAM, SPDYVersion::SPDY3); + FakeHTTPCodecCallback callbacks; + codec.setCallback(&callbacks); + auto frame = getVersionedSpdyFrame(synStream, sizeof(synStream), 3); + size_t unconsumed = parseSPDY(&codec, frame->data(), frame->length(), -1); + EXPECT_EQ(unconsumed, 0); + // Since the session compression state is inconsistent we need to send a + // session error. Expect a GOAWAY with PROTOCOL_ERROR. + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.lastParseError->getCodecStatusCode(), + spdy::rstToErrorCode(spdy::RST_PROTOCOL_ERROR)); +} + +/** + * A syn reply frame with invalid length + */ +uint8_t shortSynReply[] = +{ 0x80, 0x02, 0x00, 0x02, 0x01, 0x00, 0x00, 0x04, // length set to 4 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +template +void doShortSynReplyTest(Codec1& ingressCodec, Codec2& egressCodec) { + FakeHTTPCodecCallback callbacks; + egressCodec.setCallback(&callbacks); + auto frame = getVersionedSpdyFrame(shortSynReply, sizeof(shortSynReply), + egressCodec.getVersion()); + egressCodec.onIngress(*frame); + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.streamErrors, 0); +} + +TEST(SPDYCodecTest, ShortSynReply) { + permuteTest(doShortSynReplyTest); +} + +TEST(SPDYCodecTest, SupportsSessionFlowControl) { + SPDYCodec spdy2(TransportDirection::UPSTREAM, + SPDYVersion::SPDY2); + SPDYCodec spdy3(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + SPDYCodec spdy3_1(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3_1); + EXPECT_FALSE(spdy2.supportsSessionFlowControl()); + EXPECT_FALSE(spdy3.supportsSessionFlowControl()); + EXPECT_TRUE(spdy3_1.supportsSessionFlowControl()); +} + +// Test serializing and deserializing a header that has many values +TEST(SPDYCodecTest, HeaderWithManyValues) { + const std::string kMultiValued = "X-Multi-Valued"; + const unsigned kNumValues = 1000; + + FakeHTTPCodecCallback callbacks; + SPDYCodec egressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + HTTPMessage req; + req.setMethod("GET"); + req.getHeaders().set("HOST", "www.foo.com"); + req.setURL("https://www.foo.com"); + for (unsigned i = 0; i < kNumValues; ++i) { + req.getHeaders().add(kMultiValued, folly::to("Value", i)); + } + auto syn = getSynStream(egressCodec, 1, req); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 1); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + CHECK_NOTNULL(callbacks.msg.get()); + EXPECT_EQ(callbacks.msg->getHeaders().getNumberOfValues(kMultiValued), + kNumValues); +} + +TEST(SPDYCodecTest, LargeFrameEncoding) { + const std::string kMultiValued = "X-Multi-Valued"; + const unsigned kNumValues = 1000; + + SPDYCodec codec(TransportDirection::UPSTREAM, SPDYVersion::SPDY3); + // This will simulate the condition where we have a very small + // compresed headers size compared to the number of headers we + // have. + codec.setMaxUncompressedHeaders(0); + auto req = getGetRequest(); + for (unsigned i = 0; i < kNumValues; ++i) { + req.getHeaders().add(kMultiValued, folly::to("Value", i)); + } + auto syn = getSynStream(codec, 1, req); +} + +// Test serializing and deserializing a header that has many values +TEST(SPDYCodecTest, InvalidSettings) { + FakeHTTPCodecCallback callbacks; + + SPDYCodec egressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + folly::IOBufQueue output(folly::IOBufQueue::cacheChainLength()); + egressCodec.getEgressSettings()->setSetting( + SettingsId::INITIAL_WINDOW_SIZE, + (uint32_t)std::numeric_limits::max() + 1); + egressCodec.generateSettings(output); + auto ingress = output.move(); + ingressCodec.onIngress(*ingress); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); +} + +TEST(SPDYCodecTest, ServerPush) { + FakeHTTPCodecCallback callbacks; + SPDYCodec egressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + HTTPMessage push; + push.getHeaders().set("HOST", "www.foo.com"); + push.setURL("https://www.foo.com/"); + auto syn = getSynStream(egressCodec, 2, push, 1); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 1); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.assocStreamId, 1); +} + +/** + * A push stream with Host header missing + */ +uint8_t pushStreamWithHostMissing[] = +{ 0x80, 0x3, 0x0, 0x1, 0x2, 0x0, 0x0, 0x7c, + 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x1, + 0x0, 0x0, 0x38, 0x30, 0xe3, 0xc6, 0xa7, 0xc2, + 0x0, 0x62, 0x0, 0x9d, 0xff, 0x0, 0x0, 0x0, + 0x4, 0x0, 0x0, 0x0, 0x5, 0x3a, 0x70, 0x61, + 0x74, 0x68, 0x0, 0x0, 0x0, 0x14, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, + 0x77, 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x0, 0x0, 0x0, 0x7, 0x3a, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x65, 0x0, 0x0, 0x0, + 0x4, 0x68, 0x74, 0x74, 0x70, 0x0, 0x0, 0x0, + 0x7, 0x3a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x0, 0x0, 0x0, 0x3, 0x32, 0x30, 0x30, 0x0, + 0x0, 0x0, 0x8, 0x3a, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x0, 0x0, 0x0, 0x8, 0x48, + 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31, 0x0, + 0x0, 0x0, 0xff, 0xff +}; + +TEST(SPDYCodecTest, ServerPushHostMissing) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + auto syn = folly::IOBuf::copyBuffer(pushStreamWithHostMissing, + sizeof(pushStreamWithHostMissing)); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 1); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.assocStreamId, 0); +} + +/** + * A push stream with an odd StreamID + */ +uint8_t pushStreamWithOddId[] = +{ 0x80, 0x03, 0x00, 0x01, 0x02, 0x00, 0x00, 0x91, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x38, 0x30, 0xe3, 0xc6, 0xa7, 0xc2, + 0x00, 0x77, 0x00, 0x88, 0xff, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x05, 0x3a, 0x68, 0x6f, + 0x73, 0x74, 0x00, 0x00, 0x00, 0x0b, 0x77, 0x77, + 0x77, 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, + 0x6d, 0x00, 0x00, 0x00, 0x07, 0x3a, 0x6d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x05, 0x3a, 0x70, 0x61, 0x74, + 0x68, 0x00, 0x00, 0x00, 0x14, 0x68, 0x74, 0x74, + 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, + 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x00, 0x00, 0x00, 0x07, 0x3a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x65, 0x00, 0x00, 0x00, 0x04, + 0x68, 0x74, 0x74, 0x70, 0x00, 0x00, 0x00, 0x08, + 0x3a, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x00, 0x00, 0x00, 0x08, 0x48, 0x54, 0x54, 0x50, + 0x2f, 0x31, 0x2e, 0x31, 0x00, 0x00, 0x00, 0xff, + 0xff +}; + +TEST(SPDYCodecTest, ServerPushInvalidId) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + auto syn = folly::IOBuf::copyBuffer(pushStreamWithOddId, + sizeof(pushStreamWithOddId)); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.assocStreamId, 0); +} + +/** + * A push stream without unidirectional flag + */ +uint8_t pushStreamWithoutUnidirectional[] = +{ 0x80, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x91, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x38, 0x30, 0xe3, 0xc6, 0xa7, 0xc2, + 0x00, 0x77, 0x00, 0x88, 0xff, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x05, 0x3a, 0x68, 0x6f, + 0x73, 0x74, 0x00, 0x00, 0x00, 0x0b, 0x77, 0x77, + 0x77, 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, + 0x6d, 0x00, 0x00, 0x00, 0x07, 0x3a, 0x6d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x05, 0x3a, 0x70, 0x61, 0x74, + 0x68, 0x00, 0x00, 0x00, 0x14, 0x68, 0x74, 0x74, + 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, + 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x00, 0x00, 0x00, 0x07, 0x3a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x65, 0x00, 0x00, 0x00, 0x04, + 0x68, 0x74, 0x74, 0x70, 0x00, 0x00, 0x00, 0x08, + 0x3a, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x00, 0x00, 0x00, 0x08, 0x48, 0x54, 0x54, 0x50, + 0x2f, 0x31, 0x2e, 0x31, 0x00, 0x00, 0x00, 0xff, + 0xff +}; + +TEST(SPDYCodecTest, ServerPushInvalidFlags) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + auto syn = folly::IOBuf::copyBuffer(pushStreamWithoutUnidirectional, + sizeof(pushStreamWithoutUnidirectional)); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 1); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.assocStreamId, 0); +} + +/** + * A push stream with assocStreamID = 0 + */ +uint8_t pushStreamWithoutAssoc[] = +{ 0x80, 0x03, 0x00, 0x01, 0x02, 0x00, 0x00, 0x91, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x38, 0x30, 0xe3, 0xc6, 0xa7, 0xc2, + 0x00, 0x77, 0x00, 0x88, 0xff, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x05, 0x3a, 0x68, 0x6f, + 0x73, 0x74, 0x00, 0x00, 0x00, 0x0b, 0x77, 0x77, + 0x77, 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, + 0x6d, 0x00, 0x00, 0x00, 0x07, 0x3a, 0x6d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x05, 0x3a, 0x70, 0x61, 0x74, + 0x68, 0x00, 0x00, 0x00, 0x14, 0x68, 0x74, 0x74, + 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, + 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x00, 0x00, 0x00, 0x07, 0x3a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x65, 0x00, 0x00, 0x00, 0x04, + 0x68, 0x74, 0x74, 0x70, 0x00, 0x00, 0x00, 0x08, + 0x3a, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x00, 0x00, 0x00, 0x08, 0x48, 0x54, 0x54, 0x50, + 0x2f, 0x31, 0x2e, 0x31, 0x00, 0x00, 0x00, 0xff, + 0xff +}; + +TEST(SPDYCodecTest, ServerPushWithoutAssoc) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + auto syn = folly::IOBuf::copyBuffer(pushStreamWithoutAssoc, + sizeof(pushStreamWithoutAssoc)); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); + EXPECT_EQ(callbacks.assocStreamId, 0); +} + +TEST(SPDYCodecTest, StatusReason) { + FakeHTTPCodecCallback callbacks; + SPDYCodec egressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + HTTPMessage resp; + resp.setStatusCode(200); + auto syn = getSynStream(egressCodec, 1, resp, 0); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 1); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.msg->getStatusCode(), 200); + EXPECT_EQ(callbacks.msg->getStatusMessage(), ""); + callbacks.reset(); + + resp.setStatusCode(200); + resp.setStatusMessage("Awesome"); + syn = getSynStream(egressCodec, 1, resp, 0); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 1); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.msg->getStatusCode(), 200); + EXPECT_EQ(callbacks.msg->getStatusMessage(), "Awesome"); + callbacks.reset(); + + // Out of range + resp.setStatusCode(2000); + resp.setStatusMessage("10x OK"); + syn = getSynStream(egressCodec, 1, resp, 0); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 1); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_TRUE(callbacks.lastParseError->hasCodecStatusCode()); + EXPECT_EQ(callbacks.lastParseError->getCodecStatusCode(), + spdy::rstToErrorCode(spdy::RST_PROTOCOL_ERROR)); + callbacks.reset(); + + resp.setStatusCode(64); + resp.setStatusMessage("Ought to be enough for anybody"); + syn = getSynStream(egressCodec, 1, resp, 0); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 1); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_TRUE(callbacks.lastParseError->hasCodecStatusCode()); + EXPECT_EQ(callbacks.lastParseError->getCodecStatusCode(), + spdy::rstToErrorCode(spdy::RST_PROTOCOL_ERROR)); + callbacks.reset(); +} + +TEST(SPDYCodecTest, UpstreamPing) { + folly::IOBufQueue buf(folly::IOBufQueue::cacheChainLength()); + FakeHTTPCodecCallback callbacks; + SPDYCodec egressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + // Send a reply with no corresponding ping request + egressCodec.generatePingReply(buf, 2); + auto pingReply = buf.move(); + ingressCodec.onIngress(*pingReply); + ASSERT_EQ(callbacks.recvPingReply, 0); // should be ignored + + auto lastRequest = callbacks.recvPingRequest; + for (unsigned i = 0; i < 10; ++i) { + egressCodec.generatePingRequest(buf); + auto pingReq = buf.move(); + ingressCodec.onIngress(*pingReq); + ASSERT_GT(callbacks.recvPingRequest, lastRequest); + ASSERT_EQ(callbacks.recvPingRequest % 2, 1); + lastRequest = callbacks.recvPingRequest; + } +} + +TEST(SPDYCodecTest, DownstreamPing) { + folly::IOBufQueue buf(folly::IOBufQueue::cacheChainLength()); + FakeHTTPCodecCallback callbacks; + SPDYCodec egressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + // Send a reply with no corresponding ping request + egressCodec.generatePingReply(buf, 1); + auto pingReply = buf.move(); + ingressCodec.onIngress(*pingReply); + ASSERT_EQ(callbacks.recvPingReply, 0); // should be ignored + + auto lastRequest = callbacks.recvPingRequest; + for (unsigned i = 0; i < 10; ++i) { + egressCodec.generatePingRequest(buf); + auto pingReq = buf.move(); + ingressCodec.onIngress(*pingReq); + ASSERT_GT(callbacks.recvPingRequest, lastRequest); + ASSERT_EQ(callbacks.recvPingRequest % 2, 0); + lastRequest = callbacks.recvPingRequest; + } +} + +TEST(SPDYCodecTest, DateHeader) { + FakeHTTPCodecCallback callbacks; + SPDYCodec egressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + HTTPMessage resp; + resp.setStatusCode(200); + auto syn = getSynStream(egressCodec, 1, resp, 0); + ingressCodec.onIngress(*syn); + EXPECT_EQ(callbacks.messageBegin, 1); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_TRUE(callbacks.msg->getHeaders().exists(HTTP_HEADER_DATE)); +} + +// SYN_STREAM includes ~100k header name with 50k one-byte values +uint8_t multiValuedHeaderAttack[] = +{ 0x80, 0x03, 0x00, 0x01, 0x01, 0x00, 0x02, 0x11, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x60, 0x00, 0x38, 0xea, 0xe3, 0xc6, 0xa7, 0xc2, + 0xec, 0xd4, 0x31, 0x0e, 0x82, 0x40, 0x10, 0x46, + 0x61, 0x8c, 0x85, 0x1a, 0x43, 0x6f, 0x69, 0x4d, + 0x31, 0x24, 0xda, 0x6d, 0x4f, 0xb8, 0x80, 0x17, + 0x00, 0x8d, 0xa1, 0xd0, 0x30, 0xc9, 0x6e, 0x01, + 0x57, 0xb7, 0x72, 0x18, 0x49, 0x28, 0xe4, 0x08, + 0xef, 0xab, 0x61, 0x93, 0x4d, 0xfe, 0x7d, 0xb6, + 0xa3, 0xc3, 0xf4, 0x9c, 0xc2, 0x1c, 0x84, 0x93, + 0x2d, 0x5a, 0xfa, 0x94, 0x7a, 0x89, 0xad, 0x3c, + 0x2d, 0xbd, 0xad, 0x8d, 0x4f, 0xee, 0x1e, 0x8d, + 0x5d, 0x58, 0xa6, 0x5d, 0x57, 0x37, 0xff, 0x4d, + 0x1b, 0x0f, 0xd8, 0xb1, 0xfc, 0x8d, 0x5c, 0xb4, + 0x53, 0xff, 0x32, 0x5a, 0x38, 0xdf, 0x5e, 0xd7, + 0x2e, 0x25, 0x9d, 0xc6, 0xbf, 0x0f, 0xeb, 0x9b, + 0x5f, 0xc2, 0xbe, 0x2d, 0xca, 0x62, 0xbd, 0xe6, + 0xb9, 0x5f, 0xf2, 0x3c, 0xdf, 0xf2, 0xef, 0x81, + 0xe6, 0x51, 0x1f, 0xe3, 0xab, 0x19, 0xed, 0xc4, + 0x8b, 0x5c, 0xb3, 0xcd, 0x27, 0x1b, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xb2, 0x07, 0x07, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x40, 0xfe, 0xaf, 0x8d, 0xa0, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xc2, 0x1e, 0x1c, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf9, 0xbf, 0x36, + 0x82, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0x0a, 0x7b, 0x70, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe4, 0xff, 0xda, 0x08, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0x2a, 0xec, 0xc1, 0x41, + 0x01, 0x00, 0x00, 0x04, 0x04, 0xb0, 0x53, 0x47, + 0x3f, 0xe5, 0xbd, 0x14, 0x10, 0x61, 0x1b, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf0, 0x48, 0x6d, 0x7a, 0xa2, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0x7a, 0xec, 0xc2, 0x01, 0x09, 0x00, 0x00, 0x00, + 0x00, 0x90, 0xff, 0xaf, 0x1d, 0x11, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x15, 0x76, 0xe1, + 0x80, 0x04, 0x00, 0x00, 0x00, 0x00, 0xc8, 0xff, + 0xd7, 0x8e, 0x88, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0x0a, 0xbb, 0x70, 0x40, 0x02, 0x00, + 0x00, 0x00, 0x00, 0xe4, 0xff, 0x6b, 0x47, 0x44, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x85, + 0x5d, 0x38, 0xa6, 0x01, 0x00, 0x00, 0x60, 0x18, + 0xe4, 0x67, 0xfe, 0x05, 0xf6, 0x9f, 0x06, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xbc, 0x05, 0x00, 0x00, 0xff, + 0xff +}; + +// Ensure the codec can handle a malicious packet intended to exhaust server +// resources +TEST(SPDYCodecTest, HeaderDoS) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + auto attack = folly::IOBuf::copyBuffer(multiValuedHeaderAttack, + sizeof(multiValuedHeaderAttack)); + + ingressCodec.onIngress(*attack); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.messageComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 1); + EXPECT_EQ(callbacks.sessionErrors, 1); +} + +// Make sure SPDYCodec behaves correctly when we generate and receive double +// GOAWAYs. +TEST(SPDYCodecTest, DoubleGoawayServer) { + FakeHTTPCodecCallback callbacks; + SPDYCodec egressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + + egressCodec.enableDoubleGoawayDrain(); + ingressCodec.setCallback(&callbacks); + egressCodec.setCallback(&callbacks); + + unsigned ack = std::numeric_limits::max(); + auto f = [&] () { + folly::IOBufQueue output(folly::IOBufQueue::cacheChainLength()); + egressCodec.generateGoaway(output, ack, ErrorCode::NO_ERROR); + auto ingress = output.move(); + ingressCodec.onIngress(*ingress); + ack -= 2; + }; + + EXPECT_TRUE(egressCodec.isReusable()); + EXPECT_FALSE(egressCodec.isWaitingToDrain()); + EXPECT_TRUE(ingressCodec.isReusable()); + f(); + // server spdy codec remains reusable after the first goaway + EXPECT_TRUE(egressCodec.isReusable()); + EXPECT_TRUE(egressCodec.isWaitingToDrain()); + f(); + EXPECT_FALSE(egressCodec.isReusable()); + EXPECT_FALSE(egressCodec.isWaitingToDrain()); + + EXPECT_EQ(2, callbacks.goaways); +} + +TEST(SPDYCodecTest, DoubleGoawayClient) { + FakeHTTPCodecCallback callbacks; + SPDYCodec egressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + + egressCodec.enableDoubleGoawayDrain(); + ingressCodec.setCallback(&callbacks); + egressCodec.setCallback(&callbacks); + + unsigned ack = std::numeric_limits::max(); + auto f = [&] () { + folly::IOBufQueue output(folly::IOBufQueue::cacheChainLength()); + egressCodec.generateGoaway(output, ack, ErrorCode::NO_ERROR); + auto ingress = output.move(); + ingressCodec.onIngress(*ingress); + ack -= 2; + }; + + EXPECT_TRUE(egressCodec.isReusable()); + EXPECT_FALSE(egressCodec.isWaitingToDrain()); + f(); + // client spdy codec not reusable after the first goaway + EXPECT_FALSE(egressCodec.isReusable()); + EXPECT_TRUE(egressCodec.isWaitingToDrain()); + EXPECT_FALSE(ingressCodec.isReusable()); + f(); + EXPECT_FALSE(egressCodec.isReusable()); + EXPECT_FALSE(egressCodec.isWaitingToDrain()); + EXPECT_FALSE(ingressCodec.isReusable()); + + EXPECT_EQ(2, callbacks.goaways); +} + +// Two packets: +// - first one has an invalid header nanme +// - second one has an empty header block +uint8_t invalidHeaderPlusEmptyBlock[] = +{ 0x80, 0x03, 0x00, 0x02, 0xc4, 0x00, 0x00, 0x14, + 0xf7, 0x76, 0x2d, 0x37, 0x78, 0x9c, 0x93, 0x60, + 0x00, 0x03, 0x75, 0x06, 0xbd, 0x76, 0x21, 0xb2, + 0xd0, 0xd9, 0x54, 0x91, 0x80, 0x03, 0x00, 0x01, + 0x4e, 0x00, 0x00, 0x0a, 0xbe, 0x14, 0x31, 0x55, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00 +}; + +// Try to trick the codec into leaving the HeaderList in an inconsistent state +// (a header name without a corresponding value), and parsing this inconsistent +// HeaderList +TEST(SPDYCodecTest, OddHeaderListTest) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + ingressCodec.setCallback(&callbacks); + + auto attack = folly::IOBuf::copyBuffer(invalidHeaderPlusEmptyBlock, + sizeof(invalidHeaderPlusEmptyBlock)); + + ingressCodec.onIngress(*attack); + EXPECT_EQ(callbacks.messageBegin, 1); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.streamErrors, 1); +} + +// Send a RST_STREAM for stream=1 while parsing a SYN_STREAM+FIN for stream=3. +// Stream 3 should be unaffected +TEST(SPDYCodecTest, SendRstParsingFrame) { + NiceMock callbacks; + SPDYCodec egressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3_1); + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3_1); + folly::IOBufQueue egressCodecQueue(folly::IOBufQueue::cacheChainLength()); + folly::IOBufQueue ingressCodecQueue(folly::IOBufQueue::cacheChainLength()); + + InSequence enforceOrder; + + EXPECT_CALL(callbacks, onHeadersComplete(3, _)) + .WillOnce(InvokeWithoutArgs([&] { + ingressCodec.generateRstStream(ingressCodecQueue, + 1, ErrorCode::CANCEL); + })); + EXPECT_CALL(callbacks, onMessageComplete(3, false)); + + ingressCodec.setCallback(&callbacks); + auto syn = getSynStream(egressCodec, 3); + egressCodecQueue.append(std::move(syn)); + egressCodec.generateEOM(egressCodecQueue, 3); + auto ingress = egressCodecQueue.move(); + ingress->coalesce(); + ingressCodec.onIngress(*ingress); +} + +uint8_t invalidNumNameValuesBlock[] = { + 0x80, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x91, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x38, 0x30, 0xe3, 0xc6, 0xa7, 0xc2, + 0x00, 0x77, 0x00, 0x88, 0xff, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x05, 0x3a, 0x68, 0x6f, + 0x73, 0x74, 0x00, 0x00, 0x00, 0x0b, 0x77, 0x77, + 0x77, 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, + 0x6d, 0x00, 0x00, 0x00, 0x77, 0x3a, 0x6d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x05, 0x3a, 0x70, 0x61, 0x74, + 0x68, 0x00, 0x13, 0x00, 0x14, 0x68, 0x74, 0x74, + 0x39, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, + 0x2e, 0x66, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x00, 0x00, 0x00, 0x07, 0x3a, 0x73, 0x63, + 0x68, 0x65, 0x6d, 0x65, 0x00, 0x00, 0x00, 0x04, + 0x68, 0x74, 0x74, 0x70, 0x00, 0x00, 0x00, 0x08, + 0x3a, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x00, 0x00, 0x00, 0x08, 0x48, 0x54, 0x54, 0x50, + 0x2f, 0x31, 0x2e, 0x31, 0x00, 0x00, 0x00, 0xff, + 0xff +}; +// Ensure we generate a session-level error if the number of name values +// exceeds the size of our current buffer. +TEST(SPDYCodecTest, BadNumNameValues) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3_1); + ingressCodec.setCallback(&callbacks); + + auto attack = folly::IOBuf::copyBuffer(invalidNumNameValuesBlock, + sizeof(invalidNumNameValuesBlock)); + + ingressCodec.onIngress(*attack); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); +} + +const uint8_t kShortSettings[] = { + 0x80, 0x03, 0x00, 0x04, 0xee, 0x00, 0x00, 0x01, 0x00, 0x00 +}; +// Make sure we reject SETTINGS frames that are too short with GOAWAY +TEST(SPDYCodecTest, ShortSettings) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3_1); + ingressCodec.setCallback(&callbacks); + + auto attack = folly::IOBuf::copyBuffer(kShortSettings, + sizeof(kShortSettings)); + + ingressCodec.onIngress(*attack); + EXPECT_EQ(callbacks.messageBegin, 0); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 1); +} + +TEST(SPDYCodecTest, SegmentedHeaderBlock) { + SPDYCodec egressCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3_1_HPACK); + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3_1_HPACK); + // generate huge string to use as a header value + string huge; + uint32_t size = 20000; + char ch = 'a'; + for (uint32_t i = 0; i < size; i++) { + huge.push_back(ch); + if (ch == 'z') { + ch = 'a'; + } else { + ch++; + } + } + HTTPMessage req; + req.setMethod("GET"); + req.setURL("http://www.facebook.com"); + auto& reqHeaders = req.getHeaders(); + reqHeaders.set("HOST", "www.facebook.com"); + // setting this huge header value will cause allocation of a separate IOBuf + reqHeaders.set("X-FB-Huge", huge); + auto buf = getSynStream(egressCodec, 1, req); + FakeHTTPCodecCallback callbacks; + ingressCodec.setCallback(&callbacks); + ingressCodec.onIngress(*buf); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.msg->getHeaders().getSingleOrEmpty("x-fb-huge").size(), + size); + + // do it for responses + HTTPMessage resp; + resp.setStatusCode(200); + resp.setStatusMessage("OK"); + auto& respHeaders = resp.getHeaders(); + respHeaders.set("X-FB-Huge", huge); + auto buf2 = getSynStream(ingressCodec, 1, resp); + callbacks.reset(); + egressCodec.setCallback(&callbacks); + egressCodec.onIngress(*buf2); + EXPECT_EQ(callbacks.streamErrors, 0); + EXPECT_EQ(callbacks.sessionErrors, 0); + EXPECT_EQ(callbacks.headersComplete, 1); + EXPECT_EQ(callbacks.msg->getHeaders().getSingleOrEmpty("x-fb-huge").size(), + size); +} + +const uint8_t kColonHeaders[] = { + 0x80, 0x03, 0x00, 0x01, 0x48, 0x00, 0x00, 0x1a, 0xf6, 0xf6, 0x1a, 0xb5, + 0x00, 0x00, 0x00, 0x00, 0x17, 0x28, 0x28, 0x53, 0x62, 0x60, 0x60, 0x10, + 0x60, 0x60, 0x60, 0x60, 0xb4, 0x1a, 0xbc, 0x84, 0xa4, 0xa4 +}; + +// Test a case where we use a single colon as a header name +TEST(SPDYCodecTest, ColonHeaders) { + FakeHTTPCodecCallback callbacks; + SPDYCodec ingressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3_1); + ingressCodec.setCallback(&callbacks); + + auto testBuf = folly::IOBuf::copyBuffer(kColonHeaders, + sizeof(kColonHeaders)); + + ingressCodec.onIngress(*testBuf); + EXPECT_EQ(callbacks.headersComplete, 0); + EXPECT_EQ(callbacks.streamErrors, 1); + EXPECT_EQ(callbacks.sessionErrors, 0); + +} diff --git a/proxygen/lib/http/codec/test/TestUtils.cpp b/proxygen/lib/http/codec/test/TestUtils.cpp new file mode 100644 index 0000000000..80ffb4d3fe --- /dev/null +++ b/proxygen/lib/http/codec/test/TestUtils.cpp @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/test/TestUtils.h" + +#include +#include + +using namespace folly::io; +using namespace folly; +using namespace std; +using namespace testing; + +namespace proxygen { + +const HTTPSettings kDefaultIngressSettings{ + {SettingsId::INITIAL_WINDOW_SIZE, 65536} +}; + +std::unique_ptr getPriorityMessage(uint8_t priority) { + auto ret = folly::make_unique(); + ret->setSPDY(2); + ret->setPriority(priority); + return ret; +} + +std::unique_ptr makeBuf(uint32_t size) { + auto out = folly::IOBuf::create(size); + out->append(size); + // fill with random junk + RWPrivateCursor cursor(out.get()); + while (cursor.length() >= 8) { + cursor.write(folly::Random::rand64()); + } + while (cursor.length()) { + cursor.write((uint8_t)folly::Random::rand32()); + } + return out; +} + +std::unique_ptr> +makeMockParallelCodec(TransportDirection dir) { + auto codec = folly::make_unique>(); + EXPECT_CALL(*codec, supportsParallelRequests()) + .WillRepeatedly(testing::Return(true)); + EXPECT_CALL(*codec, isReusable()) + .WillRepeatedly(testing::Return(true)); + EXPECT_CALL(*codec, getTransportDirection()) + .WillRepeatedly(testing::Return(dir)); + EXPECT_CALL(*codec, getIngressSettings()) + .WillRepeatedly(testing::Return(&kDefaultIngressSettings)); + return codec; +} + +std::unique_ptr> +makeDownstreamParallelCodec() { + return makeMockParallelCodec(TransportDirection::DOWNSTREAM); +} + +std::unique_ptr> +makeUpstreamParallelCodec() { + return makeMockParallelCodec(TransportDirection::UPSTREAM); +} + +HTTPMessage getGetRequest(const std::string& url) { + HTTPMessage req; + req.setMethod("GET"); + req.setURL(url); + req.setHTTPVersion(1, 1); + req.getHeaders().set(HTTP_HEADER_HOST, "www.foo.com"); + return req; +} + +std::unique_ptr makeGetRequest() { + return folly::make_unique(getGetRequest()); +} + +HTTPMessage getPostRequest() { + HTTPMessage req; + req.setMethod("POST"); + req.setURL("/"); + req.setHTTPVersion(1, 1); + req.getHeaders().set(HTTP_HEADER_HOST, "www.foo.com"); + req.getHeaders().set(HTTP_HEADER_CONTENT_LENGTH, "200"); + return req; +} + +std::unique_ptr makePostRequest() { + return folly::make_unique(getPostRequest()); +} + +std::unique_ptr makeResponse(uint16_t statusCode) { + auto resp = folly::make_unique(); + resp->setStatusCode(statusCode); + return resp; +} + +std::tuple, std::unique_ptr > +makeResponse(uint16_t statusCode, size_t len) { + auto resp = makeResponse(statusCode); + resp->getHeaders().set(HTTP_HEADER_CONTENT_LENGTH, folly::to(len)); + return std::make_pair(std::move(resp), makeBuf(len)); +} + +void fakeMockCodec(MockHTTPCodec& codec) { + // For each generate* function, write some data to the chain + EXPECT_CALL(codec, generateHeader(_, _, _, _, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + const HTTPMessage& msg, + HTTPCodec::StreamID assocStream, + HTTPHeaderSize* size) { + writeBuf.append(makeBuf(10)); + })); + + EXPECT_CALL(codec, generateBody(_, _, _, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + std::shared_ptr chain, + bool eom) { + auto len = chain->computeChainDataLength(); + writeBuf.append(chain->clone()); + return len; + })); + + EXPECT_CALL(codec, generateChunkHeader(_, _, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + size_t length) { + writeBuf.append(makeBuf(length)); + return length; + })); + + EXPECT_CALL(codec, generateChunkTerminator(_, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream) { + writeBuf.append(makeBuf(4)); + return 4; + })); + + EXPECT_CALL(codec, generateTrailers(_, _, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + const HTTPHeaders& trailers) { + writeBuf.append(makeBuf(30)); + return 30; + })); + + EXPECT_CALL(codec, generateEOM(_, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream) { + writeBuf.append(makeBuf(6)); + return 6; + })); + + EXPECT_CALL(codec, generateRstStream(_, _, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + ErrorCode code) { + writeBuf.append(makeBuf(6)); + return 6; + })); + + EXPECT_CALL(codec, generateGoaway(_, _, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + uint32_t lastStream, + ErrorCode code) { + writeBuf.append(makeBuf(6)); + return 6; + })); + + EXPECT_CALL(codec, generatePingRequest(_)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf) { + writeBuf.append(makeBuf(6)); + return 6; + })); + + EXPECT_CALL(codec, generatePingReply(_, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + uint64_t id) { + writeBuf.append(makeBuf(6)); + return 6; + })); + + EXPECT_CALL(codec, generateSettings(_)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf) { + writeBuf.append(makeBuf(6)); + return 6; + })); + + EXPECT_CALL(codec, generateWindowUpdate(_, _, _)) + .WillRepeatedly(Invoke([] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + uint32_t delta) { + writeBuf.append(makeBuf(6)); + return 6; + })); + +} + +} diff --git a/proxygen/lib/http/codec/test/TestUtils.h b/proxygen/lib/http/codec/test/TestUtils.h new file mode 100644 index 0000000000..14425cd020 --- /dev/null +++ b/proxygen/lib/http/codec/test/TestUtils.h @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/test/MockHTTPCodec.h" +#include "proxygen/lib/utils/TestUtils.h" + +#include + +namespace proxygen { + +/** + * parse the input data using codec, using atOnce to determine how much data + * should go through the parser at one time + * + * atOnce < 0: use random chunk lengths + * atOnce = 0: single chunk + * atOnce > 0: use specified chunk length + */ +template +size_t parse(T* codec, + const uint8_t* inputData, + uint32_t length, + int32_t atOnce = 0) { + + const uint8_t* start = inputData; + size_t consumed = 0; + std::uniform_int_distribution lenDistribution(1, length / 2 + 1); + std::mt19937 rng; + + if (atOnce == 0) { + atOnce = length; + } + + folly::IOBufQueue input(folly::IOBufQueue::cacheChainLength()); + while (length > 0) { + if (consumed == 0) { + // Parser wants more data + uint32_t len = atOnce; + if (atOnce < 0) { + // use random chunks + len = lenDistribution(rng); + } + uint32_t chunkLen = std::min(length, len); + input.append(std::move(folly::IOBuf::copyBuffer(start, chunkLen))); + start += chunkLen; + length -= chunkLen; + } + consumed = codec->onIngress(*input.front()); + input.split(consumed); + if (input.front() == nullptr && consumed > 0) { + consumed = 0; + } + } + return input.chainLength(); +} + +class FakeHTTPCodecCallback : public HTTPCodec::Callback { + public: + FakeHTTPCodecCallback() {} + + void onMessageBegin(HTTPCodec::StreamID stream, HTTPMessage*) { + messageBegin++; + } + void onPushMessageBegin(HTTPCodec::StreamID stream, + HTTPCodec::StreamID assocStream, + HTTPMessage*) { + messageBegin++; + assocStreamId = assocStream; + } + void onHeadersComplete(HTTPCodec::StreamID stream, + std::unique_ptr inMsg) { + headersComplete++; + msg = std::move(inMsg); + } + void onBody(HTTPCodec::StreamID stream, + std::unique_ptr chain) { + bodyCalls++; + bodyLength += chain->computeChainDataLength(); + data.append(std::move(chain)); + } + void onChunkHeader(HTTPCodec::StreamID stream, size_t length) { + chunkHeaders++; + } + void onChunkComplete(HTTPCodec::StreamID stream) { + chunkComplete++; + } + void onTrailersComplete(HTTPCodec::StreamID stream, + std::unique_ptr inTrailers) { + trailers++; + } + void onMessageComplete(HTTPCodec::StreamID stream, bool upgrade) { + messageComplete++; + } + void onError(HTTPCodec::StreamID stream, + const HTTPException& error, + bool newStream) { + if (stream) { + streamErrors++; + } else { + sessionErrors++; + } + lastParseError = folly::make_unique(error); + } + + void onAbort(HTTPCodec::StreamID stream, ErrorCode code) { + ++aborts; + } + + void onGoaway(uint64_t lastGoodStreamID, ErrorCode code) { + ++goaways; + } + + void onPingRequest(uint64_t uniqueID) override { + recvPingRequest = uniqueID; + } + + void onPingReply(uint64_t uniqueID) override { + recvPingReply = uniqueID; + } + + void onWindowUpdate(HTTPCodec::StreamID stream, uint32_t amount) override { + windowUpdateCalls++; + windowUpdates[stream].push_back(amount); + } + + void onSettings(const SettingsList& settings) override { + this->settings++; + for (auto& setting: settings) { + if (setting.id == SettingsId::INITIAL_WINDOW_SIZE) { + windowSize = setting.value; + } else if (setting.id == SettingsId::MAX_CONCURRENT_STREAMS) { + maxStreams = setting.value; + } + } + } + + void onSettingsAck() override { + settingsAcks++; + } + + uint32_t numOutgoingStreams() const override { + return 0; + } + + uint32_t numIncomingStreams() const override { + return messageBegin; + } + + void reset() { + assocStreamId = 0; + messageBegin = 0; + headersComplete = 0; + messageComplete = 0; + bodyCalls = 0; + bodyLength = 0; + chunkHeaders = 0; + chunkComplete = 0; + trailers = 0; + aborts = 0; + goaways = 0; + sessionErrors = 0; + streamErrors = 0; + recvPingRequest = 0; + recvPingReply = 0; + windowUpdateCalls = 0; + settings = 0; + settingsAcks = 0; + windowSize = 0; + maxStreams = 0; + windowUpdates.clear(); + data.move(); + msg.reset(); + lastParseError.reset(); + } + + HTTPCodec::StreamID assocStreamId{0}; + uint32_t messageBegin{0}; + uint32_t headersComplete{0}; + uint32_t messageComplete{0}; + uint32_t bodyCalls{0}; + uint32_t bodyLength{0}; + uint32_t chunkHeaders{0}; + uint32_t chunkComplete{0}; + uint32_t trailers{0}; + uint32_t aborts{0}; + uint32_t goaways{0}; + uint32_t sessionErrors{0}; + uint32_t streamErrors{0}; + uint64_t recvPingRequest{0}; + uint64_t recvPingReply{0}; + uint32_t windowUpdateCalls{0}; + uint32_t settings{0}; + uint32_t settingsAcks{0}; + uint32_t windowSize{0}; + uint32_t maxStreams{0}; + std::map > windowUpdates; + folly::IOBufQueue data; + + std::unique_ptr msg; + std::unique_ptr lastParseError; +}; + +MATCHER_P(PtrBufHasLen, n, "") { + return arg->computeChainDataLength() == n; +} + +std::unique_ptr getPriorityMessage(uint8_t priority); + +std::unique_ptr makeBuf(uint32_t size = 10); + +std::unique_ptr> +makeDownstreamParallelCodec(); + +std::unique_ptr> +makeUpstreamParallelCodec(); + +HTTPMessage getGetRequest(const std::string& url = std::string("/")); +HTTPMessage getPostRequest(); +std::unique_ptr makeGetRequest(); +std::unique_ptr makePostRequest(); +std::unique_ptr makeResponse(uint16_t statusCode); + +std::tuple, std::unique_ptr > +makeResponse(uint16_t statusCode, size_t len); + +// Takes a MockHTTPCodec and fakes out its interface +void fakeMockCodec(MockHTTPCodec& codec); + +} diff --git a/proxygen/lib/http/generate_http_gperfs.rb b/proxygen/lib/http/generate_http_gperfs.rb new file mode 100755 index 0000000000..1d2009d7e9 --- /dev/null +++ b/proxygen/lib/http/generate_http_gperfs.rb @@ -0,0 +1,87 @@ +#! /opt/local/bin/ruby + +require 'optparse' + +fbcodeDir = ENV['FBCODE_DIR'] +installDir = ENV['INSTALL_DIR'] +proxygenHttpPath = File.join(fbcodeDir, "proxygen", "lib", "http") +useInstalledGperf = ENV['INSTALLED_GPERF'] + +headerFiles = [] +OptionParser.new do |opts| + opts.banner = "Usage: example.rb [options]" + + # This is modestly fragile because there's no simple way to ignore arguments + # you don't care about + opts.on("-I", "--install_dir DIR" "Install dir") do |f| + end + opts.on("-F", "--fbcode_dir DIR" "fbcode dir") do |f| + end + opts.on("-H", "--custom_header FILE", "Additional header files") do |f| + headerFiles.push(File.join(fbcodeDir, f)) + end +end.parse! + +# 1. HTTPCommonHeaders + +# This script uses the list in HTTPCommonHeaders.txt to generate +# HTTPCommonHeaders.h from HTTPCommonHeaders.template.h, +# HTTPCommonHeaders.gperf from HTTPCommonHeaders.template.gperf, +# and then HTTPCommonHeaders.cpp from HTTPCommonHeaders.gperf. + +# the sources +templateHPath = File.join(proxygenHttpPath, "HTTPCommonHeaders.template.h") +templateGperfPath = File.join(proxygenHttpPath, + "HTTPCommonHeaders.template.gperf") +headerNamesListPath = File.join(proxygenHttpPath, "HTTPCommonHeaders.txt") +headerFiles.push(headerNamesListPath) + +# the destinations +hPath = File.join(installDir, "HTTPCommonHeaders.h") +gperfPath = File.join(installDir, "HTTPCommonHeaders.gperf") +cppPath = File.join(installDir, "HTTPCommonHeaders.cpp") + +headerLines = [] +headerFiles.each {|headerFile| + File.open(headerFile, "r").each_line { |line| + headerName = line.strip + next if headerName.empty? + headerLines.push(headerName) + } +} +headerLines = headerLines.sort.uniq +insertedContentH = "" +insertedContentGperf = "%%\n" +nextEnumValue = 2 +headerLines.each {|headerName| + enumName = "HTTP_HEADER_" + headerName.gsub('-', '_').upcase + insertedContentH << " #{enumName} = #{nextEnumValue},\n" + insertedContentGperf << "#{headerName}, #{enumName}\n" + nextEnumValue += 1 +} +insertedContentGperf << "%%" + +# generate HTTPCommonHeaders.h +f = File.open(hPath, "w") +f.write( + File.open(templateHPath, "r").read.gsub('%%%%%', insertedContentH) +) +f.close + +# generate HTTPCommonHeaders.gperf +f = File.open(gperfPath, "w") +f.write( + File.open(templateGperfPath, "r").read.gsub('%%%%%', insertedContentGperf) +) +f.close + +# and finally run gperf +if useInstalledGperf.nil? + gperf = File.join(fbcodeDir, "third-party2", "gperf", + "3.0.4", "gcc-4.8.1-glibc-2.17-fb", + "c3f970a", "bin", "gperf") +else + gperf = "gperf" +end + +system("#{gperf} #{gperfPath} -m 5 --output-file=#{cppPath}") diff --git a/proxygen/lib/http/session/AckLatencyEvent.h b/proxygen/lib/http/session/AckLatencyEvent.h new file mode 100644 index 0000000000..1eb43ea5bc --- /dev/null +++ b/proxygen/lib/http/session/AckLatencyEvent.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +struct AckLatencyEvent { + // The byte number that was acknowledged. + unsigned int byteNo; + // The latency between sending the byte and receiving the ack for that byte. + std::chrono::nanoseconds latency; +}; + +} diff --git a/proxygen/lib/http/session/ByteEventTracker.cpp b/proxygen/lib/http/session/ByteEventTracker.cpp new file mode 100644 index 0000000000..3b6fc7d4cc --- /dev/null +++ b/proxygen/lib/http/session/ByteEventTracker.cpp @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/ByteEventTracker.h" + +#include "proxygen/lib/http/session/HTTPSession.h" +#include "proxygen/lib/http/session/HTTPSessionStats.h" + +#include +#include + +using apache::thrift::async::TAsyncSocket; +using apache::thrift::async::TAsyncTimeoutSet; +using apache::thrift::async::TAsyncTransport; +using std::string; +using std::vector; + +namespace proxygen { + +ByteEventTracker::~ByteEventTracker() { + CHECK(byteEvents_.empty()); +} + +ByteEventTracker::ByteEventTracker(ByteEventTracker&& other) noexcept { + nextLastByteEvent_ = other.nextLastByteEvent_; + other.nextLastByteEvent_ = nullptr; + + byteEvents_ = std::move(other.byteEvents_); + CHECK(other.byteEvents_.empty()); +} + +void ByteEventTracker::processByteEvents(uint64_t bytesWritten, + bool eorTrackingEnabled) { + bool advanceEOM = false; + + while (!byteEvents_.empty() && + (byteEvents_.front().byteOffset_ <= bytesWritten)) { + ByteEvent& event = byteEvents_.front(); + int64_t latency; + auto txn = event.getTransaction(); + + switch (event.eventType_) { + case ByteEvent::FIRST_HEADER_BYTE: + txn->onEgressHeaderFirstByte(); + break; + case ByteEvent::FIRST_BYTE: + txn->onEgressBodyFirstByte(); + break; + case ByteEvent::LAST_BYTE: + txn->onEgressBodyLastByte(); + addAckToLastByteEvent(txn, event, eorTrackingEnabled); + advanceEOM = true; + break; + case ByteEvent::PING_REPLY_SENT: + latency = event.getLatency(); + callback_->onPingReplyLatency(latency); + break; + } + + VLOG(5) << " removing ByteEvent " << event; + delete &event; + } + + if (eorTrackingEnabled && advanceEOM) { + nextLastByteEvent_ = nullptr; + for (auto& event : byteEvents_) { + if (event.eventType_ == ByteEvent::LAST_BYTE) { + nextLastByteEvent_ = &event; + break; + } + } + + VLOG(5) << "Setting nextLastByteNo to " + << (nextLastByteEvent_ ? nextLastByteEvent_->byteOffset_ : 0); + } +} + +size_t ByteEventTracker::drainByteEvents() { + size_t numEvents = 0; + // everything is dead from here on, let's just drop all extra refs to txns + while (!byteEvents_.empty()) { + delete &byteEvents_.front(); + ++numEvents; + } + nextLastByteEvent_ = nullptr; + return numEvents; +} + +void ByteEventTracker::addLastByteEvent( + HTTPTransaction* txn, + uint64_t byteNo, + bool eorTrackingEnabled) noexcept { + VLOG(5) << " adding last byte event for " << byteNo; + TransactionByteEvent* event = new TransactionByteEvent( + byteNo, ByteEvent::LAST_BYTE, HTTPTransaction::CallbackGuard(*txn)); + byteEvents_.push_back(*event); + + if (eorTrackingEnabled && !nextLastByteEvent_) { + VLOG(5) << " set nextLastByteNo to " << event->byteOffset_; + nextLastByteEvent_ = event; + } +} + +void ByteEventTracker::addPingByteEvent(size_t pingSize, + TimePoint timestamp, + uint64_t bytesScheduled) { + // register a byte event on ping reply sent, and adjust the byteOffset_ + // for others by one ping size + uint64_t offset = bytesScheduled + pingSize; + auto i = byteEvents_.rbegin(); + for (; i != byteEvents_.rend(); ++i) { + if (i->byteOffset_ > bytesScheduled) { + VLOG(5) << "pushing back ByteEvent from " << *i << " to " + << ByteEvent(i->byteOffset_ + pingSize, i->eventType_); + i->byteOffset_ += pingSize; + } else { + break; // the rest of the events are already scheduled + } + } + + ByteEvent* be = new PingByteEvent(offset, timestamp); + if (i == byteEvents_.rend()) { + byteEvents_.push_front(*be); + } else if (i == byteEvents_.rbegin()) { + byteEvents_.push_back(*be); + } else { + --i; + CHECK(i->byteOffset_ > bytesScheduled); + byteEvents_.insert(i.base(), *be); + } +} + +uint64_t ByteEventTracker::preSend(bool* cork, + bool* eom, + uint64_t bytesWritten) { + if (nextLastByteEvent_) { + uint64_t nextLastByteNo = nextLastByteEvent_->byteOffset_; + CHECK(nextLastByteNo > bytesWritten); + uint64_t needed = nextLastByteNo - bytesWritten; + VLOG(5) << "needed: " << needed << "(" << nextLastByteNo << "-" + << bytesWritten << ")"; + + return needed; + } + return 0; +} + +void ByteEventTracker::addFirstBodyByteEvent(uint64_t offset, + HTTPTransaction* txn) { + byteEvents_.push_back( + *new TransactionByteEvent( + offset, ByteEvent::FIRST_BYTE, HTTPTransaction::CallbackGuard(*txn))); +} + +void ByteEventTracker::addFirstHeaderByteEvent(uint64_t offset, + HTTPTransaction* txn) { + // onWriteSuccess() is called after the entire header has been written. + // It does not catch partial write case. + byteEvents_.push_back( + *new TransactionByteEvent(offset, + ByteEvent::FIRST_HEADER_BYTE, + HTTPTransaction::CallbackGuard(*txn))); +} + +} // proxygen diff --git a/proxygen/lib/http/session/ByteEventTracker.h b/proxygen/lib/http/session/ByteEventTracker.h new file mode 100644 index 0000000000..027e8f60fc --- /dev/null +++ b/proxygen/lib/http/session/ByteEventTracker.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/AckLatencyEvent.h" +#include "proxygen/lib/http/session/ByteEvents.h" +#include "proxygen/lib/http/session/HTTPTransaction.h" +#include "proxygen/lib/utils/Time.h" + +#include + +namespace proxygen { + +class TTLBAStats; + +// An API for using TCP events from the HTTP level +class ByteEventTracker { + public: + class Callback { + public: + virtual ~Callback() {} + virtual void onPingReplyLatency(int64_t latency) noexcept = 0; + virtual uint64_t getAppBytesWritten() noexcept = 0; + virtual uint64_t getRawBytesWritten() noexcept = 0; + virtual void onDeleteAckEvent() = 0; + }; + + virtual ~ByteEventTracker(); + explicit ByteEventTracker(Callback* callback): callback_(callback) {} + explicit ByteEventTracker(ByteEventTracker&&) noexcept; + void setCallback(Callback* callback) { callback_ = callback; } + + void addPingByteEvent(size_t pingSize, + TimePoint timestamp, + uint64_t bytesScheduled); + + void addFirstBodyByteEvent(uint64_t offset, HTTPTransaction* txn); + void addFirstHeaderByteEvent(uint64_t offset, HTTPTransaction* txn); + + virtual size_t drainByteEvents(); + virtual void processByteEvents(uint64_t bytesWritten, + bool eorTrackingEnabled); + virtual void addLastByteEvent(HTTPTransaction* txn, + uint64_t byteNo, + bool eorTrackingEnabled) noexcept; + + /** + * Returns the number of bytes needed or 0 when there's nothing to do. + */ + virtual uint64_t preSend(bool* cork, bool* eom, uint64_t bytesWritten); + + virtual void addAckToLastByteEvent(HTTPTransaction* txn, + const ByteEvent& lastByteEvent, + bool eorTrackingEnabled) {} + + virtual void deleteAckEvent( + std::vector::iterator& it) noexcept {} + + virtual bool setMaxTcpAckTracked( + uint32_t maxAckTracked, + apache::thrift::async::TAsyncTimeoutSet* ackLatencyTimeouts, + apache::thrift::async::TAsyncTransport* transport) { return false; } + + virtual void setTTLBAStats(TTLBAStats* stats) {} + + virtual void onAckLatencyEvent(const AckLatencyEvent&) {} + + private: + // byteEvents_ is in the ascending order of ByteEvent::byteOffset_ + folly::IntrusiveList byteEvents_; + + protected: + Callback* callback_; + + ByteEvent* nextLastByteEvent_{nullptr}; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/ByteEvents.cpp b/proxygen/lib/http/session/ByteEvents.cpp new file mode 100644 index 0000000000..d31ee15f16 --- /dev/null +++ b/proxygen/lib/http/session/ByteEvents.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/ByteEvents.h" + +#include "proxygen/lib/utils/Time.h" + +namespace proxygen { + +const char* const kTypeStrings[] = { + "FIRST_BYTE", + "LAST_BYTE", + "PING_REPLY_SENT", + "FIRST_HEADER_BYTE", +}; + +std::ostream& operator<<(std::ostream& os, const ByteEvent& be) { + os << folly::to( + "(", kTypeStrings[be.eventType_], ", ", be.byteOffset_, ")"); + return os; +} + +int64_t PingByteEvent::getLatency() { + return millisecondsSince(pingRequestReceivedTime_).count(); +} + +} // proxygen diff --git a/proxygen/lib/http/session/ByteEvents.h b/proxygen/lib/http/session/ByteEvents.h new file mode 100644 index 0000000000..8450cbe1d7 --- /dev/null +++ b/proxygen/lib/http/session/ByteEvents.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPTransaction.h" +#include "proxygen/lib/utils/Time.h" + +#include +#include + +namespace proxygen { + +class ByteEvent { + public: + enum EventType { + FIRST_BYTE, + LAST_BYTE, + PING_REPLY_SENT, + FIRST_HEADER_BYTE, + }; + + ByteEvent(uint64_t byteOffset, EventType eventType) + : eventType_(eventType), eomTracked_(0), byteOffset_(byteOffset) {} + virtual ~ByteEvent() {} + virtual HTTPTransaction* getTransaction() { return nullptr; } + virtual int64_t getLatency() { return -1; } + + folly::IntrusiveListHook listHook; + EventType eventType_:3; // packed w/ byteOffset_ + size_t eomTracked_:1; + uint64_t byteOffset_:(8*sizeof(uint64_t)-4); +}; + +std::ostream& operator<<(std::ostream& os, const ByteEvent& txn); + +class TransactionByteEvent : public ByteEvent { + public: + TransactionByteEvent(uint64_t byteNo, + EventType eventType, + HTTPTransaction::CallbackGuard cg) + : ByteEvent(byteNo, eventType), cg_(cg) {} + + HTTPTransaction* getTransaction() override { + return &(cg_.peekTransaction()); + } + + HTTPTransaction::CallbackGuard cg_; // refcounted transaction +}; + +class AckTimeout + : public apache::thrift::async::TAsyncTimeoutSet::Callback { + public: + /** + * The instances of AckTimeout::Callback *MUST* outlive the AckTimeout it is + * registered on. + */ + class Callback { + public: + virtual ~Callback() {} + virtual void ackTimeoutExpired(uint64_t byteNo) noexcept = 0; + }; + + AckTimeout(Callback* callback, uint64_t byteNo) + : callback_(callback), byteNo_(byteNo) {} + + void timeoutExpired() noexcept { + callback_->ackTimeoutExpired(byteNo_); + } + + private: + Callback* callback_; + uint64_t byteNo_; +}; + +class AckByteEvent : public TransactionByteEvent { + public: + AckByteEvent(AckTimeout::Callback* callback, + uint64_t byteNo, + EventType eventType, + HTTPTransaction::CallbackGuard cg) + : TransactionByteEvent(byteNo, eventType, cg), + timeout(callback, byteNo) {} + + AckTimeout timeout; +}; + +class PingByteEvent : public ByteEvent { + public: + PingByteEvent(uint64_t byteOffset, TimePoint pingRequestReceivedTime) + : ByteEvent(byteOffset, PING_REPLY_SENT), + pingRequestReceivedTime_(pingRequestReceivedTime) {} + + int64_t getLatency() override; + + TimePoint pingRequestReceivedTime_; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/CodecErrorResponseHandler.cpp b/proxygen/lib/http/session/CodecErrorResponseHandler.cpp new file mode 100644 index 0000000000..a959a8e704 --- /dev/null +++ b/proxygen/lib/http/session/CodecErrorResponseHandler.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/CodecErrorResponseHandler.h" + +#include + +using folly::IOBuf; +using std::string; +using std::unique_ptr; + +namespace proxygen { + +CodecErrorResponseHandler::CodecErrorResponseHandler(ErrorCode statusCode): + txn_(nullptr) { +} + +CodecErrorResponseHandler::~CodecErrorResponseHandler() { +} + +void +CodecErrorResponseHandler::setTransaction(HTTPTransaction* txn) noexcept { + txn_ = txn; +} + +void +CodecErrorResponseHandler::detachTransaction() noexcept { + delete this; +} + +void +CodecErrorResponseHandler::onHeadersComplete( + std::unique_ptr msg) noexcept { + VLOG(4) << "discarding headers"; +} + +void +CodecErrorResponseHandler::onBody(unique_ptr chain) noexcept { + VLOG(4) << "discarding request body"; +} + +void +CodecErrorResponseHandler::onTrailers( + unique_ptr trailers) noexcept { + VLOG(4) << "discarding request trailers"; +} + +void +CodecErrorResponseHandler::onEOM() noexcept { +} + +void +CodecErrorResponseHandler::onUpgrade(UpgradeProtocol protocol) noexcept { +} + +void CodecErrorResponseHandler::onError(const HTTPException& error) noexcept { + VLOG(4) << "processing error " << error; + txn_->sendAbort(); +} + +} // proxygen diff --git a/proxygen/lib/http/session/CodecErrorResponseHandler.h b/proxygen/lib/http/session/CodecErrorResponseHandler.h new file mode 100644 index 0000000000..beb910b7b0 --- /dev/null +++ b/proxygen/lib/http/session/CodecErrorResponseHandler.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPTransaction.h" + +namespace proxygen { + +class HTTPErrorPage; + +class CodecErrorResponseHandler: + public HTTPTransaction::Handler { +public: + explicit CodecErrorResponseHandler(ErrorCode statusCode); + + // HTTPTransaction::Handler methods + void setTransaction(HTTPTransaction* txn) noexcept; + void detachTransaction() noexcept; + void onHeadersComplete(std::unique_ptr msg) noexcept; + void onBody(std::unique_ptr chain) noexcept; + void onTrailers(std::unique_ptr trailers) noexcept; + void onEOM() noexcept; + void onUpgrade(UpgradeProtocol protocol) noexcept override; + void onError(const HTTPException& error) noexcept; + // These are no-ops since the error response is already in memory + void onEgressPaused() noexcept {}; + void onEgressResumed() noexcept {}; + +private: + ~CodecErrorResponseHandler(); + + HTTPTransaction* txn_; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPDirectResponseHandler.cpp b/proxygen/lib/http/session/HTTPDirectResponseHandler.cpp new file mode 100644 index 0000000000..9c0a0bde70 --- /dev/null +++ b/proxygen/lib/http/session/HTTPDirectResponseHandler.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPDirectResponseHandler.h" + +#include "proxygen/lib/http/session/HTTPErrorPage.h" + +#include + +using folly::IOBuf; +using std::string; +using std::unique_ptr; + +namespace proxygen { + +HTTPDirectResponseHandler::HTTPDirectResponseHandler( + unsigned statusCode, const std::string& statusMsg, + const HTTPErrorPage* errorPage): + txn_(nullptr), + errorPage_(errorPage), + statusMessage_(statusMsg), + statusCode_(statusCode), + headersSent_(false), + eomSent_(false), + forceConnectionClose_(true) { +} + +HTTPDirectResponseHandler::~HTTPDirectResponseHandler() { +} + +void +HTTPDirectResponseHandler::setTransaction(HTTPTransaction* txn) noexcept { + txn_ = txn; +} + +void +HTTPDirectResponseHandler::detachTransaction() noexcept { + delete this; +} + +void +HTTPDirectResponseHandler::onHeadersComplete( + std::unique_ptr msg) noexcept { + VLOG(4) << "processing request"; + headersSent_ = true; + HTTPMessage response; + std::unique_ptr responseBody; + response.setHTTPVersion(1, 1); + response.setStatusCode(statusCode_); + if (!statusMessage_.empty()) { + response.setStatusMessage(statusMessage_); + } else { + response.setStatusMessage(HTTPMessage::getDefaultReason(statusCode_)); + } + if (forceConnectionClose_) { + response.getHeaders().add(HTTP_HEADER_CONNECTION, "close"); + } + if (errorPage_) { + HTTPErrorPage::Page page = errorPage_->generate(0, statusCode_, + statusMessage_, nullptr, empty_string); + VLOG(4) << "sending error page with type " << page.contentType; + response.getHeaders().add(HTTP_HEADER_CONTENT_TYPE, page.contentType); + responseBody = std::move(page.content); + } + response.getHeaders().add(HTTP_HEADER_CONTENT_LENGTH, folly::to( + responseBody ? responseBody->computeChainDataLength() : 0)); + txn_->sendHeaders(response); + if (responseBody) { + txn_->sendBody(std::move(responseBody)); + } +} + +void +HTTPDirectResponseHandler::onBody(unique_ptr chain) noexcept { + VLOG(4) << "discarding request body"; +} + +void +HTTPDirectResponseHandler::onTrailers( + unique_ptr trailers) noexcept { + VLOG(4) << "discarding request trailers"; +} + +void +HTTPDirectResponseHandler::onEOM() noexcept { + eomSent_ = true; + txn_->sendEOM(); +} + +void +HTTPDirectResponseHandler::onUpgrade(UpgradeProtocol protocol) noexcept { +} + +void HTTPDirectResponseHandler::onError(const HTTPException& error) noexcept { + if (error.getDirection() == HTTPException::Direction::INGRESS) { + if (error.getProxygenError() == kErrorTimeout) { + VLOG(4) << "processing ingress timeout"; + if (!headersSent_) { + onHeadersComplete(nullptr); + } + if (!eomSent_) { + onEOM(); + } + } else { + VLOG(4) << "processing ingress error"; + if (!headersSent_) { + onHeadersComplete(nullptr); + } + if (!eomSent_) { + onEOM(); + } + } + } +} + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPDirectResponseHandler.h b/proxygen/lib/http/session/HTTPDirectResponseHandler.h new file mode 100644 index 0000000000..114a08739b --- /dev/null +++ b/proxygen/lib/http/session/HTTPDirectResponseHandler.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPTransaction.h" + +namespace proxygen { + +class HTTPErrorPage; + +class HTTPDirectResponseHandler: + public HTTPTransaction::Handler { +public: + HTTPDirectResponseHandler(unsigned statusCode, const std::string& statusMsg, + const HTTPErrorPage* errorPage = nullptr); + + void forceConnectionClose(bool close) { + forceConnectionClose_ = close; + } + // HTTPTransaction::Handler methods + void setTransaction(HTTPTransaction* txn) noexcept; + void detachTransaction() noexcept; + void onHeadersComplete(std::unique_ptr msg) noexcept; + void onBody(std::unique_ptr chain) noexcept; + void onTrailers(std::unique_ptr trailers) noexcept; + void onEOM() noexcept; + void onUpgrade(UpgradeProtocol protocol) noexcept override; + void onError(const HTTPException& error) noexcept; + // These are no-ops since the direct response is already in memory + void onEgressPaused() noexcept {}; + void onEgressResumed() noexcept {}; + +private: + ~HTTPDirectResponseHandler(); + + HTTPTransaction* txn_; + const HTTPErrorPage* errorPage_; + std::string statusMessage_; + unsigned statusCode_; + bool headersSent_:1; + bool eomSent_:1; + bool forceConnectionClose_:1; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPDownstreamSession.cpp b/proxygen/lib/http/session/HTTPDownstreamSession.cpp new file mode 100644 index 0000000000..6caab6a6c7 --- /dev/null +++ b/proxygen/lib/http/session/HTTPDownstreamSession.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPDownstreamSession.h" + +#include "proxygen/lib/http/session/HTTPSessionController.h" +#include "proxygen/lib/http/session/HTTPTransaction.h" + +namespace proxygen { + +HTTPDownstreamSession::~HTTPDownstreamSession() { +} + +void +HTTPDownstreamSession::setupOnHeadersComplete(HTTPTransaction* txn, + HTTPMessage* msg) { + CHECK(!txn->getHandler()); + + // We need to find a Handler to process the transaction. + // Note: The handler is responsible for freeing itself + // when it has finished processing the transaction. The + // transaction is responsible for freeing itself when both the + // ingress and egress messages have completed (or failed). + HTTPTransaction::Handler* handler = nullptr; + + // In the general case, delegate to the handler factory to generate + // a handler for the transaction. + handler = controller_->getRequestHandler(*txn, msg); + CHECK(handler); + + txn->setHandler(handler); + setNewTransactionPauseState(txn); +} + +HTTPTransaction::Handler* +HTTPDownstreamSession::getParseErrorHandler(HTTPTransaction* txn, + const HTTPException& error) { + // we encounter an error before we finish reading the ingress headers. + return controller_->getParseErrorHandler(txn, error, localAddr_); +} + +HTTPTransaction::Handler* +HTTPDownstreamSession::getTransactionTimeoutHandler( + HTTPTransaction* txn) { + return controller_->getTransactionTimeoutHandler(txn, localAddr_); +} + +void +HTTPDownstreamSession::onHeadersSent(const HTTPMessage& headers, + bool codecWasReusable) { + if (!codec_->isReusable()) { + // If the codec turned unreusable, some thing wrong must have happened. + // Basically, the proxy decides the connection is not reusable. + // e.g, an error message is being sent with Connection: close + if (codecWasReusable) { + uint32_t statusCode = headers.getStatusCode(); + if (statusCode >= 500) { + setCloseReason(ConnectionCloseReason::REMOTE_ERROR); + } else { + if (statusCode >= 400) { + setCloseReason(ConnectionCloseReason::ERR_RESP); + } else { + // should not be here + setCloseReason(ConnectionCloseReason::UNKNOWN); + } + } + } else { + // shouldn't happen... this case is detected by REQ_NOTREUSABLE + setCloseReason(ConnectionCloseReason::UNKNOWN); + } + } +} + +bool HTTPDownstreamSession::allTransactionsStarted() const { + for (const auto txn: transactions_) { + if (txn.second->isPushed() && !txn.second->isEgressStarted()) { + return false; + } + } + return true; +} + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPDownstreamSession.h b/proxygen/lib/http/session/HTTPDownstreamSession.h new file mode 100644 index 0000000000..6833890fae --- /dev/null +++ b/proxygen/lib/http/session/HTTPDownstreamSession.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPSession.h" + +namespace proxygen { + +class HTTPSessionStats; +class HTTPDownstreamSession final: public HTTPSession { + public: + /** + * @param sock An open socket on which any applicable TLS handshaking + * has been completed already. + * @param localAddr Address and port of the local end of the socket. + * @param peerAddr Address and port of the remote end of the socket. + * @param codec A codec with which to parse/generate messages in + * whatever HTTP-like wire format this session needs. + */ + HTTPDownstreamSession( + apache::thrift::async::TAsyncTimeoutSet* transactionTimeouts, + apache::thrift::async::TAsyncTransport::UniquePtr&& sock, + const folly::SocketAddress& localAddr, + const folly::SocketAddress& peerAddr, + HTTPSessionController* controller, + std::unique_ptr codec, + const TransportInfo& tinfo, + InfoCallback* infoCallback = nullptr): + HTTPSession(transactionTimeouts, std::move(sock), localAddr, peerAddr, + CHECK_NOTNULL(controller), std::move(codec), tinfo, + infoCallback) { + CHECK(codec_->getTransportDirection() == TransportDirection::DOWNSTREAM); + } + + private: + ~HTTPDownstreamSession() override; + + /** + * Called by onHeadersComplete(). + */ + void setupOnHeadersComplete(HTTPTransaction* txn, HTTPMessage* msg) override; + + /** + * Called by processParseError() in the downstream case. This function ensures + * that a handler is set for the transaction. + */ + HTTPTransaction::Handler* getParseErrorHandler( + HTTPTransaction* txn, const HTTPException& error) override; + + /** + * Called by transactionTimeout() in the downstream case. This function + * ensures that a handler is set for the transaction. + */ + HTTPTransaction::Handler* getTransactionTimeoutHandler( + HTTPTransaction* txn) override; + + /** + * Invoked when headers have been sent. + */ + void onHeadersSent(const HTTPMessage& headers, + bool codecWasReusable) override; + + bool allTransactionsStarted() const override; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPErrorPage.cpp b/proxygen/lib/http/session/HTTPErrorPage.cpp new file mode 100644 index 0000000000..00aefbf586 --- /dev/null +++ b/proxygen/lib/http/session/HTTPErrorPage.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPErrorPage.h" + +#include + +using std::string; + +namespace proxygen { + +HTTPStaticErrorPage::HTTPStaticErrorPage(std::unique_ptr content, + const string& contentType): + content_(std::move(content)), + contentType_(contentType) { +} + +HTTPErrorPage::Page HTTPStaticErrorPage::generate( + uint64_t requestID, + unsigned httpStatusCode, + const std::string& reason, + std::unique_ptr body, + const std::string& detailReason) const { + + return HTTPErrorPage::Page(contentType_, content_->clone()); +} + +} diff --git a/proxygen/lib/http/session/HTTPErrorPage.h b/proxygen/lib/http/session/HTTPErrorPage.h new file mode 100644 index 0000000000..b1e4becafe --- /dev/null +++ b/proxygen/lib/http/session/HTTPErrorPage.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace folly { +class IOBuf; +} + +namespace proxygen { + +/** + * An HTTPErrorPage generates the content for a web page that + * Proxygen can return in response to various error conditions. + */ +class HTTPErrorPage { +public: + struct Page { + Page(const std::string& pageContentType, + std::unique_ptr pageContent) : + contentType(pageContentType), content(std::move(pageContent)) {} + + Page(Page&& other) noexcept : + contentType(other.contentType), + content(std::move(other.content)) { + } + + const std::string contentType; + std::unique_ptr content; + }; + + virtual ~HTTPErrorPage() {} + + virtual Page generate(uint64_t requestID, + unsigned httpStatusCode, + const std::string& reason, + std::unique_ptr body, + const std::string& detailReason) const = 0; +}; + +/** + * Static error page generator. + */ +class HTTPStaticErrorPage: public HTTPErrorPage { +public: + explicit HTTPStaticErrorPage( + std::unique_ptr content, + const std::string& contentType = "text/html; charset=utf-8"); + + Page generate(uint64_t requestID, + unsigned httpStatusCode, + const std::string& reason, + std::unique_ptr body, + const std::string& detailReason) const override; + +private: + std::unique_ptr content_; + std::string contentType_; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPEvent.cpp b/proxygen/lib/http/session/HTTPEvent.cpp new file mode 100644 index 0000000000..bd3dd93c8b --- /dev/null +++ b/proxygen/lib/http/session/HTTPEvent.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPEvent.h" + +#include + +namespace proxygen { + +std::ostream& operator<<(std::ostream& os, HTTPEvent::Type e) { + switch (e) { + case HTTPEvent::Type::MESSAGE_BEGIN: + os << "message_begin"; + break; + case HTTPEvent::Type::HEADERS_COMPLETE: + os << "headers_complete"; + break; + case HTTPEvent::Type::BODY: + os << "body"; + break; + case HTTPEvent::Type::CHUNK_HEADER: + os << "chunk_header"; + break; + case HTTPEvent::Type::CHUNK_COMPLETE: + os << "chunk_complete"; + break; + case HTTPEvent::Type::TRAILERS_COMPLETE: + os << "trailers_complete"; + break; + case HTTPEvent::Type::MESSAGE_COMPLETE: + os << "message_complete"; + break; + case HTTPEvent::Type::UPGRADE: + os << "uprade"; + break; + } + + return os; +} + +} diff --git a/proxygen/lib/http/session/HTTPEvent.h b/proxygen/lib/http/session/HTTPEvent.h new file mode 100644 index 0000000000..e5745bcbfa --- /dev/null +++ b/proxygen/lib/http/session/HTTPEvent.h @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPConstants.h" +#include "proxygen/lib/http/codec/HTTPCodec.h" + +#include + +namespace proxygen { + +/** + * Helper class that holds some event in the lifecycle + * of an HTTP request or response. + * + * The main use for this class is to queue up events in + * situations where the code handling events isn't able + * to process them and the thing generating events isn't + * able to stop. + */ +class HTTPEvent { + public: + enum class Type: uint8_t { + // Ingress events + MESSAGE_BEGIN, + HEADERS_COMPLETE, + BODY, + CHUNK_HEADER, + CHUNK_COMPLETE, + TRAILERS_COMPLETE, + MESSAGE_COMPLETE, + UPGRADE + }; + + HTTPEvent(HTTPCodec::StreamID streamID, + Type event, bool upgrade = false): + streamID_(streamID), length_(0), event_(event), + upgrade_(upgrade) {} + + HTTPEvent(HTTPCodec::StreamID streamID, + Type event, size_t length): + streamID_(streamID), length_(length), event_(event), + upgrade_(false) { + // This constructor should only be used for CHUNK_HEADER. + // (Ideally we would take the event type as a template parameter + // so we could enforce this check at compile time. Unfortunately, + // that would prevent us from using this constructor with + // deferredCallbacks_.emplace().) + CHECK(event == Type::CHUNK_HEADER); + } + + HTTPEvent(HTTPCodec::StreamID streamID, + Type event, std::unique_ptr headers): + headers_(std::move(headers)), streamID_(streamID), length_(0), + event_(event), upgrade_(false) {} + + HTTPEvent(HTTPCodec::StreamID streamID, + Type event, std::unique_ptr body): + body_(std::move(body)), streamID_(streamID), length_(0), + event_(event), upgrade_(false) {} + + HTTPEvent(HTTPCodec::StreamID streamID, + Type event, std::unique_ptr trailers): + trailers_(std::move(trailers)), streamID_(streamID), length_(0), + event_(event), upgrade_(false) {} + + HTTPEvent(HTTPCodec::StreamID streamID, + Type event, std::unique_ptr error): + error_(std::move(error)), streamID_(streamID), length_(0), + event_(event), upgrade_(false) {} + + HTTPEvent(HTTPCodec::StreamID streamID, + Type event, UpgradeProtocol protocol): + streamID_(streamID), length_(0), event_(event), + upgrade_(false), protocol_(protocol) {} + + Type getEvent() const { return event_; } + + HTTPCodec::StreamID getStreamID() const { return streamID_; } + + std::unique_ptr getHeaders() { return std::move(headers_); } + + std::unique_ptr getBody() { return std::move(body_); } + + std::unique_ptr getError() { return std::move(error_); } + + bool isUpgrade() const { + CHECK(event_ == Type::MESSAGE_COMPLETE); + return upgrade_; + } + + size_t getChunkLength() const { + CHECK(event_ == Type::CHUNK_HEADER); + return length_; + } + + std::unique_ptr getTrailers() { + return std::move(trailers_); + } + + UpgradeProtocol getUpgradeProtocol() { + return protocol_; + } + + private: + std::unique_ptr headers_; + std::unique_ptr body_; + std::unique_ptr trailers_; + std::unique_ptr error_; + HTTPCodec::StreamID streamID_; + size_t length_; // Only valid when event_ == CHUNK_HEADER + Type event_; + bool upgrade_; // Only valid when event_ == MESSAGE_COMPLETE + UpgradeProtocol protocol_; +}; + +std::ostream& operator<<(std::ostream& os, HTTPEvent::Type e); + +} diff --git a/proxygen/lib/http/session/HTTPSession.cpp b/proxygen/lib/http/session/HTTPSession.cpp new file mode 100644 index 0000000000..354538f59e --- /dev/null +++ b/proxygen/lib/http/session/HTTPSession.cpp @@ -0,0 +1,2002 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPSession.h" + +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/codec/HTTPChecks.h" +#include "proxygen/lib/http/session/HTTPSessionController.h" +#include "proxygen/lib/http/session/HTTPSessionStats.h" +#include "proxygen/lib/utils/SocketOptions.h" + +#include +#include +#include +#include + +using apache::thrift::async::TAsyncSSLSocket; +using apache::thrift::async::TAsyncSocket; +using apache::thrift::async::TAsyncTimeoutSet; +using apache::thrift::async::TAsyncTransport; +using apache::thrift::async::WriteFlags; +using apache::thrift::transport::TTransportException; +using folly::IOBuf; +using folly::SocketAddress; +using std::pair; +using std::set; +using std::string; +using std::unique_ptr; +using std::vector; + +namespace { +static const uint32_t kMinReadSize = 1460; +static const uint32_t kMaxReadSize = 4000; + +// Lower = higher latency, better prioritization +// Higher = lower latency, less prioritization +static const uint32_t kMaxWritesPerLoop = 32; + +} // anonymous namespace + +namespace proxygen { + +uint32_t HTTPSession::kDefaultReadBufLimit = 65536; +uint32_t HTTPSession::kPendingWriteMax = 8192; + +HTTPSession::WriteSegment::WriteSegment( + HTTPSession* session, + uint64_t length) + : session_(session), + length_(length) { +} + +void +HTTPSession::WriteSegment::remove() { + DCHECK(session_); + DCHECK(listHook.is_linked()); + listHook.unlink(); +} + +void +HTTPSession::WriteSegment::detach() { + remove(); + session_ = nullptr; +} + +void +HTTPSession::WriteSegment::writeSuccess() noexcept { + // Unlink this write segment from the list before calling + // the session's onWriteSuccess() callback because, in the + // case where this is the last write for the connection, + // onWriteSuccess() looks for an empty write segment list + // as one of the criteria for shutting down the connection. + remove(); + + // session_ should never be nullptr for a successful write + // The session is only cleared after a write error or timeout, and all + // TAsyncTransport write failures are fatal. If session_ is nullptr at this + // point it means the TAsyncTransport implementation is not failing + // subsequent writes correctly after an error. + session_->onWriteSuccess(length_); + delete this; +} + +void +HTTPSession::WriteSegment::writeError(size_t bytesWritten, + const TTransportException& ex) noexcept { + // After one segment fails to write, we clear the session_ + // pointer in all subsequent write segments, so we ignore their + // writeError() callbacks. + if (session_) { + remove(); + session_->onWriteError(bytesWritten, ex); + } + delete this; +} + +HTTPSession::HTTPSession( + TAsyncTimeoutSet* transactionTimeouts, + TAsyncTransport::UniquePtr sock, + const SocketAddress& localAddr, + const SocketAddress& peerAddr, + HTTPSessionController* controller, + unique_ptr codec, + const TransportInfo& tinfo, + InfoCallback* infoCallback): + localAddr_(localAddr), + peerAddr_(peerAddr), + sock_(std::move(sock)), + controller_(controller), + codec_(std::move(codec)), + infoCallback_(infoCallback), + writeTimeout_(this), + transactionTimeouts_(CHECK_NOTNULL(transactionTimeouts)), + transportInfo_(tinfo), + direction_(codec_->getTransportDirection()), + draining_(false), + needsChromeWorkaround_(false), + ingressUpgraded_(false), + started_(false), + readsPaused_(true), + readsShutdown_(false), + writesPaused_(false), + writesShutdown_(false), + writesDraining_(false), + resetAfterDrainingWrites_(false), + ingressError_(false), + inLoopCallback_(false) { + + codec_.add(); + + if (!codec_->supportsParallelRequests()) { + // until we support upstream pipelining + maxConcurrentIncomingStreams_ = 1; + maxConcurrentOutgoingStreamsConfig_ = isDownstream() ? 0 : 1; + } + + HTTPSettings* settings = codec_->getEgressSettings(); + if (settings) { + settings->setSetting(SettingsId::MAX_CONCURRENT_STREAMS, + maxConcurrentIncomingStreams_); + } + if (codec_->supportsSessionFlowControl()) { + connFlowControl_ = new FlowControlFilter(*this, + writeBuf_, + codec_.call(), + kDefaultReadBufLimit); + codec_.addFilters(std::unique_ptr(connFlowControl_)); + } + + if (!codec_->supportsPushTransactions()) { + maxConcurrentPushTransactions_ = 0; + } + + // If we receive IPv4-mapped IPv6 addresses, convert them to IPv4. + localAddr_.tryConvertToIPv4(); + peerAddr_.tryConvertToIPv4(); + + if (infoCallback_) { + infoCallback_->onCreate(*this); + } + codec_.setCallback(this); + + if (controller_) { + controller_->attachSession(this); + } +} + +HTTPSession::~HTTPSession() { + VLOG(4) << *this << " closing"; + + CHECK(transactions_.empty()); + CHECK(txnEgressQueue_.empty()); + DCHECK(!sock_->getReadCallback()); + + if (infoCallback_) { + infoCallback_->onDestroy(*this); + } + if (controller_) { + controller_->detachSession(this); + controller_ = nullptr; + } +} + +void HTTPSession::startNow() { + CHECK(!started_); + started_ = true; + codec_->generateSettings(writeBuf_); + scheduleWrite(); + resumeReads(); +} + +void HTTPSession::setInfoCallback(InfoCallback* cb) { + infoCallback_ = cb; +} + +void HTTPSession::setSessionStats(HTTPSessionStats* stats) { + sessionStats_ = stats; + if (byteEventTracker_) { + byteEventTracker_->setTTLBAStats(stats); + } +} + +void HTTPSession::setFlowControl(size_t initialReceiveWindow, + size_t receiveStreamWindowSize) { + CHECK(!started_); + initialReceiveWindow_ = initialReceiveWindow; + receiveStreamWindowSize_ = receiveStreamWindowSize; + HTTPSettings* settings = codec_->getEgressSettings(); + if (settings) { + settings->setSetting(SettingsId::INITIAL_WINDOW_SIZE, + initialReceiveWindow_); + } +} + +void HTTPSession::setMaxConcurrentOutgoingStreams(uint32_t num) { + CHECK(!started_); + if (codec_->supportsParallelRequests()) { + maxConcurrentOutgoingStreamsConfig_ = num; + } +} + +void HTTPSession::setMaxConcurrentPushTransactions(uint32_t num) { + CHECK(!started_); + if (codec_->supportsPushTransactions()) { + maxConcurrentPushTransactions_ = num; + } +} + +void +HTTPSession::readTimeoutExpired() noexcept { + VLOG(3) << "session-level timeout on " << *this; + + if (liveTransactions_ != 0) { + // There's at least one open transaction with a read timeout scheduled. + // We got here because the session timeout == the transaction timeout. + // Ignore, since the transaction is going to timeout very soon. + VLOG(4) << *this << + "ignoring session timeout, transaction timeout imminent"; + resetTimeout(); + return; + } + + if (!transactions_.empty()) { + // There are one or more transactions, but none of them are live. + // That's valid if they've all received their full ingress messages + // and are waiting for their Handlers to process those messages. + VLOG(4) << *this << + "ignoring session timeout, no transactions awaiting reads"; + resetTimeout(); + return; + } + + VLOG(4) << *this << " Timeout with nothing pending"; + + setCloseReason(ConnectionCloseReason::TIMEOUT); + shutdownTransport(true, true); +} + +void +HTTPSession::writeTimeoutExpired() noexcept { + VLOG(4) << "Write timeout for " << *this; + + CHECK(!pendingWrites_.empty()); + DestructorGuard g(this); + + setCloseReason(ConnectionCloseReason::TIMEOUT); + shutdownTransportWithReset(kErrorWriteTimeout); +} + +void +HTTPSession::describe(std::ostream& os) const { + if (isDownstream()) { + os << "[downstream = " << peerAddr_ << ", " << localAddr_ << " = local]"; + } else { + os << "[local = " << localAddr_ << ", " << peerAddr_ << " = upstream]"; + } +} + +bool +HTTPSession::isBusy() const { + return !transactions_.empty() || codec_->isBusy(); +} + +void HTTPSession::notifyPendingEgress() noexcept { + scheduleWrite(); +} + +void +HTTPSession::notifyPendingShutdown() { + VLOG(4) << *this << " notified pending shutdown"; + drain(); +} + +void +HTTPSession::closeWhenIdle() { + // If drain() already called, this is a noop + drain(); + // Generate the second GOAWAY now. No-op if second GOAWAY already sent. + if (codec_->generateGoaway(writeBuf_, + codec_->getLastIncomingStreamID(), + ErrorCode::NO_ERROR)) { + scheduleWrite(); + } + if (!isBusy() && !hasMoreWrites()) { + // if we're already idle, close now + dropConnection(); + } +} + +void +HTTPSession::dropConnection() { + VLOG(4) << "dropping " << *this; + if (!sock_ || (readsShutdown_ && writesShutdown_)) { + VLOG(4) << *this << " already shutdown"; + return; + } + + setCloseReason(ConnectionCloseReason::SHUTDOWN); + if (transactions_.empty() && !hasMoreWrites()) { + shutdownTransport(true, true); + } else { + shutdownTransportWithReset(kErrorDropped); + } +} + +void +HTTPSession::dumpConnectionState(uint8_t loglevel) { +} + +bool HTTPSession::isUpstream() const { + return direction_ == TransportDirection::UPSTREAM; +} + +bool HTTPSession::isDownstream() const { + return direction_ == TransportDirection::DOWNSTREAM; +} + +void +HTTPSession::getReadBuffer(void** buf, size_t* bufSize) { + pair readSpace = readBuf_.preallocate(kMinReadSize, + kMaxReadSize); + *buf = readSpace.first; + *bufSize = readSpace.second; +} + +void +HTTPSession::readDataAvailable(size_t readSize) noexcept { + VLOG(10) << "read completed on " << *this << ", bytes=" << readSize; + + DestructorGuard dg(this); + resetTimeout(); + readBuf_.postallocate(readSize); + + if (infoCallback_) { + infoCallback_->onRead(*this, readSize); + } + + processReadData(); +} + +void +HTTPSession::processReadData() { + // Pass the ingress data through the codec to parse it. The codec + // will invoke various methods of the HTTPSession as callbacks. + const IOBuf* currentReadBuf; + // It's possible for the last buffer in a chain to be empty here. + // TAsyncTransport saw fd activity so asked for a read buffer, but it was + // SSL traffic and not enough to decrypt a whole record. Later we invoke + // this function from the loop callback. + while (!ingressError_ && + !readsPaused_ && + !readsShutdown_ && + ((currentReadBuf = readBuf_.front()) != nullptr && + currentReadBuf->length() != 0)) { + // We're about to parse, make sure the parser is not paused + codec_->setParserPaused(false); + size_t bytesParsed = codec_->onIngress(*currentReadBuf); + if (bytesParsed == 0) { + // If the codec didn't make any progress with current input, we + // better get more. + break; + } + readBuf_.trimStart(bytesParsed); + } +} + +void +HTTPSession::readEOF() noexcept { + DestructorGuard guard(this); + VLOG(4) << "EOF on " << *this; + // for SSL only: error without any bytes from the client might happen + // due to client-side issues with the SSL cert. Note that it can also + // happen if the client sends a SPDY frame header but no body. + if (infoCallback_ + && transportInfo_.ssl && transactionSeqNo_ == 0 && readBuf_.empty()) { + infoCallback_->onIngressError(*this, kErrorClientSilent); + } + + // Shut down reads, and also shut down writes if there are no + // transactions. (If there are active transactions, leave the + // write side of the socket open so those transactions can + // finish generating responses.) + setCloseReason(ConnectionCloseReason::READ_EOF); + shutdownTransport(true, transactions_.empty()); +} + +void +HTTPSession::readError( + const TTransportException& ex) noexcept { + DestructorGuard guard(this); + VLOG(4) << "read error on " << *this << ": " << ex.what(); + if (infoCallback_ && ( + ERR_GET_LIB(ex.getErrno()) == ERR_LIB_USER && + ERR_GET_REASON(ex.getErrno()) == + (int)TAsyncSSLSocket::SSL_CLIENT_RENEGOTIATION_ATTEMPT)) { + infoCallback_->onIngressError(*this, kErrorClientRenegotiation); + } + + // We're definitely finished reading. Don't close the write side + // of the socket if there are outstanding transactions, though. + // Instead, give the transactions a chance to produce any remaining + // output. + if (ERR_GET_LIB(ex.getErrno()) == ERR_LIB_SSL) { + transportInfo_.sslError = ERR_GET_REASON(ex.getErrno()); + } + setCloseReason(ConnectionCloseReason::IO_READ_ERROR); + shutdownTransport(true, transactions_.empty()); +} + +HTTPTransaction* +HTTPSession::newPushedTransaction(HTTPCodec::StreamID assocStreamId, + HTTPTransaction::PushHandler* handler, + int8_t priority) noexcept { + if (!codec_->supportsPushTransactions()) { + return nullptr; + } + CHECK(isDownstream()); + CHECK_NOTNULL(handler); + if (draining_ || (pushedTxns_ >= maxConcurrentPushTransactions_)) { + // This session doesn't support any more push transactions + // This could be an actual problem - since a single downstream SPDY session + // might be connected to N upstream hosts, each of which send M pushes, + // which exceeds the limit. + // should we queue? + return nullptr; + } + + HTTPCodec::StreamID streamID = codec_->createStream(); + HTTPTransaction* txn = new HTTPTransaction( + direction_, streamID, transactionSeqNo_, *this, + txnEgressQueue_, transactionTimeouts_, sessionStats_, + codec_->supportsStreamFlowControl(), + initialReceiveWindow_, + getCodecSendWindowSize(), + priority, assocStreamId); + + if (!addTransaction(txn)) { + delete txn; + return nullptr; + } + + transactionSeqNo_++; + + txn->setReceiveWindow(receiveStreamWindowSize_); + txn->setHandler(handler); + setNewTransactionPauseState(txn); + return txn; +} + +size_t HTTPSession::getCodecSendWindowSize() const { + const HTTPSettings* settings = codec_->getIngressSettings(); + if (settings) { + return settings->getSetting(SettingsId::INITIAL_WINDOW_SIZE, 65536); + } + return 65536; +} + +void +HTTPSession::setNewTransactionPauseState(HTTPTransaction* txn) { + if (writesPaused_) { + // If writes are paused, start this txn off in the egress paused state + VLOG(4) << *this << " starting streamID=" << txn->getID() + << " egress paused. pendingWriteSize_=" << pendingWriteSize_ + << ", numActiveWrites_=" << numActiveWrites_ + << ", kPendingWriteMax=" << kPendingWriteMax; + txn->pauseEgress(); + } +} + +void +HTTPSession::onMessageBegin(HTTPCodec::StreamID streamID, HTTPMessage* msg) { + onMessageBeginImpl(streamID, 0, msg); +} + +void +HTTPSession::onPushMessageBegin(HTTPCodec::StreamID streamID, + HTTPCodec::StreamID assocStreamID, + HTTPMessage* msg) { + onMessageBeginImpl(streamID, assocStreamID, msg); +} + +HTTPTransaction* +HTTPSession::onMessageBeginImpl(HTTPCodec::StreamID streamID, + HTTPCodec::StreamID assocStreamID, + HTTPMessage* msg) { + VLOG(4) << "processing new message on " << *this << ", streamID=" << streamID; + if (infoCallback_) { + infoCallback_->onRequestBegin(*this); + } + auto txn = findTransaction(streamID); + if (txn) { + if (isDownstream() && txn->isPushed()) { + // Push streams are unidirectional (half-closed). If the downstream + // attempts to send ingress, abort with STREAM_CLOSED error. + HTTPException ex(HTTPException::Direction::INGRESS_AND_EGRESS); + ex.setCodecStatusCode(ErrorCode::STREAM_CLOSED); + txn->onError(ex); + } + // If this transaction is already registered, no need to add it now + return txn; + } + + HTTPTransaction* assocStream = nullptr; + if (assocStreamID > 0) { + assocStream = findTransaction(assocStreamID); + if (!assocStream || assocStream->isIngressEOMSeen()) { + VLOG(1) << "Can't find assoc txn=" << assocStreamID << ", or assoc txn " + "cannot push"; + invalidStream(streamID, ErrorCode::PROTOCOL_ERROR); + return nullptr; + } + } + + bool useFlowControl = codec_->supportsStreamFlowControl(); + txn = new HTTPTransaction( + direction_, streamID, transactionSeqNo_, *this, + txnEgressQueue_, transactionTimeouts_, sessionStats_, + useFlowControl, + initialReceiveWindow_, + getCodecSendWindowSize(), + msg ? msg->getPriority() : 0, assocStreamID); + txn->setReceiveWindow(receiveStreamWindowSize_); + if (assocStream && !assocStream->onPushedTransaction(txn)) { + VLOG(1) << "Failed to add pushed transaction " << streamID << " on " + << *this; + delete txn; + txn = nullptr; + } + if (txn && !addTransaction(txn)) { + VLOG(1) << "Cannot add stream ID " << streamID << " on " << *this; + delete txn; + txn = nullptr; + } + if (!txn) { + HTTPException ex(HTTPException::Direction::INGRESS_AND_EGRESS); + ex.setCodecStatusCode(ErrorCode::REFUSED_STREAM); + onError(streamID, ex, true); + return nullptr; + } + // move the sequence number index + ++transactionSeqNo_; + + if (!codec_->supportsParallelRequests() && transactions_.size() > 1) { + // The previous transaction hasn't completed yet. Pause reads until + // it completes; this requires pausing both transactions. + DCHECK(transactions_.size() == 2); + auto prevTxn = transactions_.begin()->second; + if (!prevTxn->isIngressPaused()) { + DCHECK(prevTxn->isIngressComplete()); + prevTxn->pauseIngress(); + } + DCHECK(liveTransactions_ == 1); + txn->pauseIngress(); + } + return txn; +} + +void +HTTPSession::onHeadersComplete(HTTPCodec::StreamID streamID, + unique_ptr msg) { + // The codec's parser detected the end of an ingress message's + // headers. + VLOG(4) << "processing ingress headers complete for " << *this << + ", streamID=" << streamID; + + if (!codec_->isReusable()) { + setCloseReason(ConnectionCloseReason::REQ_NOTREUSABLE); + } + + if (infoCallback_) { + infoCallback_->onIngressMessage(*this, *msg.get()); + } + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + invalidStream(streamID); + return; + } + msg->setSecureInfo(transportInfo_.sslVersion, transportInfo_.sslCipher); + msg->setSecure(transportInfo_.ssl); + + // TODO: remove this once the percent of Chrome < 28 traffic is less + // than 0.1% + if (txn->getSequenceNumber() == 0 && codec_->supportsParallelRequests()) { + auto& agent = msg->getHeaders().getSingleOrEmpty(HTTP_HEADER_USER_AGENT); + static const string search = "Chrome/"; + auto found = agent.find(search); + VLOG(4) << "The agent is " << agent << " and found is " << found; + if (found != string::npos) { + auto startNum = found + search.length(); + // Versions of Chrome under 28 need this workaround + if (agent.length() > startNum + 3 && + ((agent[startNum] == '1' && agent[startNum + 1] >= '0' && + agent[startNum + 1] <= '9') || + (agent[startNum] == '2' && agent[startNum + 1] >= '0' && + agent[startNum + 1] < '8')) && agent[startNum + 2] == '.') { + VLOG(4) << *this << " Using chrome spdy GOAWAY workaround"; + needsChromeWorkaround_ = true; + } + } + } + + setupOnHeadersComplete(txn, msg.get()); + + // The txn may have already been aborted by the handler. + // Verify that the txn still exists before ingress callbacks. + txn = findTransaction(streamID); + if (!txn) { + return; + } + + if (!txn->getHandler()) { + txn->sendAbort(); + return; + } + + // Tell the Transaction to start processing the message now + // that the full ingress headers have arrived. + txn->onIngressHeadersComplete(std::move(msg)); +} + +void +HTTPSession::onBody(HTTPCodec::StreamID streamID, + unique_ptr chain) { + DestructorGuard dg(this); + // The codec's parser detected part of the ingress message's + // entity-body. + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + invalidStream(streamID); + return; + } + auto oldSize = pendingReadSize_; + pendingReadSize_ += chain->computeChainDataLength(); + txn->onIngressBody(std::move(chain)); + if (oldSize < pendingReadSize_) { + // Transaction must have buffered something and not called + // notifyBodyProcessed() on it. + VLOG(4) << *this << " Enqueued ingress. Ingress buffer uses " + << pendingReadSize_ << " of " << kDefaultReadBufLimit + << " bytes."; + if (pendingReadSize_ > kDefaultReadBufLimit && + oldSize <= kDefaultReadBufLimit) { + VLOG(4) << *this << " pausing due to read limit exceeded."; + if (infoCallback_) { + infoCallback_->onIngressLimitExceeded(*this); + } + pauseReads(); + } + } +} + +void HTTPSession::onChunkHeader(HTTPCodec::StreamID streamID, + size_t length) { + // The codec's parser detected a chunk header (meaning that this + // connection probably is HTTP/1.1). + // + // After calling onChunkHeader(), the codec will call onBody() zero + // or more times and then call onChunkComplete(). + // + // The reason for this callback on the chunk header is to support + // an optimization. In general, the job of the codec is to present + // the HTTPSession with an abstract view of a message, + // with all the details of wire formatting hidden. However, there's + // one important case where we want to know about chunking: reverse + // proxying where both the client and server streams are HTTP/1.1. + // In that scenario, we preserve the server's chunk boundaries when + // sending the response to the client, in order to avoid possibly + // making the egress packetization worse by rechunking. + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + invalidStream(streamID); + return; + } + txn->onIngressChunkHeader(length); +} + +void HTTPSession::onChunkComplete(HTTPCodec::StreamID streamID) { + // The codec's parser detected the end of the message body chunk + // associated with the most recent call to onChunkHeader(). + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + invalidStream(streamID); + return; + } + txn->onIngressChunkComplete(); +} + +void +HTTPSession::onTrailersComplete(HTTPCodec::StreamID streamID, + unique_ptr trailers) { + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + invalidStream(streamID); + return; + } + txn->onIngressTrailers(std::move(trailers)); +} + +void +HTTPSession::onMessageComplete(HTTPCodec::StreamID streamID, + bool upgrade) { + DestructorGuard dg(this); + // The codec's parser detected the end of the ingress message for + // this transaction. + VLOG(4) << "processing ingress message complete for " << *this << + ", streamID=" << streamID; + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + invalidStream(streamID); + return; + } + + if (upgrade && !codec_->supportsParallelRequests()) { + /* Send the upgrade callback to the transaction and the handler. + * Currently we support upgrades for only HTTP sessions and not SPDY + * sessions. + */ + ingressUpgraded_ = true; + txn->onIngressUpgrade(UpgradeProtocol::TCP); + return; + } + + // txnIngressFinished = !1xx response + const bool txnIngressFinished = + txn->isDownstream() || !txn->extraResponseExpected(); + if (txnIngressFinished) { + decrementTransactionCount(txn, true, false); + } + txn->onIngressEOM(); + + // The codec knows, based on the semantics of whatever protocol it + // supports, whether it's valid for any more ingress messages to arrive + // after this one. For example, an HTTP/1.1 request containing + // "Connection: close" indicates the end of the ingress, whereas a + // SPDY session generally can handle more messages at any time. + // + // If the connection is not reusable, we close the read side of it + // but not the write side. There are two reasons why more writes + // may occur after this point: + // * If there are previous writes buffered up in the pendingWrites_ + // queue, we need to attempt to complete them. + // * The Handler associated with the transaction may want to + // produce more egress data when the ingress message is fully + // complete. (As a common example, an application that handles + // form POSTs may not be able to even start generating a response + // until it has received the full request body.) + // + // There may be additional checks that need to be performed that are + // specific to requests or responses, so we call the subclass too. + if (!codec_->isReusable() && + txnIngressFinished && + !codec_->supportsParallelRequests()) { + VLOG(4) << *this << " cannot reuse ingress"; + shutdownTransport(true, false); + } +} + +void HTTPSession::onError(HTTPCodec::StreamID streamID, + const HTTPException& error, bool newTxn) { + // The codec detected an error in the ingress stream, possibly bad + // syntax, a truncated message, or bad semantics in the frame. If reads + // are paused, queue up the event; otherwise, process it now. + VLOG(4) << "Error on " << *this << ", streamID=" << streamID + << ", " << error; + + if (ingressError_) { + return; + } + if (!codec_->supportsParallelRequests()) { + // this error should only prevent us from reading/handling more errors + // on serial streams + ingressError_ = true; + } + if ((streamID == 0) && infoCallback_) { + infoCallback_->onIngressError(*this, kErrorMessage); + } + + if (!streamID) { + ingressError_ = true; + onSessionParseError(error); + return; + } + + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + if (error.hasHttpStatusCode() && streamID != 0) { + // If the error has an HTTP code, then parsing was fine, it just was + // illegal in a higher level way + txn = onMessageBeginImpl(streamID, 0, nullptr); + if (txn) { + handleErrorDirectly(txn, error); + } + } else if (newTxn) { + onNewTransactionParseError(streamID, error); + } else { + VLOG(4) << *this << " parse error with invalid transaction"; + invalidStream(streamID); + } + return; + } + + if (!txn->getHandler() && + txn->getEgressState() == HTTPTransactionEgressSM::State::Start) { + handleErrorDirectly(txn, error); + return; + } + + txn->onError(error); +} + +void HTTPSession::onAbort(HTTPCodec::StreamID streamID, + ErrorCode code) { + VLOG(4) << "stream abort on " << *this << ", streamID=" << streamID + << ", code=" << getErrorCodeString(code); + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + VLOG(4) << *this << " abort for unrecognized transaction, streamID= " + << streamID; + return; + } + HTTPException ex(HTTPException::Direction::INGRESS_AND_EGRESS); + ex.setProxygenError(kErrorStreamAbort); + DestructorGuard dg(this); + if (isDownstream() && txn->getAssocTxnId() == 0 && + code == ErrorCode::CANCEL) { + // Cancelling the assoc txn cancels all push txns + for (auto pushTxnId : txn->getPushedTransactions()) { + auto pushTxn = findTransaction(pushTxnId); + DCHECK(pushTxn != nullptr); + pushTxn->onError(ex); + } + } + txn->onError(ex); +} + +void HTTPSession::onGoaway(uint64_t lastGoodStreamID, + ErrorCode code) { + DestructorGuard g(this); + VLOG(4) << "GOAWAY on " << *this << ", code=" << getErrorCodeString(code); + + setCloseReason(ConnectionCloseReason::GOAWAY); + + // Drain active transactions and prevent new transactions + drain(); + + // Abort transactions which have been initiated but not created + // successfully at the remote end. Upstream transactions are created + // with odd transaction IDs and downstream transactions with even IDs. + vector ids; + for (auto txn: transactions_) { + auto streamID = txn.first; + if (((bool)(streamID & 0x01) == isUpstream()) && + (streamID > lastGoodStreamID)) { + ids.push_back(streamID); + } + } + errorOnTransactionIds(ids, kErrorStreamUnacknowledged); +} + +void HTTPSession::onPingRequest(uint64_t uniqueID) { + VLOG(4) << *this << " got ping request with id=" << uniqueID; + + TimePoint timestamp = getCurrentTime(); + + // Insert the ping reply to the head of writeBuf_ + folly::IOBufQueue pingBuf(folly::IOBufQueue::cacheChainLength()); + codec_->generatePingReply(pingBuf, uniqueID); + size_t pingSize = pingBuf.chainLength(); + pingBuf.append(writeBuf_.move()); + writeBuf_.append(pingBuf.move()); + + if (byteEventTracker_) { + byteEventTracker_->addPingByteEvent(pingSize, timestamp, bytesScheduled_); + } + + scheduleWrite(); +} + +void HTTPSession::onPingReply(uint64_t uniqueID) { + VLOG(4) << *this << " got ping reply with id=" << uniqueID; +} + +void HTTPSession::onWindowUpdate(HTTPCodec::StreamID streamID, + uint32_t amount) { + VLOG(4) << *this << " got window update on streamID=" << streamID << " for " + << amount << " bytes."; + HTTPTransaction* txn = findTransaction(streamID); + if (!txn) { + // We MUST be using SPDY/3+ if we got WINDOW_UPDATE. The spec says that - + // + // A sender should ignore all the WINDOW_UPDATE frames associated with the + // stream after it send the last frame for the stream. + // + // TODO: Only ignore if this is from some past transaction + return; + } + txn->onIngressWindowUpdate(amount); +} + +void HTTPSession::onSettings(const SettingsList& settings) { + for (auto& setting: settings) { + if (setting.id == SettingsId::INITIAL_WINDOW_SIZE) { + onSetSendWindow(setting.value); + } else if (setting.id == SettingsId::MAX_CONCURRENT_STREAMS) { + onSetMaxInitiatedStreams(setting.value); + } + } +} + +void HTTPSession::onSetSendWindow(uint32_t windowSize) { + VLOG(4) << *this << " got send window size adjustment. new=" << windowSize; + invokeOnAllTransactions(&HTTPTransaction::onIngressSetSendWindow, + windowSize); +} + +void HTTPSession::onSetMaxInitiatedStreams(uint32_t maxTxns) { + VLOG(4) << *this << " got new maximum number of concurrent txns " + << "we can initiate: " << maxTxns; + const bool didSupport = supportsMoreTransactions(); + maxConcurrentOutgoingStreamsRemote_ = maxTxns; + if (infoCallback_ && didSupport != supportsMoreTransactions()) { + if (didSupport) { + infoCallback_->onSettingsOutgoingStreamsFull(*this); + } else { + infoCallback_->onSettingsOutgoingStreamsNotFull(*this); + } + } +} + +void HTTPSession::pauseIngress(HTTPTransaction* txn) noexcept { + VLOG(4) << *this << " pausing streamID=" << txn->getID() << + ", liveTransactions_ was " << liveTransactions_; + CHECK(liveTransactions_ > 0); + --liveTransactions_; + if (liveTransactions_ == 0) { + pauseReads(); + } +} + +void HTTPSession::resumeIngress(HTTPTransaction* txn) noexcept { + VLOG(4) << *this << " resuming streamID=" << txn->getID() << + ", liveTransactions_ was " << liveTransactions_; + ++liveTransactions_; + if (liveTransactions_ == 1) { + resumeReads(); + } +} + +void +HTTPSession::transactionTimeout(HTTPTransaction* txn) noexcept { + // A transaction has timed out. If the transaction does not have + // a Handler yet, because we haven't yet received the full request + // headers, we give it a DirectResponseHandler that generates an + // error page. + VLOG(3) << "Transaction timeout for streamID=" << txn->getID(); + if (!codec_->supportsParallelRequests()) { + // this error should only prevent us from reading/handling more errors + // on serial streams + ingressError_ = true; + } + + if (!txn->getHandler() && + txn->getEgressState() == HTTPTransactionEgressSM::State::Start) { + VLOG(4) << *this << " creating direct error handler"; + auto handler = getTransactionTimeoutHandler(txn); + txn->setHandler(handler); + if (infoCallback_) { + infoCallback_->onIngressError(*this, kErrorTimeout); + } + } + + // Tell the transaction about the timeout. The transaction will + // communicate the timeout to the handler, and the handler will + // decide how to proceed. + txn->onIngressTimeout(); +} + +void HTTPSession::sendHeaders(HTTPTransaction* txn, + const HTTPMessage& headers, + HTTPHeaderSize* size) noexcept { + CHECK(started_); + if (shouldShutdown()) { + // For HTTP/1.1, add Connection: close + drainImpl(); + } + const bool wasReusable = codec_->isReusable(); + const uint64_t oldOffset = sessionByteOffset(); + codec_->generateHeader(writeBuf_, + txn->getID(), + headers, + txn->getAssocTxnId(), + size); + const uint64_t newOffset = sessionByteOffset(); + + // only do it for downstream now to bypass handling upstream reuse cases + if (isDownstream() && + newOffset > oldOffset && + // catch 100-ish response? + !txn->testAndSetFirstHeaderByteSent() && byteEventTracker_) { + byteEventTracker_->addFirstHeaderByteEvent(newOffset, txn); + } + + if (size) { + VLOG(4) << *this << " sending headers, size=" << size->compressed + << ", uncompressedSize=" << size->uncompressed; + } + scheduleWrite(); + onHeadersSent(headers, wasReusable); +} + +size_t +HTTPSession::sendBody(HTTPTransaction* txn, + std::unique_ptr body, + bool includeEOM) noexcept { + uint64_t offset = sessionByteOffset(); + size_t encodedSize = codec_->generateBody(writeBuf_, + txn->getID(), + std::move(body), + includeEOM); + if (encodedSize > 0 && !txn->testAndSetFirstByteSent() && byteEventTracker_) { + byteEventTracker_->addFirstBodyByteEvent(offset, txn); + } + if (includeEOM) { + if (!txn->testAndSetFirstByteSent()) { + txn->onEgressBodyFirstByte(); + } + if (encodedSize > 0 && byteEventTracker_) { + byteEventTracker_->addLastByteEvent(txn, sessionByteOffset(), + sock_->isEorTrackingEnabled()); + } + + VLOG(4) << *this << " sending EOM in body for streamID=" << txn->getID(); + onEgressMessageFinished(txn); + } + return encodedSize; +} + +size_t HTTPSession::sendChunkHeader(HTTPTransaction* txn, + size_t length) noexcept { + size_t encodedSize = codec_->generateChunkHeader(writeBuf_, + txn->getID(), + length); + scheduleWrite(); + return encodedSize; +} + +size_t HTTPSession::sendChunkTerminator( + HTTPTransaction* txn) noexcept { + size_t encodedSize = codec_->generateChunkTerminator(writeBuf_, + txn->getID()); + scheduleWrite(); + return encodedSize; +} + +size_t +HTTPSession::sendTrailers(HTTPTransaction* txn, + const HTTPHeaders& trailers) noexcept { + size_t encodedSize = codec_->generateTrailers(writeBuf_, + txn->getID(), + trailers); + scheduleWrite(); + return encodedSize; +} + +void +HTTPSession::onEgressMessageFinished(HTTPTransaction* txn, bool withRST) { + // If the semantics of the protocol don't permit more messages + // to be read or sent on this connection, close the socket in one or + // more directions. + CHECK(!transactions_.empty()); + if (infoCallback_) { + infoCallback_->onRequestEnd(*this, txn->getMaxDeferredSize()); + } + decrementTransactionCount(txn, false, true); + if (withRST || ((!codec_->isReusable() || readsShutdown_) && + transactions_.size() == 1)) { + // We should shutdown reads if we are closing with RST or we aren't + // interested in any further messages (ie if we are a downstream session). + // Upgraded sessions have independent ingress and egress, and the reads + // need not be shutdown on egress finish. + if (withRST) { + // Let any queued writes complete, but send a RST when done. + VLOG(4) << *this << " resetting egress after this message"; + resetAfterDrainingWrites_ = true; + setCloseReason(ConnectionCloseReason::TRANSACTION_ABORT); + shutdownTransport(true, true); + } else { + // the reason is already set (either not reusable or readshutdown). + + // Defer normal shutdowns until the end of the loop. This + // handles an edge case with direct responses with Connection: + // close served before ingress EOM. The remainder of the ingress + // message may be in the parse loop, so give it a chance to + // finish out and avoid a kErrorEOF + + // Just for safety, bump the refcount on this session to keep it + // live until the loopCb runs + auto dg = new DestructorGuard(this); + sock_->getEventBase()->runInLoop([this, dg] { + VLOG(4) << *this << " shutdown from onEgressMessageFinished"; + bool shutdownReads = isDownstream() && !ingressUpgraded_; + shutdownTransport(shutdownReads, true); + delete dg; + }, true); + } + } +} + +size_t +HTTPSession::sendEOM(HTTPTransaction* txn) noexcept { + // Ask the codec to generate an end-of-message indicator for the + // transaction. Depending on the protocol, this may be a no-op. + // Schedule a network write to send out whatever egress we might + // have queued up. + VLOG(4) << *this << " sending EOM for streamID=" << txn->getID(); + size_t encodedSize = codec_->generateEOM(writeBuf_, txn->getID()); + // PRIO_TODO: boost this transaction's priority? evaluate impact... + if (!txn->testAndSetFirstByteSent()) { + txn->onEgressBodyFirstByte(); + } + txn->onEgressBodyLastByte(); + if (encodedSize > 0 && byteEventTracker_) { + byteEventTracker_->addLastByteEvent(txn, sessionByteOffset(), + sock_->isEorTrackingEnabled()); + } + // in case encodedSize == 0 we won't get TTLBA which is acceptable + // noting the fact that we don't have a response body + + onEgressMessageFinished(txn); + return encodedSize; +} + +size_t HTTPSession::sendAbort(HTTPTransaction* txn, + ErrorCode statusCode) noexcept { + // Ask the codec to generate an abort indicator for the transaction. + // Depending on the protocol, this may be a no-op. + // Schedule a network write to send out whatever egress we might + // have queued up. + VLOG(4) << *this << " sending abort for streamID=" << txn->getID(); + // drain this transaction's writeBuf instead of flushing it + // then enqueue the abort directly into the Session buffer, + // hence with max priority. + size_t encodedSize = codec_->generateRstStream(writeBuf_, + txn->getID(), + statusCode); + + if (!codec_->isReusable()) { + // HTTP 1x codec does not support per stream abort so this will + // render the codec not reusable + setCloseReason(ConnectionCloseReason::TRANSACTION_ABORT); + } + + scheduleWrite(); + + // If the codec wasn't able to write a L7 message for the abort, then + // fall back to closing the transport with a TCP level RST + onEgressMessageFinished(txn, !encodedSize); + return encodedSize; +} + +void +HTTPSession::decrementTransactionCount(HTTPTransaction* txn, + bool ingressEOM, bool egressEOM) { + if ((isUpstream() && !txn->isPushed()) || + (isDownstream() && txn->isPushed())) { + if (ingressEOM && txn->testAndClearActive()) { + outgoingStreams_--; + } + } else { + if (egressEOM && txn->testAndClearActive()) { + incomingStreams_--; + } + } +} + +void +HTTPSession::detach(HTTPTransaction* txn) noexcept { + DestructorGuard guard(this); + HTTPCodec::StreamID streamID = txn->getID(); + auto it = transactions_.find(txn->getID()); + DCHECK(it != transactions_.end()); + if (!txn->isIngressPaused()) { + VLOG(4) << *this << " removing streamID=" << streamID << + ", liveTransactions was " << liveTransactions_; + CHECK(liveTransactions_ > 0); + liveTransactions_--; + } else { + VLOG(4) << *this << " removing streamID=" << streamID; + } + if (txn->isPushed()) { + CHECK(pushedTxns_ > 0); + pushedTxns_--; + auto assocTxn = findTransaction(txn->getAssocTxnId()); + if (assocTxn) { + assocTxn->removePushedTransaction(streamID); + } + } + transactions_.erase(it); + decrementTransactionCount(txn, true, true); + if (infoCallback_) { + if (transactions_.empty()) { + infoCallback_->onDeactivateConnection(*this); + } else { + infoCallback_->onTransactionDetached(*this); + } + } + if (!readsShutdown_) { + if (!codec_->supportsParallelRequests() && !transactions_.empty()) { + // If we had more than one transaction, then someone tried to pipeline and + // we paused reads + DCHECK(transactions_.size() == 1); + auto next_txn = transactions_.begin()->second; + DCHECK(next_txn->isIngressPaused()); + DCHECK(!next_txn->isIngressComplete()); + next_txn->resumeIngress(); + return; + } else { + // this will resume reads if they were paused (eg: 0 HTTP transactions) + resumeReads(); + } + } + + if (liveTransactions_ == 0 && transactions_.empty() && !isScheduled()) { + resetTimeout(); + } + + if (transactions_.empty() && + draining_ && + allTransactionsStarted() && + needsChromeWorkaround_ && + !writesShutdown_ && + codec_->isReusable()) { + VLOG(4) << *this << "Writing out delayed abort for Chrome workaround"; + codec_->generateGoaway(writeBuf_, + codec_->getLastIncomingStreamID(), + ErrorCode::NO_ERROR); + scheduleWrite(); + return; + } + + // It's possible that this is the last transaction in the session, + // so check whether the conditions for shutdown are satisfied. + if (transactions_.empty()) { + if (shouldShutdown()) { + writesDraining_ = true; + } + // Handle the case where we are draining writes but all remaining + // transactions terminated with no egress. + if (writesDraining_ && !writesShutdown_ && !hasMoreWrites()) { + shutdownTransport(false, true); + return; + } + } + checkForShutdown(); +} + +size_t +HTTPSession::sendWindowUpdate(HTTPTransaction* txn, + uint32_t bytes) noexcept { + size_t sent = codec_->generateWindowUpdate(writeBuf_, txn->getID(), bytes); + if (sent) { + scheduleWrite(); + } + return sent; +} + +void +HTTPSession::notifyIngressBodyProcessed(uint32_t bytes) noexcept { + CHECK(pendingReadSize_ >= bytes); + auto oldSize = pendingReadSize_; + pendingReadSize_ -= bytes; + VLOG(4) << *this << " Dequeued " << bytes << " bytes of ingress. " + << "Ingress buffer uses " << pendingReadSize_ << " of " + << kDefaultReadBufLimit << " bytes."; + if (connFlowControl_ && + connFlowControl_->ingressBytesProcessed(writeBuf_, bytes)) { + scheduleWrite(); + } + if (oldSize > kDefaultReadBufLimit && + pendingReadSize_ <= kDefaultReadBufLimit) { + resumeReads(); + } +} + +const SocketAddress& HTTPSession::getLocalAddress() const noexcept { + return localAddr_; +} + +const SocketAddress& HTTPSession::getPeerAddress() const noexcept { + return peerAddr_; +} + +TransportInfo& HTTPSession::getSetupTransportInfo() noexcept { + return transportInfo_; +} + +const TransportInfo& HTTPSession::getSetupTransportInfo() const noexcept { + return transportInfo_; +} + +bool HTTPSession::getCurrentTransportInfo(TransportInfo* tinfo) { + TAsyncSocket* sock = dynamic_cast(sock_.get()); + if (sock) { + tinfo->initWithSocket(sock); + // some fields are the same with the setup transport info + tinfo->setupTime = transportInfo_.setupTime; + tinfo->ssl = transportInfo_.ssl; + tinfo->sslSetupTime = transportInfo_.sslSetupTime; + tinfo->sslVersion = transportInfo_.sslVersion; + tinfo->sslCipher = transportInfo_.sslCipher; + tinfo->sslResume = transportInfo_.sslResume; + tinfo->sslNextProtocol = transportInfo_.sslNextProtocol; + tinfo->sslError = transportInfo_.sslError; +#if defined(__linux__) || defined(__FreeBSD__) + // update connection transport info with the latest RTT + if (tinfo->tcpinfo.tcpi_rtt > 0) { + transportInfo_.tcpinfo.tcpi_rtt = tinfo->tcpinfo.tcpi_rtt; + transportInfo_.rtt = std::chrono::microseconds(tinfo->tcpinfo.tcpi_rtt); + } +#endif + return true; + } + return false; +} + +void HTTPSession::setByteEventTracker( + std::unique_ptr byteEventTracker) { + byteEventTracker_ = std::move(byteEventTracker); + byteEventTracker_->setCallback(this); + byteEventTracker_->setTTLBAStats(sessionStats_); +} + +unique_ptr HTTPSession::getNextToSend(bool* cork, bool* eom) { + // limit ourselves to one outstanding write at a time (onWriteSuccess calls + // scheduleWrite) + if (numActiveWrites_ > 0 || writesShutdown_) { + VLOG(4) << "skipping write during this loop, numActiveWrites_=" << + numActiveWrites_ << " writesShutdown_=" << writesShutdown_; + return nullptr; + } + + // We always tack on at least one body packet to the current write buf + // This ensures that a short HTTPS response will go out in a single SSL record + while (!txnEgressQueue_.empty()) { + uint32_t allowed = std::numeric_limits::max(); + if (connFlowControl_) { + allowed = connFlowControl_->getAvailableSend(); + if (allowed == 0) { + VLOG(4) << "Session-level send window is full, skipping " + << "body writes this loop"; + break; + } + } + auto txn = txnEgressQueue_.top(); + // returns true if there is more egress pending for this txn + if (txn->onWriteReady(allowed) || writeBuf_.front()) { + break; + } + } + *eom = false; + if (byteEventTracker_) { + uint64_t needed = byteEventTracker_->preSend(cork, eom, bytesWritten_); + if (needed > 0) { + VLOG(5) << *this << " writeBuf_.chainLength(): " + << writeBuf_.chainLength() << " txnEgressQueue_.empty(): " + << txnEgressQueue_.empty(); + + if (needed < writeBuf_.chainLength()) { + // split the next EOM chunk + VLOG(5) << *this << " splitting " << needed << " bytes out of a " + << writeBuf_.chainLength() << " bytes IOBuf"; + *cork = !txnEgressQueue_.empty(); + if (sessionStats_) { + sessionStats_->recordTTLBAIOBSplitByEom(); + } + return writeBuf_.split(needed); + } else { + CHECK(needed == writeBuf_.chainLength()); + } + } + } + + // cork if there are txns with pending egress + *cork = !txnEgressQueue_.empty(); + return writeBuf_.move(); +} + +void +HTTPSession::runLoopCallback() noexcept { + // We schedule this callback to run at the end of an event + // loop iteration if either of two conditions has happened: + // * The session has generated some egress data (see scheduleWrite()) + // * Reads have become unpaused (see resumeReads()) + DestructorGuard dg(this); + inLoopCallback_ = true; + folly::ScopeGuard scopeg = folly::makeGuard( + [this] { inLoopCallback_ = false;}); + VLOG(4) << *this << " in loop callback"; + + for (uint32_t i = 0; i < kMaxWritesPerLoop; ++i) { + bool cork = true; + bool eom = false; + unique_ptr writeBuf = getNextToSend(&cork, &eom); + + if (!writeBuf) { + break; + } + uint64_t len = writeBuf->computeChainDataLength(); + VLOG(11) << *this + << " bytes of egress to be written: " << len + << " cork:" << cork << " eom:" << eom; + if (len == 0) { + checkForShutdown(); + return; + } + + WriteSegment* segment = new WriteSegment(this, len); + segment->setCork(cork); + segment->setEOR(eom); + + pendingWrites_.push_back(*segment); + if (!writeTimeout_.isScheduled()) { + // Any performance concern here? + transactionTimeouts_->scheduleTimeout(&writeTimeout_); + } + numActiveWrites_++; + VLOG(4) << *this << " writing " << len << ", activeWrites=" + << numActiveWrites_ << " cork=" << cork << " eom=" << eom; + bytesScheduled_ += len; + sock_->writeChain(segment, std::move(writeBuf), segment->getFlags()); + if (numActiveWrites_ > 0) { + updateWriteBufSize(len); + break; + } + // writeChain can result in a writeError and trigger the shutdown code path + } + if (numActiveWrites_ == 0 && !writesShutdown_ && hasMoreWrites() && + (!connFlowControl_ || connFlowControl_->getAvailableSend())) { + scheduleWrite(); + } + + if (!readsPaused_ && !readsShutdown_) { + processReadData(); + + // Install the read callback if necessary + if (!readsPaused_ && !readsShutdown_ && !sock_->getReadCallback()) { + sock_->setReadCallback(this); + } + } + checkForShutdown(); +} + +void +HTTPSession::scheduleWrite() { + // Do all the network writes for this connection in one batch at + // the end of the current event loop iteration. Writing in a + // batch helps us packetize the network traffic more efficiently, + // as well as saving a few system calls. + if (!isLoopCallbackScheduled() && + (writeBuf_.front() || !txnEgressQueue_.empty())) { + VLOG(4) << *this << " scheduling write callback"; + sock_->getEventBase()->runInLoop(this); + } +} + +bool HTTPSession::egressLimitExceeded() const { + return pendingWriteSize_ >= kPendingWriteMax || numActiveWrites_ > 0; +} + +void +HTTPSession::updateWriteBufSize(int64_t delta) { + DCHECK(delta >= 0 || uint64_t(-delta) <= pendingWriteSize_); + pendingWriteSize_ += delta; + + if (egressLimitExceeded() && !writesPaused_) { + // Exceeded limit. Pause reading on the incoming stream. + VLOG(3) << "Pausing egress for " << *this; + writesPaused_ = true; + invokeOnAllTransactions(&HTTPTransaction::pauseEgress); + } else if (!egressLimitExceeded() && writesPaused_ && !writesShutdown_) { + // Dropped below limit. Resume reading on the incoming stream if needed. + VLOG(3) << "Resuming egress for " << *this; + writesPaused_ = false; + invokeOnAllTransactions(&HTTPTransaction::resumeEgress); + } +} + +void +HTTPSession::shutdownTransport(bool shutdownReads, + bool shutdownWrites) { + DestructorGuard guard(this); + + // shutdowns not accounted for, shouldn't see any + setCloseReason(ConnectionCloseReason::UNKNOWN); + + VLOG(4) << "shutdown request for " << *this << + ": reads=" << shutdownReads << " (currently " << readsShutdown_ << + "), writes=" << shutdownWrites << " (currently " << writesShutdown_ << + ")"; + + bool notifyEgressShutdown = false; + bool notifyIngressShutdown = false; + + ProxygenError error; + if (transportInfo_.sslError) { + error = kErrorSSL; + } else if (sock_->error()) { + VLOG(3) << "shutdown request for " << *this + << " on bad socket. Shutting down writes too."; + if (closeReason_ == ConnectionCloseReason::IO_WRITE_ERROR) { + error = kErrorWrite; + } else { + error = kErrorConnectionReset; + } + shutdownWrites = true; + } else { + error = kErrorEOF; + } + + if (shutdownWrites && !writesShutdown_) { + if (codec_->generateGoaway(writeBuf_, + codec_->getLastIncomingStreamID(), + ErrorCode::NO_ERROR)) { + scheduleWrite(); + } + if (!hasMoreWrites() && + (transactions_.empty() || codec_->closeOnEgressComplete())) { + writesShutdown_ = true; + if (byteEventTracker_) { + byteEventTracker_->drainByteEvents(); + } + if (resetAfterDrainingWrites_) { + VLOG(4) << *this << " writes drained, sending RST"; + sock_->closeWithReset(); + shutdownReads = true; + } else { + VLOG(4) << *this << " writes drained, closing"; + sock_->shutdownWriteNow(); + } + notifyEgressShutdown = true; + } else if (!writesDraining_) { + writesDraining_ = true; + notifyEgressShutdown = true; + } // else writes are already draining; don't double notify + } + + if (shutdownReads && !readsShutdown_) { + notifyIngressShutdown = true; + // TODO: send an RST if readBuf_ is non empty? + sock_->setReadCallback(nullptr); + readsShutdown_ = true; + if (!transactions_.empty() && error == kErrorConnectionReset) { + if (infoCallback_ != nullptr) { + infoCallback_->onIngressError(*this, error); + } + } else if (error == kErrorEOF) { + // Report to the codec that the ingress stream has ended + codec_->onIngressEOF(); + } + // Once reads are shutdown the parser should stop processing + codec_->setParserPaused(true); + } + + if (notifyIngressShutdown || notifyEgressShutdown) { + auto dir = (notifyIngressShutdown && notifyEgressShutdown) ? + HTTPException::Direction::INGRESS_AND_EGRESS : + (notifyIngressShutdown ? HTTPException::Direction::INGRESS : + HTTPException::Direction::EGRESS); + HTTPException ex(dir); + ex.setProxygenError(error); + invokeOnAllTransactions(&HTTPTransaction::onError, ex); + } + + checkForShutdown(); +} + +void HTTPSession::shutdownTransportWithReset(ProxygenError errorCode) { + DestructorGuard guard(this); + VLOG(4) << "shutdownTransportWithReset"; + + if (isLoopCallbackScheduled()) { + cancelLoopCallback(); + } + if (!readsShutdown_) { + sock_->setReadCallback(nullptr); + readsShutdown_ = true; + } + if (!writesShutdown_) { + writesShutdown_ = true; + IOBuf::destroy(writeBuf_.move()); + while (!pendingWrites_.empty()) { + pendingWrites_.front().detach(); + numActiveWrites_--; + } + VLOG(4) << *this << " cancel write timer"; + writeTimeout_.cancelTimeout(); + sock_->closeWithReset(); + } + errorOnAllTransactions(errorCode); + // drainByteEvents() can call detach(txn), which can in turn call + // shutdownTransport if we were already draining. To prevent double + // calling onError() to the transactions, we call drainByteEvents() + // after we've given the explicit error. + if (byteEventTracker_) { + byteEventTracker_->drainByteEvents(); + } + checkForShutdown(); +} + +void +HTTPSession::checkForShutdown() { + VLOG(10) << *this << " checking for shutdown, readShutdown=" << + readsShutdown_ << ", writesShutdown=" << writesShutdown_ << + ", transaction set empty=" << transactions_.empty(); + + // Two conditions are required to destroy the HTTPSession: + // * All writes have been finished. + // * There are no transactions remaining on the session. + if (writesShutdown_ && transactions_.empty() && + !isLoopCallbackScheduled()) { + VLOG(4) << "destroying " << *this; + sock_->setReadCallback(nullptr); + readsShutdown_ = true; + sock_->closeNow(); + destroy(); + } +} + +void +HTTPSession::drain() { + if (!draining_) { + VLOG(4) << *this << " draining"; + draining_ = true; + setCloseReason(ConnectionCloseReason::SHUTDOWN); + + if (allTransactionsStarted()) { + drainImpl(); + } + if (transactions_.empty() && isUpstream()) { + // We don't do this for downstream since we need to wait for + // inflight requests to arrive + VLOG(4) << *this << " shutdown from drain"; + shutdownTransport(true, true); + } + } else { + VLOG(4) << *this << " already draining"; + } +} + +void HTTPSession::drainImpl() { + if (codec_->isReusable() || codec_->isWaitingToDrain()) { + setCloseReason(ConnectionCloseReason::SHUTDOWN); + if (needsChromeWorkaround_) { + // Delay writing out the GOAWAY for chrome < 28 + VLOG(3) << *this << " setting max parallel transactions to zero " + "for chrome workaround"; + HTTPSettings* settings = codec_->getEgressSettings(); + if (settings) { + settings->setSetting(SettingsId::MAX_CONCURRENT_STREAMS, + maxConcurrentIncomingStreams_); + } + codec_->generateSettings(writeBuf_); + } else { + codec_->generateGoaway(writeBuf_, + getGracefulGoawayAck(), + ErrorCode::NO_ERROR); + } + scheduleWrite(); + } +} + +bool HTTPSession::shouldShutdown() const { + return draining_ && + allTransactionsStarted() && + (!codec_->supportsParallelRequests() || + isUpstream() || + !codec_->isReusable()); +} + +size_t HTTPSession::sendPing() { + const size_t bytes = codec_->generatePingRequest(writeBuf_); + if (bytes) { + scheduleWrite(); + } + return bytes; +} + +HTTPTransaction* +HTTPSession::findTransaction(HTTPCodec::StreamID streamID) { + auto it = transactions_.find(streamID); + if (it == transactions_.end()) { + return nullptr; + } else { + return it->second; + } +} + +bool +HTTPSession::addTransaction(HTTPTransaction* txn) { + if (!sock_->good()) { + // Refuse to add a transaction on a closing session + return false; + } + + if (transactions_.empty() && infoCallback_) { + infoCallback_->onActivateConnection(*this); + } + + auto match_pair = transactions_.emplace(txn->getID(), txn); + if (match_pair.second) { + VLOG(4) << *this << " adding streamID=" << txn->getID() + << ", liveTransactions was " << liveTransactions_; + liveTransactions_++; + } + + if ((isUpstream() && !txn->isPushed()) || + (isDownstream() && txn->isPushed())) { + outgoingStreams_++; + } else { + incomingStreams_++; + } + + if (txn->isPushed()) { + pushedTxns_++; + } + + return match_pair.second; +} + +void +HTTPSession::onWriteSuccess(uint64_t bytesWritten) { + DestructorGuard dg(this); + bytesWritten_ += bytesWritten; + transportInfo_.totalBytes += bytesWritten; + CHECK(writeTimeout_.isScheduled()); + if (pendingWrites_.empty()) { + VLOG(10) << "Cancel write timer on last successful write"; + writeTimeout_.cancelTimeout(); + } else { + VLOG(10) << "Refresh write timer on writeSuccess"; + transactionTimeouts_->scheduleTimeout(&writeTimeout_); + } + + if (infoCallback_) { + infoCallback_->onWrite(*this, bytesWritten); + } + + VLOG(5) << "total bytesWritten_: " << bytesWritten_; + + if (byteEventTracker_) { + byteEventTracker_->processByteEvents(bytesWritten_, + sock_->isEorTrackingEnabled()); + } + + if ((!codec_->isReusable() || readsShutdown_) && (transactions_.empty())) { + if (!codec_->isReusable()) { + // Shouldn't happen unless there is a bug. This can only happen when + // someone calls shutdownTransport, but did not specify a reason before. + setCloseReason(ConnectionCloseReason::UNKNOWN); + } + VLOG(4) << *this << " shutdown from onWriteSuccess"; + shutdownTransport(true, true); + } + numActiveWrites_--; + if (!inLoopCallback_) { + updateWriteBufSize(-bytesWritten); + // PRIO_FIXME: this is done because of the corking business... + // in the future we may want to have a pull model + // whereby the socket asks us for a given amount of + // data to send... + if (numActiveWrites_ == 0 && hasMoreWrites()) { + runLoopCallback(); + } + } + onWriteCompleted(); +} + +void +HTTPSession::onWriteError(size_t bytesWritten, + const TTransportException& ex) { + VLOG(4) << *this << " write error: " << ex.what(); + if (infoCallback_) { + infoCallback_->onWrite(*this, bytesWritten); + } + + // Save the SSL error, if there was one. It will be recorded later + if (ERR_GET_LIB(ex.getErrno()) == ERR_LIB_SSL) { + transportInfo_.sslError = ERR_GET_REASON(ex.getErrno()); + } + + setCloseReason(ConnectionCloseReason::IO_WRITE_ERROR); + shutdownTransportWithReset(kErrorWrite); +} + +void +HTTPSession::onWriteCompleted() { + if (!writesDraining_) { + return; + } + + if (numActiveWrites_) { + return; + } + + // Don't shutdown if there might be more writes + if (!pendingWrites_.empty()) { + return; + } + + // All finished draining writes, so shut down the egress + shutdownTransport(false, true); +} + +void HTTPSession::onSessionParseError(const HTTPException& error) { + VLOG(4) << *this << " session layer parse error. Terminate the session."; + if (error.hasCodecStatusCode()) { + codec_->generateGoaway(writeBuf_, + codec_->getLastIncomingStreamID(), + error.getCodecStatusCode()); + scheduleWrite(); + } + setCloseReason(ConnectionCloseReason::SESSION_PARSE_ERROR); + shutdownTransport(true, true); +} + +void HTTPSession::onNewTransactionParseError(HTTPCodec::StreamID streamID, + const HTTPException& error) { + VLOG(4) << *this << " parse error with new transaction"; + if (error.hasCodecStatusCode()) { + codec_->generateRstStream(writeBuf_, streamID, error.getCodecStatusCode()); + scheduleWrite(); + } + if (!codec_->isReusable()) { + // HTTP 1x codec does not support per stream abort so this will + // render the codec not reusable + setCloseReason(ConnectionCloseReason::SESSION_PARSE_ERROR); + } +} + +void +HTTPSession::handleErrorDirectly(HTTPTransaction* txn, + const HTTPException& error) { + VLOG(4) << *this << " creating direct error handler"; + DCHECK(txn); + auto handler = getParseErrorHandler(txn, error); + if (!handler) { + txn->sendAbort(); + return; + } + txn->setHandler(handler); + if (infoCallback_) { + infoCallback_->onIngressError(*this, error.getProxygenError()); + } + txn->onError(error); +} + +void +HTTPSession::pauseReads() { + // Make sure the parser is paused. Note that if reads are shutdown + // before they are paused, we never make it past the if. + codec_->setParserPaused(true); + if (readsShutdown_ || readsPaused_ || + (codec_->supportsParallelRequests() && + pendingReadSize_ <= kDefaultReadBufLimit)) { + return; + } + VLOG(4) << *this << ": pausing reads"; + if (infoCallback_) { + infoCallback_->onIngressPaused(*this); + } + cancelTimeout(); + sock_->setReadCallback(nullptr); + readsPaused_ = true; +} + +void +HTTPSession::resumeReads() { + if (readsShutdown_ || !readsPaused_ || + (codec_->supportsParallelRequests() && + pendingReadSize_ > kDefaultReadBufLimit)) { + return; + } + VLOG(4) << *this << ": resuming reads"; + resetTimeout(); + readsPaused_ = false; + codec_->setParserPaused(false); + if (!isLoopCallbackScheduled()) { + sock_->getEventBase()->runInLoop(this); + } +} + +bool +HTTPSession::hasMoreWrites() const { + VLOG(10) << __PRETTY_FUNCTION__ + << " numActiveWrites_: " << numActiveWrites_ + << " pendingWrites_.empty(): " << pendingWrites_.empty() + << " pendingWrites_.size(): " << pendingWrites_.size() + << " txnEgressQueue_.empty(): " << txnEgressQueue_.empty(); + + return (numActiveWrites_ != 0) || + !pendingWrites_.empty() || writeBuf_.front() || + !txnEgressQueue_.empty(); +} + +void HTTPSession::errorOnAllTransactions(ProxygenError err) { + std::vector ids; + for (auto txn: transactions_) { + ids.push_back(txn.first); + } + errorOnTransactionIds(ids, err); +} + +void HTTPSession::errorOnTransactionIds( + const std::vector& ids, + ProxygenError err) { + + for (auto id: ids) { + auto txn = findTransaction(id); + if (txn != nullptr) { + HTTPException ex(HTTPException::Direction::INGRESS_AND_EGRESS); + ex.setProxygenError(err); + txn->onError(ex); + } + } +} + +void HTTPSession::onConnectionSendWindowOpen() { + // We can write more now. Schedule a write. + scheduleWrite(); +} + +HTTPCodec::StreamID HTTPSession::getGracefulGoawayAck() const { + if (!codec_->isReusable() || codec_->isWaitingToDrain()) { + // TODO: just track last stream ID inside HTTPSession since this logic + // is shared between HTTP/2 and SPDY + return codec_->getLastIncomingStreamID(); + } + // return the maximum possible stream id + return std::numeric_limits::max(); +} + +void HTTPSession::invalidStream(HTTPCodec::StreamID stream, ErrorCode code) { + if (!codec_->supportsParallelRequests()) { + LOG(ERROR) << "Invalid stream on non-parallel codec."; + return; + } + + HTTPException err(HTTPException::Direction::INGRESS_AND_EGRESS, + "invalid stream=", stream); + // TODO: Below line will change for HTTP/2 -- just call a const getter + // function for the status code. + err.setCodecStatusCode(code); + onError(stream, err, true); +} + +void HTTPSession::onPingReplyLatency(int64_t latency) noexcept { + if (infoCallback_ && latency >= 0) { + infoCallback_->onPingReply(latency); + } +} + +uint64_t HTTPSession::getAppBytesWritten() noexcept { + return sock_->getAppBytesWritten(); +} + +uint64_t HTTPSession::getRawBytesWritten() noexcept { + return sock_->getRawBytesWritten(); +} + +void HTTPSession::onDeleteAckEvent() { + if (readsShutdown_) { + shutdownTransport(true, transactions_.empty()); + } +} + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPSession.h b/proxygen/lib/http/session/HTTPSession.h new file mode 100644 index 0000000000..03bdbb431d --- /dev/null +++ b/proxygen/lib/http/session/HTTPSession.h @@ -0,0 +1,832 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPConstants.h" +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/codec/FlowControlFilter.h" +#include "proxygen/lib/http/codec/HTTPCodec.h" +#include "proxygen/lib/http/codec/HTTPCodecFilter.h" +#include "proxygen/lib/http/session/ByteEventTracker.h" +#include "proxygen/lib/http/session/HTTPEvent.h" +#include "proxygen/lib/http/session/HTTPTransaction.h" +#include "proxygen/lib/services/TransportInfo.h" +#include "proxygen/lib/utils/Time.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace proxygen { + +class HTTPSessionController; +class HTTPSessionStats; + +class HTTPSession: + private FlowControlFilter::Callback, + private HTTPCodec::Callback, + private folly::EventBase::LoopCallback, + public ByteEventTracker::Callback, + public HTTPTransaction::Transport, + public apache::thrift::async::TAsyncTransport::ReadCallback, + public folly::wangle::ManagedConnection { + public: + typedef std::unique_ptr UniquePtr; + + /** + * Optional callback interface that the HTTPSession + * notifies of connection lifecycle events. + */ + class InfoCallback { + public: + virtual ~InfoCallback() {} + + // Note: you must not start any asynchronous work from onCreate() + virtual void onCreate(const HTTPSession&) = 0; + virtual void onIngressError(const HTTPSession&, ProxygenError) = 0; + virtual void onRead(const HTTPSession&, size_t bytesRead) = 0; + virtual void onWrite(const HTTPSession&, size_t bytesWritten) = 0; + virtual void onRequestBegin(const HTTPSession&) = 0; + virtual void onRequestEnd(const HTTPSession&, + uint32_t maxIngressQueueSize) = 0; + virtual void onActivateConnection(const HTTPSession&) = 0; + virtual void onDeactivateConnection(const HTTPSession&) = 0; + // Note: you must not start any asynchronous work from onDestroy() + virtual void onDestroy(const HTTPSession&) = 0; + virtual void onIngressMessage(const HTTPSession&, + const HTTPMessage&) = 0; + virtual void onIngressLimitExceeded(const HTTPSession&) = 0; + virtual void onIngressPaused(const HTTPSession&) = 0; + virtual void onTransactionDetached(const HTTPSession&) = 0; + virtual void onPingReply(int64_t latency) = 0; + virtual void onSettingsOutgoingStreamsFull(const HTTPSession&) = 0; + virtual void onSettingsOutgoingStreamsNotFull(const HTTPSession&) = 0; + }; + + class WriteTimeout : + public apache::thrift::async::TAsyncTimeoutSet::Callback { + public: + explicit WriteTimeout(HTTPSession* session) : session_(session) {} + virtual ~WriteTimeout() {} + + void timeoutExpired() noexcept { + session_->writeTimeoutExpired(); + } + private: + HTTPSession* session_; + }; + + /** + * Set the read buffer limit to be used for all new HTTPSession objects. + */ + static void setDefaultReadBufferLimit(uint32_t limit) { + kDefaultReadBufLimit = limit; + VLOG(1) << "read buffer limit: " << int(limit / 1000) << "KB"; + } + + void setInfoCallback(InfoCallback* callback); + + void setSessionStats(HTTPSessionStats* stats); + + apache::thrift::async::TAsyncTransport* getTransport() { + return sock_.get(); + } + + const apache::thrift::async::TAsyncTransport* getTransport() const { + return sock_.get(); + } + + bool hasActiveTransactions() const { + return !transactions_.empty(); + } + + /** + * Returns true iff a new outgoing transaction can be made on this session + */ + bool supportsMoreTransactions() const { + return (outgoingStreams_ < maxConcurrentOutgoingStreamsConfig_) && + (outgoingStreams_ < maxConcurrentOutgoingStreamsRemote_); + } + + uint32_t getNumOutgoingStreams() const { + return outgoingStreams_; + } + + uint32_t getNumIncomingStreams() const { + return incomingStreams_; + } + + uint32_t getMaxConcurrentOutgoingStreams() const { + return std::min(maxConcurrentOutgoingStreamsConfig_, + maxConcurrentOutgoingStreamsRemote_); + } + + uint32_t getMaxConcurrentPushTransactions() const { + return maxConcurrentPushTransactions_; + } + + bool writesDraining() const { + return writesDraining_; + } + + const HTTPSessionController* getController() const { return controller_; } + HTTPSessionController* getController() { return controller_; } + + /** + * Start closing the socket. + * @param shutdownReads Whether to close the read side of the + * socket. All transactions which are not ingress complete will receive + * an error. + * @param shutdownWrites Whether to close the write side of the + * socket. All transactions which are not egress complete will receive + * an error. + */ + void shutdownTransport(bool shutdownReads = true, + bool shutdownWrites = true); + + /** + * Immediately close the socket in both directions, discarding any + * queued writes that haven't yet been transferred to the kernel, + * and send a RST to the client. + * All transactions receive onWriteError. + * + * @param errorCode Error code sent with the onWriteError to transactions. + */ + void shutdownTransportWithReset(ProxygenError errorCode); + + ConnectionCloseReason getConnectionCloseReason() const { + return closeReason_; + } + + HTTPCodecFilterChain& getCodecFilterChain() { + return codec_; + } + + /** + * Set flow control properties on the session. + * + * @param initialReceiveWindow size of initial receive window + * for all ingress streams; set via + * the initial SETTINGS frame + * @param receiveStreamWindowSize per-stream receive window; sent + * via a WINDOW_UPDATE frame + */ + void setFlowControl( + size_t initialReceiveWindow, + size_t receiveStreamWindowSize); + + /** + * Set the maximum number of outgoing transactions this session can open + * at once. Note: you can only call function before startNow() is called + * since the remote side can change this value. + */ + void setMaxConcurrentOutgoingStreams(uint32_t num); + + /* + * The maximum number of concurrent push transactions that can be supported + * on this session. + */ + void setMaxConcurrentPushTransactions(uint32_t num); + + /** + * Get the number of egress bytes this session will buffer before + * pausing all transactions' egress. + */ + static uint64_t getPendingWriteMax() { + return kPendingWriteMax; + } + + /** + * Start reading from the transport and send any introductory messages + * to the remote side. This function must be called once per session to + * begin reads. + */ + void startNow(); + + /** + * Returns true if this session is draining. This can happen if drain() + * is called explicitly, if a GOAWAY frame is received, or during shutdown. + */ + bool isDraining() const override { return draining_; } + + /** + * Causes a ping to be sent on the session. If the underlying protocol + * doesn't support pings, this will return 0. Otherwise, it will return + * the number of bytes written on the transport to send the ping. + */ + size_t sendPing(); + + // ManagedConnection methods + void timeoutExpired() noexcept { + readTimeoutExpired(); + } + void describe(std::ostream& os) const override; + bool isBusy() const override; + void notifyPendingShutdown() override; + void closeWhenIdle() override; + void dropConnection() override; + void dumpConnectionState(uint8_t loglevel) override; + + bool isUpstream() const; + bool isDownstream() const; + + protected: + /** + * HTTPSession is an abstract base class and cannot be instantiated + * directly. If you want to handle requests and send responses (act as a + * server), construct a HTTPDownstreamSession. If you want to make + * requests and handle responses (act as a client), construct a + * HTTPUpstreamSession. + * + * @param transactionTimeouts Timeout for each transaction in the session. + * @param sock An open socket on which any applicable TLS + * handshaking has been completed already. + * @param localAddr Address and port of the local end of + * the socket. + * @param peerAddr Address and port of the remote end of + * the socket. + * @param controller Controller which can create the handler for + * a new transaction. + * @param codec A codec with which to parse/generate messages + * in whatever HTTP-like wire format this + * session needs. + * @param tinfo Struct containing the transport's TCP/SSL + * level info. + * @param InfoCallback Optional callback to be informed of session + * lifecycle events. + */ + HTTPSession( + apache::thrift::async::TAsyncTimeoutSet* transactionTimeouts, + apache::thrift::async::TAsyncTransport::UniquePtr sock, + const folly::SocketAddress& localAddr, + const folly::SocketAddress& peerAddr, + HTTPSessionController* controller, + std::unique_ptr codec, + const TransportInfo& tinfo, + InfoCallback* infoCallback = nullptr); + + virtual ~HTTPSession(); + + /** + * Called by onHeadersComplete(). This function allows downstream and + * upstream to do any setup (like preparing a handler) when headers are + * first received from the remote side on a given transaction. + */ + virtual void setupOnHeadersComplete(HTTPTransaction* txn, + HTTPMessage* msg) = 0; + + /** + * Called by handleErrorDirectly (when handling parse errors) if the + * transaction has no handler. + */ + virtual HTTPTransaction::Handler* getParseErrorHandler( + HTTPTransaction* txn, const HTTPException& error) = 0; + + /** + * Called by transactionTimeout if the transaction has no handler. + */ + virtual HTTPTransaction::Handler* getTransactionTimeoutHandler( + HTTPTransaction* txn) = 0; + + /** + * Invoked when headers have been sent. + */ + virtual void onHeadersSent(const HTTPMessage& headers, + bool codecWasReusable) {} + + virtual bool allTransactionsStarted() const = 0; + + void setNewTransactionPauseState(HTTPTransaction* txn); + + /** + * Invoked when the transaction finishes sending a message and + * appropriately shuts down reads and/or writes with respect to + * downstream or upstream semantics. + */ + void onEgressMessageFinished(HTTPTransaction* txn, + bool withRST = false); + + /** + * Gets the next IOBuf to send (either writeBuf_ or new egress from + * the priority queue), and sets cork appropriately + */ + std::unique_ptr getNextToSend(bool* cork, bool* eom); + + void decrementTransactionCount(HTTPTransaction* txn, + bool ingressEOM, bool egressEOM); + + size_t getCodecSendWindowSize() const; + + /** + * Drains the current transactions and prevents new transactions from being + * created on this session. If this is an upstream session and the + * number of transactions reaches zero, this session will shutdown the + * transport and delete itself. For downstream sessions, an explicit + * call to dropConnection() or shutdownTransport() is required. + */ + void drain(); + + /** + * Helper class to track write buffers until they have been fully written and + * can be deleted. + */ + class WriteSegment : + public apache::thrift::async::TAsyncTransport::WriteCallback { + public: + WriteSegment(HTTPSession* session, uint64_t length); + + void setCork(bool cork) { + if (cork) { + flags_ = flags_ | apache::thrift::async::WriteFlags::CORK; + } else { + unSet(flags_, apache::thrift::async::WriteFlags::CORK); + } + } + + void setEOR(bool eor) { + if (eor) { + flags_ = flags_ | apache::thrift::async::WriteFlags::EOR; + } else { + unSet(flags_, apache::thrift::async::WriteFlags::EOR); + } + } + + /** + * Clear the session. This is used if the session + * does not want to receive future notification about this segment. + */ + void detach(); + + apache::thrift::async::WriteFlags getFlags() { + return flags_; + } + + uint64_t getLength() const { + return length_; + } + + // TAsyncTransport::WriteCallback methods + virtual void writeSuccess() noexcept; + virtual void writeError( + size_t bytesWritten, + const apache::thrift::transport::TTransportException&) noexcept; + + folly::IntrusiveListHook listHook; + private: + + /** + * Unlink this segment from the list. + */ + void remove(); + + HTTPSession* session_; + uint64_t length_; + apache::thrift::async::WriteFlags flags_{ + apache::thrift::async::WriteFlags::NONE}; + }; + typedef folly::IntrusiveList + WriteSegmentList; + + void readTimeoutExpired() noexcept; + void writeTimeoutExpired() noexcept; + + // TAsyncTransport::ReadCallback methods + void getReadBuffer(void** buf, size_t* bufSize); + void readDataAvailable(size_t readSize) noexcept; + void processReadData(); + void readEOF() noexcept; + void readError( + const apache::thrift::transport::TTransportException&) noexcept; + + // HTTPCodec::Callback methods + void onMessageBegin(HTTPCodec::StreamID streamID, HTTPMessage* msg); + void onPushMessageBegin(HTTPCodec::StreamID streamID, + HTTPCodec::StreamID assocStreamID, + HTTPMessage* msg); + void onHeadersComplete(HTTPCodec::StreamID streamID, + std::unique_ptr msg); + void onBody(HTTPCodec::StreamID streamID, + std::unique_ptr chain); + void onChunkHeader(HTTPCodec::StreamID stream, size_t length); + void onChunkComplete(HTTPCodec::StreamID stream); + void onTrailersComplete(HTTPCodec::StreamID streamID, + std::unique_ptr trailers); + void onMessageComplete(HTTPCodec::StreamID streamID, bool upgrade); + void onError(HTTPCodec::StreamID streamID, + const HTTPException& error, bool newTxn); + void onAbort(HTTPCodec::StreamID streamID, + ErrorCode code); + void onGoaway(uint64_t lastGoodStreamID, + ErrorCode code); + void onPingRequest(uint64_t uniqueID); + void onPingReply(uint64_t uniqueID); + void onWindowUpdate(HTTPCodec::StreamID stream, uint32_t amount); + void onSettings(const SettingsList& settings); + uint32_t numOutgoingStreams() const { return outgoingStreams_; } + uint32_t numIncomingStreams() const { return incomingStreams_; } + + // HTTPTransaction::Transport methods + void pauseIngress(HTTPTransaction* txn) noexcept override; + void resumeIngress(HTTPTransaction* txn) noexcept override; + void transactionTimeout(HTTPTransaction* txn) noexcept override; + void sendHeaders(HTTPTransaction* txn, + const HTTPMessage& headers, + HTTPHeaderSize* size) noexcept override; + size_t sendBody(HTTPTransaction* txn, std::unique_ptr, + bool includeEOM) noexcept override; + size_t sendChunkHeader(HTTPTransaction* txn, + size_t length) noexcept override; + size_t sendChunkTerminator(HTTPTransaction* txn) noexcept override; + size_t sendTrailers(HTTPTransaction* txn, + const HTTPHeaders& trailers) noexcept override; + size_t sendEOM(HTTPTransaction* txn) noexcept override; + size_t sendAbort(HTTPTransaction* txn, + ErrorCode statusCode) noexcept override; + void detach(HTTPTransaction* txn) noexcept override; + size_t sendWindowUpdate(HTTPTransaction* txn, + uint32_t bytes) noexcept override; + void notifyPendingEgress() noexcept override; + void notifyIngressBodyProcessed(uint32_t bytes) noexcept override; + HTTPTransaction* newPushedTransaction(HTTPCodec::StreamID assocStreamId, + HTTPTransaction::PushHandler* handler, + int8_t priority) noexcept override; + + public: + const folly::SocketAddress& getLocalAddress() + const noexcept override; + const folly::SocketAddress& getPeerAddress() + const noexcept; + + TransportInfo& getSetupTransportInfo() noexcept; + const TransportInfo& getSetupTransportInfo() const noexcept override; + bool getCurrentTransportInfo(TransportInfo* tinfo) override; + HTTPCodec& getCodec() noexcept { + return *CHECK_NOTNULL(codec_.call()); + } + const HTTPCodec& getCodec() const noexcept override { + return *CHECK_NOTNULL(codec_.call()); + } + + void setByteEventTracker(std::unique_ptr byteEventTracker); + ByteEventTracker* getByteEventTracker() { return byteEventTracker_.get(); } + + protected: + + /** + * Handle new messages from the codec and create a txn for the message. + * @returns the created transaction. + */ + HTTPTransaction* onMessageBeginImpl(HTTPCodec::StreamID streamID, + HTTPCodec::StreamID assocStreamID, + HTTPMessage* msg); + + // EventBase::LoopCallback methods + void runLoopCallback() noexcept override; + + /** + * Schedule a write to occur at the end of this event loop. + */ + void scheduleWrite(); + + /** + * Update the size of the unwritten egress data and invoke + * callbacks if the size has crossed the buffering limit. + */ + void updateWriteBufSize(int64_t delta); + + /** + * Returns true iff egress should stop on this session. + */ + bool egressLimitExceeded() const; + + /** + * Tells us what would be the offset of the next byte to be + * enqueued within the whole session. + */ + inline uint64_t sessionByteOffset() { + return bytesScheduled_ + writeBuf_.chainLength(); + } + + /** + * Check whether the socket is shut down in both directions; if it is, + * initiate the destruction of this HTTPSession. + */ + void checkForShutdown(); + + /** + * Get the HTTPTransaction for the given transaction ID, or nullptr if that + * transaction ID does not exist within this HTTPSession. + */ + HTTPTransaction* findTransaction(HTTPCodec::StreamID streamID); + + /** + * Add a new transaction. + * @return true on success, or false if a transaction with the same + * ID already exists + */ + bool addTransaction(HTTPTransaction* txn); + + /** Invoked by WriteSegment on completion of a write. */ + void onWriteSuccess(uint64_t bytesWritten); + + /** Invoked by WriteSegment on write failure. */ + void onWriteError(size_t bytesWritten, + const apache::thrift::transport::TTransportException& ex); + + /** Check whether to shut down the transport after a write completes. */ + void onWriteCompleted(); + + /** Stop reading from the transport until resumeReads() is called */ + void pauseReads(); + + /** + * Send a session layer abort and shutdown the transport for reads and + * writes. + */ + void onSessionParseError(const HTTPException& error); + + /** + * Send a transaction abort and leave the session and transport intact. + */ + void onNewTransactionParseError(HTTPCodec::StreamID streamID, + const HTTPException& error); + + /** + * Install a direct response handler for the transaction based on the + * error. + */ + void handleErrorDirectly(HTTPTransaction* txn, + const HTTPException& error); + + /** + * Unpause reading from the transport. + * @note If any codec callbacks arrived while reads were paused, + * they will be processed before network reads resume. + */ + void resumeReads(); + + /** Check whether the session has any writes in progress or upcoming */ + bool hasMoreWrites() const; + + /** + * This function invokes a callback on all transactions. It is safe, + * but runs in O(n*log n) and if the callback *adds* transactions, + * they will not get the callback. + */ + template + void invokeOnAllTransactions(void (HTTPTransaction::*fn)(Args1...), + Args2&&... args) { + DestructorGuard g(this); + std::vector ids; + for (auto txn: transactions_) { + ids.push_back(txn.first); + } + for (auto idit = ids.begin(); idit != ids.end() && !transactions_.empty(); + ++idit) { + auto txn = findTransaction(*idit); + if (txn != nullptr) { + (txn->*fn)(std::forward(args)...); + } + } + } + + /** + * This function invokes a callback on all transactions. It is safe, + * but runs in O(n*log n) and if the callback *adds* transactions, + * they will not get the callback. + */ + void errorOnAllTransactions(ProxygenError err); + + void errorOnTransactionIds(const std::vector& ids, + ProxygenError err); + + void setCloseReason(ConnectionCloseReason reason) { + if (closeReason_ == ConnectionCloseReason::kMAX_REASON) { + closeReason_ = reason; + } + } + + /** + * Returns true iff this session should shutdown at this time. Default + * behavior is to not shutdown. + */ + bool shouldShutdown() const; + + void drainImpl(); + + /** Chain of ingress IOBufs */ + folly::IOBufQueue readBuf_{folly::IOBufQueue::cacheChainLength()}; + + /** Queue of egress IOBufs */ + folly::IOBufQueue writeBuf_{folly::IOBufQueue::cacheChainLength()}; + + /** Priority queue of transactions with egress pending */ + HTTPTransaction::PriorityQueue txnEgressQueue_; + + std::map transactions_; + + /** Count of transactions awaiting input */ + uint32_t liveTransactions_{0}; + + /** Transaction sequence number */ + uint32_t transactionSeqNo_{0}; + + /** Address of this end of the TCP connection */ + folly::SocketAddress localAddr_; + + /** Address of the remote end of the TCP connection */ + folly::SocketAddress peerAddr_; + + WriteSegmentList pendingWrites_; + + apache::thrift::async::TAsyncTransport::UniquePtr sock_; + + HTTPSessionController* controller_{nullptr}; + + HTTPCodecFilterChain codec_; + + InfoCallback* infoCallback_{nullptr}; + + /** + * The root cause reason this connection was closed. + */ + ConnectionCloseReason closeReason_ + {ConnectionCloseReason::kMAX_REASON}; + + WriteTimeout writeTimeout_; + + apache::thrift::async::TAsyncTimeoutSet* transactionTimeouts_{nullptr}; + + HTTPSessionStats* sessionStats_{nullptr}; + + TransportInfo transportInfo_; + + /** + * Connection level flow control for SPDY >= 3.1 and HTTP/2 + */ + FlowControlFilter* connFlowControl_{nullptr}; + + /** + * The maximum number of concurrent push transactions that can be supported + * on this session + */ + uint32_t maxConcurrentPushTransactions_{100}; + + /** + * The number of open push transactions + */ + uint32_t pushedTxns_{0}; + + /** + * Bytes of egress data sent to the socket but not yet written + * to the network. + */ + uint64_t pendingWriteSize_{0}; + + /** + * The maximum number of concurrent transactions that this session may + * create, as configured locally. + */ + uint32_t maxConcurrentOutgoingStreamsConfig_{100}; + + /** + * The received setting for the maximum number of concurrent + * transactions that this session may create. We may assume the + * remote allows unlimited transactions until we get a SETTINGS frame, + * but to be reasonable, assume the remote doesn't allow more than 100K + * concurrent transactions on one connection. + */ + uint32_t maxConcurrentOutgoingStreamsRemote_{100000}; + + /** + * The maximum number of concurrent transactions that this session's peer + * may create. + */ + uint32_t maxConcurrentIncomingStreams_{100}; + + /** + * The number concurrent transactions initiated by this session + */ + uint32_t outgoingStreams_{0}; + + /** + * The number of concurrent transactions initiated by this sessions's peer + */ + uint32_t incomingStreams_{0}; + + /** + * Bytes of ingress data read from the socket, but not yet sent to a + * transaction. + */ + uint32_t pendingReadSize_{0}; + + /** + * Number of writes submitted to the transport for which we haven't yet + * received completion or failure callbacks. + */ + unsigned numActiveWrites_{0}; + + /** + * Number of bytes written so far. + */ + uint64_t bytesWritten_{0}; + + /** + * Number of bytes scheduled so far. + */ + uint64_t bytesScheduled_{0}; + + // Flow control settings + size_t initialReceiveWindow_{65536}; + size_t receiveStreamWindowSize_{65536}; + + const TransportDirection direction_; + + /** + * Indicates if the session is waiting for existing transactions to close. + * Once all transactions close, the session will be deleted. + */ + bool draining_:1; + + // TODO: remove this once the percent of Chrome < 28 traffic is less + // than 0.1% + bool needsChromeWorkaround_:1; + + /** + * Indicates whether an upgrade request has been received from the codec. + */ + bool ingressUpgraded_:1; + + bool started_:1; + bool readsPaused_:1; + bool readsShutdown_:1; + bool writesPaused_:1; + bool writesShutdown_:1; + bool writesDraining_:1; + bool resetAfterDrainingWrites_:1; + // indicates a fatal error that prevents further ingress data processing + bool ingressError_:1; + bool inLoopCallback_:1; + + /** + * Maximum number of ingress body bytes that can be buffered across all + * transactions for this single session/connection. + */ + static uint32_t kDefaultReadBufLimit; + + /** + * Maximum number of bytes that can be buffered in sock_ before + * this session will start applying backpressure to its transactions. + */ + static uint32_t kPendingWriteMax; + + private: + void onSetSendWindow(uint32_t windowSize); + void onSetMaxInitiatedStreams(uint32_t maxTxns); + + void addLastByteEvent(HTTPTransaction* txn, uint64_t byteNo) noexcept; + + void addAckToLastByteEvent(HTTPTransaction* txn, + const ByteEvent& lastByteEvent); + + /** + * Callback function from the flow control filter if the full window + * becomes not full. + */ + void onConnectionSendWindowOpen() override; + + /** + * Get the id of the stream we should ack in a graceful GOAWAY + */ + HTTPCodec::StreamID getGracefulGoawayAck() const; + + /** + * Invoked when the codec processes callbacks for a stream we are no + * longer tracking. + */ + void invalidStream(HTTPCodec::StreamID stream, + ErrorCode code = ErrorCode::_SPDY_INVALID_STREAM); + + //ByteEventTracker::Callback functions + void onPingReplyLatency(int64_t latency) noexcept override; + uint64_t getAppBytesWritten() noexcept override; + uint64_t getRawBytesWritten() noexcept override; + void onDeleteAckEvent() override; + + std::unique_ptr byteEventTracker_{ + folly::make_unique(this)}; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPSessionAcceptor.cpp b/proxygen/lib/http/session/HTTPSessionAcceptor.cpp new file mode 100644 index 0000000000..a51224558e --- /dev/null +++ b/proxygen/lib/http/session/HTTPSessionAcceptor.cpp @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPSessionAcceptor.h" + +#include "proxygen/lib/http/codec/HTTP1xCodec.h" +#include "proxygen/lib/http/session/HTTPDirectResponseHandler.h" + +using apache::thrift::async::TAsyncSocket; +using folly::SocketAddress; +using std::list; +using std::string; +using std::unique_ptr; + +namespace proxygen { + +const SocketAddress HTTPSessionAcceptor::unknownSocketAddress_("0.0.0.0", 0); + +HTTPSessionAcceptor::HTTPSessionAcceptor( + const AcceptorConfiguration& accConfig): + HTTPAcceptor(accConfig), + simpleController_(this) { + if (!isSSL()) { + auto version = SPDYCodec::getVersion(accConfig.plaintextProtocol); + if (version) { + alwaysUseSPDYVersion_ = *version; + } + } +} + +HTTPSessionAcceptor::~HTTPSessionAcceptor() { +} + +const HTTPErrorPage* HTTPSessionAcceptor::getErrorPage( + const SocketAddress& addr) const { + const HTTPErrorPage* errorPage = nullptr; + if (isInternal()) { + if (addr.isPrivateAddress()) { + errorPage = diagnosticErrorPage_.get(); + } + } + if (errorPage == nullptr) { + errorPage = defaultErrorPage_.get(); + } + return errorPage; +} + +void HTTPSessionAcceptor::onNewConnection( + TAsyncSocket::UniquePtr sock, + const SocketAddress* peerAddress, + const string& nextProtocol, + const TransportInfo& tinfo) { + unique_ptr codec; + SPDYVersion spdyVersion; + + if (!isSSL() && alwaysUseSPDYVersion_) { + codec = folly::make_unique( + TransportDirection::DOWNSTREAM, + alwaysUseSPDYVersion_.value(), + accConfig_.spdyCompressionLevel); + } else if (nextProtocol.empty() || + HTTP1xCodec::supportsNextProtocol(nextProtocol)) { + codec = folly::make_unique(TransportDirection::DOWNSTREAM); + } else if (auto version = SPDYCodec::getVersion(nextProtocol)) { + codec = folly::make_unique( + TransportDirection::DOWNSTREAM, + *version, + accConfig_.spdyCompressionLevel); + } else { + // Either we advertised a protocol we don't support or the + // client requested a protocol we didn't advertise. + VLOG(2) << "Client requested unrecognized next protocol " << nextProtocol; + return; + } + + CHECK(codec); + + auto controller = getController(); + SocketAddress localAddress; + try { + sock->getLocalAddress(&localAddress); + } catch (...) { + VLOG(3) << "couldn't get local address for socket"; + localAddress = unknownSocketAddress_; + } + VLOG(4) << "Created new session for peer " << *peerAddress; + HTTPDownstreamSession* session = + new HTTPDownstreamSession(getTransactionTimeoutSet(), std::move(sock), + localAddress, *peerAddress, + controller, std::move(codec), tinfo, this); + session->setSessionStats(downstreamSessionStats_); + Acceptor::addConnection(session); + session->startNow(); +} + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPSessionAcceptor.h b/proxygen/lib/http/session/HTTPSessionAcceptor.h new file mode 100644 index 0000000000..7537400e64 --- /dev/null +++ b/proxygen/lib/http/session/HTTPSessionAcceptor.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/SPDYCodec.h" +#include "proxygen/lib/http/session/HTTPDownstreamSession.h" +#include "proxygen/lib/http/session/HTTPErrorPage.h" +#include "proxygen/lib/http/session/SimpleController.h" +#include "proxygen/lib/services/HTTPAcceptor.h" + +namespace proxygen { + +class HTTPSessionStats; + +/** + * Specialization of Acceptor that serves as an abstract base for + * acceptors that support HTTP and related protocols. + */ +class HTTPSessionAcceptor: + public HTTPAcceptor, + private HTTPSession::InfoCallback { +public: + explicit HTTPSessionAcceptor(const AcceptorConfiguration& accConfig); + virtual ~HTTPSessionAcceptor(); + + /** + * Set the default error page generator. + */ + void setDefaultErrorPage(std::unique_ptr generator) { + defaultErrorPage_ = std::move(generator); + } + + /** + * Access the default error page generator. + */ + const HTTPErrorPage* getDefaultErrorPage() const { + return defaultErrorPage_.get(); + } + + /** + * Set an alternate error page generator to use for internal clients. + */ + void setDiagnosticErrorPage(std::unique_ptr generator) { + diagnosticErrorPage_ = std::move(generator); + } + + /** + * Access the diagnostic error page generator. + */ + const HTTPErrorPage* getDiagnosticErrorPage() const { + return diagnosticErrorPage_.get(); + } + + /** + * Access the right error page generator for a connection. + * @param localAddr Address of the local end of the connection. + * @return The diagnostic error page generator if one has been + * set AND the connection is to an internal VIP, or + * else the default error page generator if one has + * been set, or else nullptr. + */ + virtual const HTTPErrorPage* getErrorPage( + const folly::SocketAddress& addr) const; + + /** + * Create a Handler for a new transaction. The transaction and HTTP message + * (request) are passed so the implementation can construct different + * handlers based on these. The transaction will be explicitly set on the + * handler later via setTransaction. The request message will be passed + * in onHeadersComplete. + */ + virtual HTTPTransaction::Handler* newHandler( + HTTPTransaction& txn, HTTPMessage* msg) noexcept = 0; + +protected: + /** + * This function is invoked when a new session is created to get the + * controller to associate with the new session. Child classes may + * override this function to provide their own more sophisticated + * controller here. + */ + virtual HTTPSessionController* getController() { + return &simpleController_; + } + + HTTPSessionStats* downstreamSessionStats_{nullptr}; + + // Acceptor methods + void onNewConnection( + apache::thrift::async::TAsyncSocket::UniquePtr sock, + const folly::SocketAddress* address, + const std::string& nextProtocol, + const TransportInfo& tinfo) override; + +private: + HTTPSessionAcceptor(const HTTPSessionAcceptor&) = delete; + HTTPSessionAcceptor& operator=(const HTTPSessionAcceptor&) = delete; + + // HTTPSession::InfoCallback methods + void onCreate(const HTTPSession&) override {} + void onIngressError(const HTTPSession&, ProxygenError error) override {} + void onRead(const HTTPSession&, size_t bytesRead) override {} + void onWrite(const HTTPSession&, size_t bytesWritten) override {} + void onRequestBegin(const HTTPSession&) override {} + void onRequestEnd(const HTTPSession&, + uint32_t maxIngressQueueSize) override {} + void onActivateConnection(const HTTPSession&) override {} + void onDeactivateConnection(const HTTPSession&) override {} + void onDestroy(const HTTPSession&) override {} + void onIngressMessage(const HTTPSession&, const HTTPMessage&) override {} + void onIngressLimitExceeded(const HTTPSession&) override {} + void onIngressPaused(const HTTPSession&) override {} + void onTransactionDetached(const HTTPSession&) override {} + void onPingReply(int64_t latency) override {} + void onSettingsOutgoingStreamsFull(const HTTPSession&) override {} + void onSettingsOutgoingStreamsNotFull(const HTTPSession&) override {} + + /** General-case error page generator */ + std::unique_ptr defaultErrorPage_; + + /** Generator of more detailed error pages for internal clients */ + std::unique_ptr diagnosticErrorPage_; + + folly::Optional alwaysUseSPDYVersion_{}; + + SimpleController simpleController_; + + /** + * 0.0.0.0:0, a valid address to use if getsockname() or getpeername() fails + */ + static const folly::SocketAddress unknownSocketAddress_; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPSessionController.h b/proxygen/lib/http/session/HTTPSessionController.h new file mode 100644 index 0000000000..f8273297e8 --- /dev/null +++ b/proxygen/lib/http/session/HTTPSessionController.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace folly { +class SocketAddress; +} + +namespace proxygen { + +class HTTPException; +class HTTPMessage; +class HTTPSession; +class HTTPTransaction; +class HTTPTransactionHandler; + +class HTTPSessionController { + public: + virtual ~HTTPSessionController() {} + + /** + * Will be invoked whenever HTTPSession successfully parses a + * request + * + * The controller creates a Handler for a new transaction. The + * transaction and HTTP message (request) are passed so the + * implementation can construct different handlers based on these. + * The transaction will be explicitly set on the handler later via + * setTransaction. The request message will be passed in + * onHeadersComplete. + */ + virtual HTTPTransactionHandler* getRequestHandler( + HTTPTransaction& txn, HTTPMessage* msg) = 0; + + /** + * Will be invoked when HTTPSession is unable to parse a new request + * on the connection because of bad input. + * + * error contains specific information about what went wrong + */ + virtual HTTPTransactionHandler* getParseErrorHandler( + HTTPTransaction* txn, + const HTTPException& error, + const folly::SocketAddress& localAddress) = 0; + + /** + * Will be invoked when HTTPSession times out parsing a new request. + */ + virtual HTTPTransactionHandler* getTransactionTimeoutHandler( + HTTPTransaction* txn, + const folly::SocketAddress& localAddress) = 0; + + /** + * Inform the controller it is associated with this particular session. + */ + virtual void attachSession(HTTPSession* session) = 0; + + /** + * Informed at the end when the given HTTPSession is going away. + */ + virtual void detachSession(const HTTPSession* session) = 0; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPSessionStats.h b/proxygen/lib/http/session/HTTPSessionStats.h new file mode 100644 index 0000000000..892cc903e3 --- /dev/null +++ b/proxygen/lib/http/session/HTTPSessionStats.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/TTLBAStats.h" + +namespace proxygen { + +// This may be retired with a byte events refactor +class HTTPSessionStats : public TTLBAStats { + public: + virtual ~HTTPSessionStats() noexcept {} + + virtual void recordTransactionOpened() noexcept = 0; + virtual void recordTransactionClosed() noexcept = 0; +}; + +} diff --git a/proxygen/lib/http/session/HTTPTransaction.cpp b/proxygen/lib/http/session/HTTPTransaction.cpp new file mode 100644 index 0000000000..ce040252ed --- /dev/null +++ b/proxygen/lib/http/session/HTTPTransaction.cpp @@ -0,0 +1,937 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPTransaction.h" + +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/codec/SPDYConstants.h" +#include "proxygen/lib/http/session/HTTPSessionStats.h" + +#include +#include + +using apache::thrift::async::TAsyncTimeoutSet; +using folly::IOBuf; +using std::unique_ptr; + +namespace proxygen { + +uint64_t HTTPTransaction::egressBodySizeLimit_ = 4096; +uint64_t HTTPTransaction::egressBufferLimit_ = 8192; + +HTTPTransaction::HTTPTransaction(TransportDirection direction, + HTTPCodec::StreamID id, + uint32_t seqNo, + Transport& transport, + PriorityQueue& egressQueue, + TAsyncTimeoutSet* transactionTimeouts, + HTTPSessionStats* stats, + bool useFlowControl, + uint32_t receiveInitialWindowSize, + uint32_t sendInitialWindowSize, + int8_t priority, + HTTPCodec::StreamID assocId): + deferredEgressBody_(folly::IOBufQueue::cacheChainLength()), + direction_(direction), + id_(id), + seqNo_(seqNo), + transport_(transport), + transactionTimeouts_(transactionTimeouts), + stats_(stats), + recvWindow_(receiveInitialWindowSize), + sendWindow_(sendInitialWindowSize), + egressQueue_(egressQueue), + assocStreamId_(assocId), + priority_(priority << spdy::SPDY_PRIO_SHIFT_FACTOR), + ingressPaused_(false), + egressPaused_(false), + handlerEgressPaused_(false), + useFlowControl_(useFlowControl), + aborted_(false), + deleting_(false), + enqueued_(false), + firstByteSent_(false), + firstHeaderByteSent_(false), + inResume_(false), + inActiveSet_(true) { + + if (assocStreamId_) { + if (isUpstream()) { + egressState_ = HTTPTransactionEgressSM::State::SendingDone; + } else { + ingressState_ = HTTPTransactionIngressSM::State::ReceivingDone; + } + } + + refreshTimeout(); + if (stats_) { + stats_->recordTransactionOpened(); + } +} + +HTTPTransaction::~HTTPTransaction() { + if (stats_) { + stats_->recordTransactionClosed(); + } + if (isEnqueued()) { + dequeue(); + } +} + +void HTTPTransaction::onIngressHeadersComplete( + std::unique_ptr msg) { + msg->setSeqNo(seqNo_); + if (transportCallback_) { + transportCallback_->headerBytesReceived(msg->getIngressHeaderSize()); + } + if (isUpstream() && !isPushed()) { + lastResponseStatus_ = msg->getStatusCode(); + } + if (!validateIngressStateTransition( + HTTPTransactionIngressSM::Event::onHeaders)) { + return; + } + if (mustQueueIngress()) { + checkCreateDeferredIngress(); + deferredIngress_->emplace(id_, HTTPEvent::Type::HEADERS_COMPLETE, + std::move(msg)); + VLOG(4) << *this << " Queued ingress event of type " << + HTTPEvent::Type::HEADERS_COMPLETE; + } else { + processIngressHeadersComplete(std::move(msg)); + } +} + +void HTTPTransaction::processIngressHeadersComplete( + std::unique_ptr msg) { + CallbackGuard guard(*this); + if (aborted_) { + return; + } + refreshTimeout(); + if (handler_ && !isIngressComplete()) { + handler_->onHeadersComplete(std::move(msg)); + } +} + +void HTTPTransaction::onIngressBody(unique_ptr chain) { + if (isIngressEOMSeen()) { + sendAbort(ErrorCode::STREAM_CLOSED); + return; + } + auto len = chain->computeChainDataLength(); + if (len == 0) { + return; + } + if (transportCallback_) { + transportCallback_->bodyBytesReceived(len); + } + if (!validateIngressStateTransition( + HTTPTransactionIngressSM::Event::onBody)) { + return; + } + if (mustQueueIngress()) { + // register the bytes in the receive window + if (!recvWindow_.reserve(len, useFlowControl_)) { + sendAbort(ErrorCode::FLOW_CONTROL_ERROR); + } else { + checkCreateDeferredIngress(); + deferredIngress_->emplace(id_, HTTPEvent::Type::BODY, + std::move(chain)); + VLOG(4) << *this << " Queued ingress event of type " << + HTTPEvent::Type::BODY << " size=" << len; + } + } else { + processIngressBody(std::move(chain), len); + } +} + +void HTTPTransaction::processIngressBody(unique_ptr chain, size_t len) { + CallbackGuard guard(*this); + if (aborted_) { + return; + } + refreshTimeout(); + transport_.notifyIngressBodyProcessed(len); + if (handler_) { + if (!isIngressComplete()) { + handler_->onBody(std::move(chain)); + } + + if (useFlowControl_ && !isIngressEOMSeen()) { + recvToAck_ += len; + if (recvToAck_ > 0) { + uint32_t divisor = 2; + if (transport_.isDraining()) { + // only send window updates for draining transports when window is + // closed + divisor = 1; + } + if (recvToAck_ >= 0 && + uint32_t(recvToAck_) >= (recvWindow_.getCapacity() / divisor)) { + VLOG(4) << *this << " recv_window is " << recvWindow_.getSize() + << " / " << recvWindow_.getCapacity() << " after acking " + << recvToAck_; + transport_.sendWindowUpdate(this, recvToAck_); + recvToAck_ = 0; + } + } + } // else don't care about window updates + } +} + +void HTTPTransaction::onIngressChunkHeader(size_t length) { + if (!validateIngressStateTransition( + HTTPTransactionIngressSM::Event::onChunkHeader)) { + return; + } + if (mustQueueIngress()) { + checkCreateDeferredIngress(); + deferredIngress_->emplace(id_, HTTPEvent::Type::CHUNK_HEADER, length); + VLOG(4) << *this << " Queued ingress event of type " << + HTTPEvent::Type::CHUNK_HEADER << " size=" << length; + } else { + processIngressChunkHeader(length); + } +} + +void HTTPTransaction::processIngressChunkHeader(size_t length) { + CallbackGuard guard(*this); + if (aborted_) { + return; + } + refreshTimeout(); + if (handler_ && !isIngressComplete()) { + handler_->onChunkHeader(length); + } +} + +void HTTPTransaction::onIngressChunkComplete() { + if (!validateIngressStateTransition( + HTTPTransactionIngressSM::Event::onChunkComplete)) { + return; + } + if (mustQueueIngress()) { + checkCreateDeferredIngress(); + deferredIngress_->emplace(id_, HTTPEvent::Type::CHUNK_COMPLETE); + VLOG(4) << *this << " Queued ingress event of type " << + HTTPEvent::Type::CHUNK_COMPLETE; + } else { + processIngressChunkComplete(); + } +} + +void HTTPTransaction::processIngressChunkComplete() { + CallbackGuard guard(*this); + if (aborted_) { + return; + } + refreshTimeout(); + if (handler_ && !isIngressComplete()) { + handler_->onChunkComplete(); + } +} + +void HTTPTransaction::onIngressTrailers(unique_ptr trailers) { + if (!validateIngressStateTransition( + HTTPTransactionIngressSM::Event::onTrailers)) { + return; + } + if (mustQueueIngress()) { + checkCreateDeferredIngress(); + deferredIngress_->emplace(id_, HTTPEvent::Type::TRAILERS_COMPLETE, + std::move(trailers)); + VLOG(4) << *this << " Queued ingress event of type " << + HTTPEvent::Type::TRAILERS_COMPLETE; + } else { + processIngressTrailers(std::move(trailers)); + } +} + +void HTTPTransaction::processIngressTrailers(unique_ptr trailers) { + CallbackGuard guard(*this); + if (aborted_) { + return; + } + refreshTimeout(); + if (handler_ && !isIngressComplete()) { + handler_->onTrailers(std::move(trailers)); + } +} + +void HTTPTransaction::onIngressUpgrade(UpgradeProtocol protocol) { + if (!validateIngressStateTransition( + HTTPTransactionIngressSM::Event::onUpgrade)) { + return; + } + if (mustQueueIngress()) { + checkCreateDeferredIngress(); + deferredIngress_->emplace(id_, HTTPEvent::Type::UPGRADE, protocol); + VLOG(4) << *this << " Queued ingress event of type " << + HTTPEvent::Type::UPGRADE; + } else { + processIngressUpgrade(protocol); + } +} + +void HTTPTransaction::processIngressUpgrade(UpgradeProtocol protocol) { + CallbackGuard guard(*this); + if (aborted_) { + return; + } + if (handler_ && !isIngressComplete()) { + handler_->onUpgrade(protocol); + } +} + +void HTTPTransaction::onIngressEOM() { + if (isIngressEOMSeen()) { + // This can happen when HTTPSession calls onIngressEOF() + sendAbort(ErrorCode::STREAM_CLOSED); + return; + } + // TODO: change the codec to not give an EOM callback after a 100 response? + // We could then delete the below 'if' + if (isUpstream() && extraResponseExpected()) { + VLOG(4) << "Ignoring EOM on initial 100 response on " << *this; + return; + } + if (!validateIngressStateTransition( + HTTPTransactionIngressSM::Event::onEOM)) { + return; + } + if (mustQueueIngress()) { + checkCreateDeferredIngress(); + deferredIngress_->emplace(id_, HTTPEvent::Type::MESSAGE_COMPLETE); + VLOG(4) << *this << " Queued ingress event of type " << + HTTPEvent::Type::MESSAGE_COMPLETE; + } else { + processIngressEOM(); + } +} + +void HTTPTransaction::processIngressEOM() { + CallbackGuard guard(*this); + if (aborted_) { + return; + } + VLOG(4) << "ingress EOM on " << *this; + const bool wasComplete = isIngressComplete(); + if (!validateIngressStateTransition( + HTTPTransactionIngressSM::Event::eomFlushed)) { + return; + } + if (handler_) { + if (!wasComplete) { + handler_->onEOM(); + } + } else { + markEgressComplete(); + } + updateReadTimeout(); +} + +bool HTTPTransaction::isExpectingIngress() const { + return (!ingressPaused_ && + (!isIngressEOMSeen() || + (useFlowControl_ && sendWindow_.getSize() <= 0))); +} + +void HTTPTransaction::updateReadTimeout() { + if (isExpectingIngress()) { + refreshTimeout(); + } else { + cancelTimeout(); + } +} + +void HTTPTransaction::markIngressComplete() { + VLOG(4) << "Marking ingress complete on " << *this; + ingressState_ = HTTPTransactionIngressSM::State::ReceivingDone; + deferredIngress_.reset(); + cancelTimeout(); +} + +void HTTPTransaction::markEgressComplete() { + deferredEgressBody_.move(); + if (isEnqueued()) { + dequeue(); + } + egressState_ = HTTPTransactionEgressSM::State::SendingDone; +} + +bool HTTPTransaction::validateIngressStateTransition( + HTTPTransactionIngressSM::Event event) { + CallbackGuard guard(*this); + + if (!HTTPTransactionIngressSM::transit(ingressState_, event)) { + HTTPException ex(HTTPException::Direction::INGRESS_AND_EGRESS); + ex.setProxygenError(kErrorIngressStateTransition); + ex.setCodecStatusCode(ErrorCode::PROTOCOL_ERROR); + // This will invoke sendAbort() and also inform the handler of the + // error and detach the handler. + onError(ex); + return false; + } + return true; +} + +void HTTPTransaction::onError(const HTTPException& error) { + CallbackGuard guard(*this); + + const bool wasAborted = aborted_; // see comment below + const bool wasEgressComplete = isEgressComplete(); + const bool wasIngressComplete = isIngressComplete(); + auto notify = handler_; + + if (error.getProxygenError() == kErrorStreamAbort) { + DCHECK(error.getDirection() == + HTTPException::Direction::INGRESS_AND_EGRESS); + aborted_ = true; + } else if (error.hasCodecStatusCode()) { + DCHECK(error.getDirection() == + HTTPException::Direction::INGRESS_AND_EGRESS); + sendAbort(error.getCodecStatusCode()); + } + + switch (error.getDirection()) { + case HTTPException::Direction::INGRESS_AND_EGRESS: + markEgressComplete(); + markIngressComplete(); + if (wasEgressComplete && wasIngressComplete && + // We mark egress complete before we get acknowledgement of the + // write segment finishing successfully. + // TODO: instead of using CallbackGuard hacks to keep txn around, use + // an explicit callback function and set egress complete after last + // byte flushes (or egress error occurs), see #3912823 + (error.getProxygenError() != kErrorWriteTimeout || wasAborted)) { + notify = nullptr; + } + break; + case HTTPException::Direction::EGRESS: + markEgressComplete(); + if (wasEgressComplete) { + notify = nullptr; + } + break; + case HTTPException::Direction::INGRESS: + if (isIngressEOMSeen()) { + // Not an error + return; + } + markIngressComplete(); + if (wasIngressComplete) { + notify = nullptr; + } + break; + } + if (notify) { + notify->onError(error); + } +} + +void HTTPTransaction::onIngressTimeout() { + CallbackGuard guard(*this); + VLOG(4) << "ingress timeout on " << *this; + pauseIngress(); + markIngressComplete(); + if (handler_) { + HTTPException ex(HTTPException::Direction::INGRESS); + ex.setProxygenError(kErrorTimeout); + handler_->onError(ex); + } else { + markEgressComplete(); + } +} + +void HTTPTransaction::onIngressWindowUpdate(const uint32_t amount) { + if (!useFlowControl_) { + return; + } + CallbackGuard guard(*this); + VLOG(4) << *this << " Remote side ack'd " << amount << " bytes"; + updateReadTimeout(); + if (sendWindow_.free(amount)) { + notifyTransportPendingEgress(); + } else { + sendAbort(ErrorCode::FLOW_CONTROL_ERROR); + } +} + +void HTTPTransaction::onIngressSetSendWindow(const uint32_t newWindowSize) { + if (!useFlowControl_) { + return; + } + updateReadTimeout(); + if (sendWindow_.setCapacity(newWindowSize)) { + notifyTransportPendingEgress(); + } else { + sendAbort(ErrorCode::FLOW_CONTROL_ERROR); + } +} + +void HTTPTransaction::onEgressTimeout() { + CallbackGuard guard(*this); + VLOG(4) << "egress timeout on " << *this; + if (handler_) { + HTTPException ex(HTTPException::Direction::EGRESS); + ex.setProxygenError(kErrorTimeout); + handler_->onError(ex); + } else { + markEgressComplete(); + } +} + +void HTTPTransaction::onEgressHeaderFirstByte() { + CallbackGuard guard(*this); + if (transportCallback_) { + transportCallback_->firstHeaderByteFlushed(); + } +} + +void HTTPTransaction::onEgressBodyFirstByte() { + CallbackGuard guard(*this); + if (transportCallback_) { + transportCallback_->firstByteFlushed(); + } +} + +void HTTPTransaction::onEgressBodyLastByte() { + CallbackGuard guard(*this); + if (transportCallback_) { + transportCallback_->lastByteFlushed(); + } +} + +void HTTPTransaction::onEgressLastByteAck(std::chrono::milliseconds latency) { + CallbackGuard guard(*this); + if (transportCallback_) { + transportCallback_->lastByteAcked(latency); + } +} + +void HTTPTransaction::sendHeaders(const HTTPMessage& headers) { + CHECK(HTTPTransactionEgressSM::transit( + egressState_, HTTPTransactionEgressSM::Event::sendHeaders)); + DCHECK(!isEgressComplete()); + if (isDownstream() && !isPushed()) { + lastResponseStatus_ = headers.getStatusCode(); + } + HTTPHeaderSize size; + transport_.sendHeaders(this, headers, &size); + if (transportCallback_) { + transportCallback_->headerBytesGenerated(size); + } +} + +void HTTPTransaction::sendBody(std::unique_ptr body) { + CHECK(HTTPTransactionEgressSM::transit( + egressState_, HTTPTransactionEgressSM::Event::sendBody)); + deferredEgressBody_.append(std::move(body)); + notifyTransportPendingEgress(); +} + +bool HTTPTransaction::onWriteReady(const uint32_t maxEgress) { + CallbackGuard guard(*this); + DCHECK(isEnqueued()); + // this txn is being serviced so lower it's priority -> higher numerical value + priority_ |= 0x2; + sendDeferredBody(maxEgress); + return isEnqueued(); +} + +// Send up to maxEgress body bytes, including pendingEOM if appropriate +size_t HTTPTransaction::sendDeferredBody(const uint32_t maxEgress) { + const int32_t windowAvailable = sendWindow_.getSize(); + const uint32_t sendWindow = useFlowControl_ ? std::min( + maxEgress, windowAvailable > 0 ? windowAvailable : 0) : maxEgress; + + // We shouldn't be called if we have no pending body/EOM, egress is paused, or + // the send window is closed + CHECK((deferredEgressBody_.chainLength() > 0 || + isEgressEOMQueued()) && + !egressPaused_ && sendWindow > 0); + + const size_t bytesLeft = deferredEgressBody_.chainLength(); + size_t canSend = std::min(sendWindow, bytesLeft); + size_t curLen = 0; + size_t nbytes = 0; + bool willSendEOM = false; + + if (chunkHeaders_.empty()) { + // limit this txn to egressBodySizeLimit_ at a time + curLen = std::min(canSend, egressBodySizeLimit_); + std::unique_ptr body = deferredEgressBody_.split(curLen); + willSendEOM = hasPendingEOM(); + DCHECK(curLen > 0 || willSendEOM); + if (curLen > 0) { + if (willSendEOM) { + // we have to dequeue BEFORE sending the EOM =( + dequeue(); + } + nbytes = sendBodyNow(std::move(body), curLen, willSendEOM); + willSendEOM = false; + } // else we got called with only a pending EOM, handled below + } else { + // This body is expliticly chunked + while (!chunkHeaders_.empty() && canSend > 0) { + Chunk& chunk = chunkHeaders_.front(); + if (!chunk.headerSent) { + nbytes += transport_.sendChunkHeader(this, chunk.length); + chunk.headerSent = true; + } + canSend = std::min(canSend, egressBodySizeLimit_); + curLen = std::min(chunk.length, canSend); + std::unique_ptr cur = deferredEgressBody_.split(curLen); + VLOG(4) << "sending " << curLen << " fin=false"; + nbytes += sendBodyNow(std::move(cur), curLen, false); + canSend -= curLen; + chunk.length -= curLen; + if (chunk.length == 0) { + nbytes += transport_.sendChunkTerminator(this); + chunkHeaders_.pop_front(); + } else { + DCHECK(canSend == 0); + } + } + willSendEOM = hasPendingEOM(); + } + // Send any queued eom + if (willSendEOM) { + nbytes += sendEOMNow(); + } + + // Update the handler's pause state + notifyTransportPendingEgress(); + updateHandlerPauseState(); + + if (transportCallback_) { + transportCallback_->bodyBytesGenerated(nbytes); + } + return nbytes; +} + +size_t HTTPTransaction::sendEOMNow() { + size_t nbytes = 0; + VLOG(4) << "egress EOM on " << *this; + if (trailers_) { + VLOG(4) << "egress trailers on " << *this; + nbytes += transport_.sendTrailers(this, *trailers_.get()); + trailers_.reset(); + } + // TODO: with ByteEvent refactor, we will have to delay changing this + // state until later + CHECK(HTTPTransactionEgressSM::transit( + egressState_, HTTPTransactionEgressSM::Event::eomFlushed)); + nbytes += transport_.sendEOM(this); + return nbytes; +} + +size_t HTTPTransaction::sendBodyNow(std::unique_ptr body, + size_t bodyLen, bool sendEom) { + DCHECK(body); + DCHECK(bodyLen > 0); + size_t nbytes = 0; + VLOG(4) << *this << " Sending " << bodyLen << " bytes of body. eom=" + << ((sendEom) ? "yes" : "no"); + if (useFlowControl_) { + CHECK(sendWindow_.reserve(bodyLen)); + VLOG(4) << *this << " send_window is " + << sendWindow_.getSize() << " / " << sendWindow_.getCapacity(); + } + if (sendEom) { + CHECK(HTTPTransactionEgressSM::transit( + egressState_, HTTPTransactionEgressSM::Event::eomFlushed)); + } + updateReadTimeout(); + nbytes = transport_.sendBody(this, std::move(body), sendEom); + return nbytes; +} + +void +HTTPTransaction::sendEOM() { + CallbackGuard guard(*this); + CHECK(HTTPTransactionEgressSM::transit( + egressState_, HTTPTransactionEgressSM::Event::sendEOM)); + if (deferredEgressBody_.chainLength() == 0 && chunkHeaders_.empty()) { + // there is nothing left to send, egress the EOM directly. For SPDY + // this will jump the txn queue + if (!isEnqueued()) { + size_t nbytes = sendEOMNow(); + transport_.notifyPendingEgress(); + if (transportCallback_) { + transportCallback_->bodyBytesGenerated(nbytes); + } + } else { + // If the txn is enqueued, sendDeferredBody() + // should take care of sending the EOM. + // Nevertheless we never expect this condition to occur, + // so, log. + LOG(ERROR) << "Queued egress EOM with no body on " + << *this + << "[egressState=" << egressState_ << ", " + << "ingressState=" << ingressState_ << ", " + << "egressPaused=" << egressPaused_ << ", " + << "ingressPaused=" << ingressPaused_ << ", " + << "aborted=" << aborted_ << ", " + << "enqueued=" << enqueued_ << ", " + << "chainLength=" << deferredEgressBody_.chainLength() << "]"; + } + } else { + VLOG(4) << "Queued egress EOM on " << *this; + notifyTransportPendingEgress(); + } + if (ingressPaused_ && !isIngressComplete()) { + resumeIngress(); + } +} + +void HTTPTransaction::sendAbort() { + sendAbort(isUpstream() ? ErrorCode::CANCEL + : ErrorCode::PROTOCOL_ERROR); +} + +void HTTPTransaction::sendAbort(ErrorCode statusCode) { + CallbackGuard guard(*this); + markIngressComplete(); + markEgressComplete(); + if (aborted_) { + // This can happen in cases where the abort is sent before notifying the + // handler, but its logic also wants to abort + VLOG(4) << "skipping redundant abort"; + return; + } + VLOG(4) << "aborting transaction " << *this; + aborted_ = true; + size_t nbytes = transport_.sendAbort(this, statusCode); + if (transportCallback_) { + HTTPHeaderSize size; + size.uncompressed = nbytes; + transportCallback_->headerBytesGenerated(size); + } +} + +void +HTTPTransaction::checkForCompletion() { + DCHECK(callbackDepth_ == 0); + if (deleting_) { + return; + } + if (isEgressComplete() && isIngressComplete() && !isEnqueued()) { + VLOG(4) << "destroying transaction " << *this; + deleting_ = true; + if (handler_) { + handler_->detachTransaction(); + handler_ = nullptr; + } + transportCallback_ = nullptr; + const auto bytesBuffered = recvWindow_.getOutstanding(); + if (bytesBuffered) { + transport_.notifyIngressBodyProcessed(bytesBuffered); + } + transport_.detach(this); + delete this; + } +} + +void HTTPTransaction::pauseIngress() { + VLOG(4) << *this << " pauseIngress request"; + CallbackGuard guard(*this); + if (ingressPaused_) { + VLOG(4) << *this << " can't pause ingress; ingressPaused=" << + ingressPaused_; + return; + } + ingressPaused_ = true; + cancelTimeout(); + transport_.pauseIngress(this); +} + +void HTTPTransaction::resumeIngress() { + VLOG(4) << *this << " resumeIngress request"; + CallbackGuard guard(*this); + if (!ingressPaused_ || isIngressComplete()) { + VLOG(4) << *this << " can't resume ingress; ingressPaused=" + << ingressPaused_ << ", ingressComplete=" + << isIngressComplete() << " inResume_=" << inResume_; + return; + } + ingressPaused_ = false; + transport_.resumeIngress(this); + if (inResume_) { + VLOG(4) << *this << " skipping recursive resume loop"; + return; + } + inResume_ = true; + + if (deferredIngress_ && (maxDeferredIngress_ <= deferredIngress_->size())) { + maxDeferredIngress_ = deferredIngress_->size(); + } + + // Process any deferred ingress callbacks + // Note: we recheck the ingressPaused_ state because a callback + // invoked by the resumeIngress() call above could have re-paused + // the transaction. + while (!ingressPaused_ && deferredIngress_ && !deferredIngress_->empty()) { + HTTPEvent& callback(deferredIngress_->front()); + VLOG(4) << *this << " Processing deferred ingress callback of type " << + callback.getEvent(); + switch (callback.getEvent()) { + case HTTPEvent::Type::MESSAGE_BEGIN: + LOG(FATAL) << "unreachable"; + break; + case HTTPEvent::Type::HEADERS_COMPLETE: + processIngressHeadersComplete(std::move(callback.getHeaders())); + break; + case HTTPEvent::Type::BODY: { + unique_ptr data = callback.getBody(); + auto len = data->computeChainDataLength(); + CHECK(recvWindow_.free(len)); + processIngressBody(std::move(data), len); + } break; + case HTTPEvent::Type::CHUNK_HEADER: + processIngressChunkHeader(callback.getChunkLength()); + break; + case HTTPEvent::Type::CHUNK_COMPLETE: + processIngressChunkComplete(); + break; + case HTTPEvent::Type::TRAILERS_COMPLETE: + processIngressTrailers(callback.getTrailers()); + break; + case HTTPEvent::Type::MESSAGE_COMPLETE: + processIngressEOM(); + break; + case HTTPEvent::Type::UPGRADE: + processIngressUpgrade(callback.getUpgradeProtocol()); + break; + } + if (deferredIngress_) { + deferredIngress_->pop(); + } + } + updateReadTimeout(); + inResume_ = false; +} + +void HTTPTransaction::pauseEgress() { + VLOG(4) << *this << " asked to pause egress"; + CallbackGuard guard(*this); + if (egressPaused_) { + VLOG(4) << *this << " egress already paused"; + return; + } + egressPaused_ = true; + notifyTransportPendingEgress(); +} + +void HTTPTransaction::resumeEgress() { + VLOG(4) << *this << " asked to resume egress"; + CallbackGuard guard(*this); + if (!egressPaused_) { + VLOG(4) << *this << " egress already not paused"; + return; + } + egressPaused_ = false; + notifyTransportPendingEgress(); +} + +void HTTPTransaction::notifyTransportPendingEgress() { + if (!egressPaused_ && + (deferredEgressBody_.chainLength() > 0 || + isEgressEOMQueued()) && + (!useFlowControl_ || sendWindow_.getSize() > 0)) { + if (isEnqueued()) { + // We're already in the queue, jiggle our priority + priority_ ^= 0x3; + egressQueue_.update(queueHandle_); + } else { + // Insert into the queue and let the session know we've got something + queueHandle_ = egressQueue_.push(this); + enqueued_ = true; + transport_.notifyPendingEgress(); + } + } else if (isEnqueued()) { + dequeue(); + } + updateHandlerPauseState(); +} + +void HTTPTransaction::updateHandlerPauseState() { + bool handlerShouldBePaused = egressPaused_ || + (useFlowControl_ && sendWindow_.getSize() <= 0) || + (deferredEgressBody_.chainLength() >= egressBufferLimit_); + if (handler_ && handlerShouldBePaused != handlerEgressPaused_) { + if (handlerShouldBePaused) { + handlerEgressPaused_ = true; + handler_->onEgressPaused(); + } else { + handlerEgressPaused_ = false; + handler_->onEgressResumed(); + } + } +} + +bool HTTPTransaction::mustQueueIngress() const { + return ingressPaused_ || (deferredIngress_ && !deferredIngress_->empty()); +} + +void HTTPTransaction::checkCreateDeferredIngress() { + if (!deferredIngress_) { + deferredIngress_ = folly::make_unique>(); + } +} + +bool HTTPTransaction::onPushedTransaction(HTTPTransaction* pushTxn) { + CallbackGuard guard(*this); + CHECK(pushTxn->assocStreamId_ == id_); + if (!handler_) { + VLOG(1) << "Cannot add a pushed txn to an unhandled txn"; + return false; + } + handler_->onPushedTransaction(pushTxn); + if (!pushTxn->getHandler()) { + VLOG(1) << "Failed to create a handler for push transaction"; + return false; + } + pushedTransactions_.insert(pushTxn->getID()); + return true; +} + +void +HTTPTransaction::describe(std::ostream& os) const { + transport_.describe(os); + os << " streamID=" << id_; +} + +/* + * TODO: when HTTPSession sends a SETTINGS frame indicating a + * different initial window, it should call this function on all its + * transactions. + */ +void HTTPTransaction::setReceiveWindow(uint32_t capacity) { + // Depending on whether delta is positive or negative it will cause the + // window to either increase or decrease. + int32_t delta = capacity - recvWindow_.getCapacity(); + if (!recvWindow_.setCapacity(capacity)) { + return; + } + recvToAck_ += delta; + if (recvToAck_ > 0) { + transport_.sendWindowUpdate(this, recvToAck_); + recvToAck_ = 0; + } +} + +std::ostream& +operator<<(std::ostream& os, const HTTPTransaction& txn) { + txn.describe(os); + return os; +} + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPTransaction.h b/proxygen/lib/http/session/HTTPTransaction.h new file mode 100644 index 0000000000..fbb9ff7502 --- /dev/null +++ b/proxygen/lib/http/session/HTTPTransaction.h @@ -0,0 +1,1155 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPConstants.h" +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/http/ProxygenErrorEnum.h" +#include "proxygen/lib/http/Window.h" +#include "proxygen/lib/http/codec/HTTPCodec.h" +#include "proxygen/lib/http/session/HTTPEvent.h" +#include "proxygen/lib/http/session/HTTPTransactionEgressSM.h" +#include "proxygen/lib/http/session/HTTPTransactionIngressSM.h" +#include "proxygen/lib/services/TransportInfo.h" + +#include +#include +#include +#include +#include + +namespace proxygen { + +/** + * An HTTPTransaction represents a single request/response pair + * for some HTTP-like protocol. It works with a Transport that + * performs the network processing and wire-protocol formatting + * and a Handler that implements some sort of application logic. + * + * The typical sequence of events for a simple application is: + * + * * The application accepts a connection and creates a Transport. + * * The Transport reads from the connection, parses whatever + * protocol the client is speaking, and creates a Transaction + * to represent the first request. + * * Once the Transport has received the full request headers, + * it creates a Handler, plugs the handler into the Transaction, + * and calls the Transaction's onIngressHeadersComplete() method. + * * The Transaction calls the Handler's onHeadersComplete() method + * and the Handler begins processing the request. + * * If there is a request body, the Transport streams it through + * the Transaction to the Handler. + * * When the Handler is ready to produce a response, it streams + * the response through the Transaction to the Transport. + * * When the Transaction has seen the end of both the request + * and the response, it detaches itself from the Handler and + * Transport and deletes itself. + * * The Handler deletes itself at some point after the Transaction + * has detached from it. + * * The Transport may, depending on the protocol, process other + * requests after -- or even in parallel with -- that first + * request. Each request gets its own Transaction and Handler. + * + * For some applications, like proxying, a Handler implementation + * may obtain one or more upstream connections, each represented + * by another Transport, and create outgoing requests on the upstream + * connection(s), with each request represented as a new Transaction. + * + * With a multiplexing protocol like SPDY on both sides of a proxy, + * the cardinality relationship can be: + * + * +-----------+ +-----------+ +-------+ + * (Client-side) | Transport |1---*|Transaction|1---1|Handler| + * +-----------+ +-----------+ +-------+ + * 1 + * | + * | + * 1 + * +---------+ +-----------+ + * (Server-side) |Transport|1---*|Transaction| + * +---------+ +-----------+ + * + * A key design goal of HTTPTransaction is to serve as a protocol- + * independent abstraction that insulates Handlers from the semantics + * different of HTTP-like protocols. + */ + +class HTTPSessionStats; +class HTTPTransaction; +class HTTPTransactionHandler { + public: + + /** + * Called once per transaction. This notifies the handler of which + * transaction it should talk to and will receive callbacks from. + */ + virtual void setTransaction(HTTPTransaction* txn) noexcept = 0; + + /** + * Called once after a transaction successfully completes. It + * will be called even if a read or write error happened earlier. + * This is a terminal callback, which means that the HTTPTransaction + * object that gives this call will be invalid after this function + * completes. + */ + virtual void detachTransaction() noexcept = 0; + + /** + * Called at most once per transaction. This is usually the first + * ingress callback. It is possible to get a read error before this + * however. If you had previously called pauseIngress(), this callback + * will be delayed until you call resumeIngress(). + */ + virtual void onHeadersComplete(std::unique_ptr msg) noexcept = 0; + + /** + * Can be called multiple times per transaction. If you had previously + * called pauseIngress(), this callback will be delayed until you call + * resumeIngress(). + */ + virtual void onBody(std::unique_ptr chain) noexcept = 0; + + /** + * Can be called multiple times per transaction. If you had previously + * called pauseIngress(), this callback will be delayed until you call + * resumeIngress(). This signifies the beginning of a chunk of length + * 'length'. You will receive onBody() after this. Also, the length will + * be greater than zero. + */ + virtual void onChunkHeader(size_t length) noexcept {}; + + /** + * Can be called multiple times per transaction. If you had previously + * called pauseIngress(), this callback will be delayed until you call + * resumeIngress(). This signifies the end of a chunk. + */ + virtual void onChunkComplete() noexcept {}; + + /** + * Can be called any number of times per transaction. If you had + * previously called pauseIngress(), this callback will be delayed until + * you call resumeIngress(). Trailers can be received once right before + * the EOM of a chunked HTTP/1.1 reponse or multiple times per + * transaction from SPDY and HTTP/2.0 HEADERS frames. + */ + virtual void onTrailers(std::unique_ptr trailers) noexcept + = 0; + + /** + * Can be called once per transaction. If you had previously called + * pauseIngress(), this callback will be delayed until you call + * resumeIngress(). After this callback is received, there will be no + * more normal ingress callbacks received (onEgress*() and onError() + * may still be invoked). The Handler should consider + * ingress complete after receiving this message. This Transaction is + * still valid, and work may still occur on it until detachTransaction + * is called. + */ + virtual void onEOM() noexcept = 0; + + /** + * Can be called once per transaction. If you had previously called + * pauseIngress(), this callback will be delayed until you call + * resumeIngress(). After this callback is invoked, further data + * will be forwarded using the onBody() callback. Once the data transfer + * is completed (EOF recevied in case of CONNECT), onEOM() callback will + * be invoked. + */ + virtual void onUpgrade(UpgradeProtocol protocol) noexcept = 0; + + /** + * Can be called at any time before detachTransaction(). This callback + * implies that an error has occurred. To determine if ingress or egress + * is affected, check the direciont on the HTTPException. If the + * direction is INGRESS, it MAY still be possible to send egress. + */ + virtual void onError(const HTTPException& error) noexcept = 0; + + /** + * If the remote side's receive buffer fills up, this callback will be + * invoked so you can attempt to stop sending to the remote side. + */ + virtual void onEgressPaused() noexcept = 0; + + /** + * This callback lets you know that the remote side has resumed reading + * and you can now continue to send data. + */ + virtual void onEgressResumed() noexcept = 0; + + /** + * Ask the handler to construct a handler for a pushed transaction associated + * with its transaction. + * + * TODO: Reconsider default implementation here. If the handler + * does not implement, better set max initiated to 0 in a settings frame? + */ + virtual void onPushedTransaction(HTTPTransaction* txn) noexcept {} + + virtual ~HTTPTransactionHandler() {} +}; + +class HTTPPushTransactionHandler : public HTTPTransactionHandler { + public: + virtual ~HTTPPushTransactionHandler() {} + + void onHeadersComplete(std::unique_ptr msg) noexcept final { + LOG(FATAL) << "push txn received headers"; + } + + void onBody(std::unique_ptr chain) noexcept final { + LOG(FATAL) << "push txn received body"; + } + + void onChunkHeader(size_t length) noexcept final { + LOG(FATAL) << "push txn received chunk header"; + } + + void onChunkComplete() noexcept final { + LOG(FATAL) << "push txn received chunk complete"; + } + + void onTrailers(std::unique_ptr trailers) + noexcept final { + LOG(FATAL) << "push txn received trailers"; + } + + void onEOM() noexcept final { + LOG(FATAL) << "push txn received EOM"; + } + + void onUpgrade(UpgradeProtocol protocol) noexcept final { + LOG(FATAL) << "push txn received upgrade"; + } + + void onPushedTransaction(HTTPTransaction* txn) noexcept final { + LOG(FATAL) << "push txn received push txn"; + } +}; + +class HTTPTransaction : + public apache::thrift::async::TAsyncTimeoutSet::Callback { + public: + typedef HTTPTransactionHandler Handler; + typedef HTTPPushTransactionHandler PushHandler; + + class Transport { + public: + virtual ~Transport() {} + + virtual void pauseIngress(HTTPTransaction* txn) noexcept = 0; + + virtual void resumeIngress(HTTPTransaction* txn) noexcept = 0; + + virtual void transactionTimeout(HTTPTransaction* txn) noexcept = 0; + + virtual void sendHeaders(HTTPTransaction* txn, + const HTTPMessage& headers, + HTTPHeaderSize* size) noexcept = 0; + + virtual size_t sendBody(HTTPTransaction* txn, + std::unique_ptr, + bool eom) noexcept = 0; + + virtual size_t sendChunkHeader(HTTPTransaction* txn, + size_t length) noexcept = 0; + + virtual size_t sendChunkTerminator(HTTPTransaction* txn) noexcept = 0; + + virtual size_t sendTrailers(HTTPTransaction* txn, + const HTTPHeaders& trailers) noexcept = 0; + + virtual size_t sendEOM(HTTPTransaction* txn) noexcept = 0; + + virtual size_t sendAbort(HTTPTransaction* txn, + ErrorCode statusCode) noexcept = 0; + + virtual size_t sendWindowUpdate(HTTPTransaction* txn, + uint32_t bytes) noexcept = 0; + + virtual void notifyPendingEgress() noexcept = 0; + + virtual void detach(HTTPTransaction* txn) noexcept = 0; + + virtual void notifyIngressBodyProcessed(uint32_t bytes) noexcept = 0; + + virtual const folly::SocketAddress& getLocalAddress() + const noexcept = 0; + + virtual const folly::SocketAddress& getPeerAddress() + const noexcept = 0; + + virtual void describe(std::ostream&) const = 0; + + virtual const TransportInfo& getSetupTransportInfo() const noexcept = 0; + + virtual bool getCurrentTransportInfo(TransportInfo* tinfo) = 0; + + virtual const HTTPCodec& getCodec() const noexcept = 0; + + virtual bool isDraining() const = 0; + + virtual HTTPTransaction* newPushedTransaction( + HTTPCodec::StreamID assocStreamId, + HTTPTransaction::PushHandler* handler, + int8_t priority) noexcept = 0; + }; + + /** + * Callback interface to be notified of events on the byte stream. + */ + class TransportCallback { + public: + virtual void firstHeaderByteFlushed() noexcept = 0; + + virtual void firstByteFlushed() noexcept = 0; + + virtual void lastByteFlushed() noexcept = 0; + + virtual void lastByteAcked(std::chrono::milliseconds latency) noexcept = 0; + + virtual void headerBytesGenerated(HTTPHeaderSize& size) noexcept = 0; + + virtual void headerBytesReceived(const HTTPHeaderSize& size) noexcept = 0; + + virtual void bodyBytesGenerated(uint32_t nbytes) noexcept = 0; + + virtual void bodyBytesReceived(uint32_t size) noexcept = 0; + + virtual ~TransportCallback() {}; + }; + + struct LessP { + bool operator()(const HTTPTransaction* left, + const HTTPTransaction* right) const { + // larger values are logically smaller + return left->priority_ > right->priority_; + } + }; + + typedef boost::heap::d_ary_heap, + boost::heap::stable, + boost::heap::compare, + boost::heap::mutable_, + boost::heap::constant_time_size + > PriorityQueue; + + /** + * readBufLimit and sendWindow are only used if useFlowControl is + * true. Furthermore, if flow control is enabled, no guarantees can be + * made on the borders of the L7 chunking/data frames of the outbound + * messages. + * + * priority is only used by SPDY. The -1 default makes sure that all + * plain HTTP transactions land up in the same queue as the control data. + */ + HTTPTransaction(TransportDirection direction, + HTTPCodec::StreamID id, + uint32_t seqNo, + Transport& transport, + PriorityQueue& egressQueue, + apache::thrift::async::TAsyncTimeoutSet* transactionTimeouts, + HTTPSessionStats* stats = nullptr, + bool useFlowControl = false, + uint32_t receiveInitialWindowSize = 0, + uint32_t sendInitialWindowSize = 0, + int8_t priority = -1, + HTTPCodec::StreamID assocStreamId = 0); + + virtual ~HTTPTransaction() override; + + HTTPCodec::StreamID getID() const { return id_; } + + uint32_t getSequenceNumber() const { return seqNo_; } + + const Transport& getTransport() const { return transport_; } + + Transport& getTransport() { return transport_; } + + void setHandler(Handler* handler) { + handler_ = handler; + if (handler_) { + handler_->setTransaction(this); + } + } + + const Handler* getHandler() const { + return handler_; + } + + uint32_t getPriority() const { + return priority_; + } + + HTTPTransactionEgressSM::State getEgressState() const { + return egressState_; + } + + HTTPTransactionIngressSM::State getIngressState() const { + return ingressState_; + } + + bool isUpstream() const { + return direction_ == TransportDirection::UPSTREAM; + } + + bool isDownstream() const { + return direction_ == TransportDirection::DOWNSTREAM; + } + + void getLocalAddress(folly::SocketAddress& addr) const { + addr = transport_.getLocalAddress(); + } + + void getPeerAddress(folly::SocketAddress& addr) const { + addr = transport_.getPeerAddress(); + } + + const folly::SocketAddress& getLocalAddress() + const noexcept { + return transport_.getLocalAddress(); + } + + const folly::SocketAddress& getPeerAddress() + const noexcept { + return transport_.getPeerAddress(); + } + + const TransportInfo& getSetupTransportInfo() const noexcept { + return transport_.getSetupTransportInfo(); + } + + void getCurrentTransportInfo(TransportInfo* tinfo) const { + transport_.getCurrentTransportInfo(tinfo); + } + + /** + * Check whether more response is expected. One or more 1xx status + * responses can be received prior to the regular response. + * Note: 101 is handled by the codec using a separate onUpgrade callback + */ + virtual bool extraResponseExpected() const { + return (lastResponseStatus_ >= 100 && lastResponseStatus_ < 200) + && lastResponseStatus_ != 101; + } + + /** + * Change the size of the receive window and propagate the change to the + * remote end using a window update. + * + * TODO: when HTTPSession sends a SETTINGS frame indicating a + * different initial window, it should call this function on all its + * transactions. + */ + virtual void setReceiveWindow(uint32_t capacity); + + /** + * Get the receive window of the transaction + */ + virtual const Window& getReceiveWindow() const { + return recvWindow_; + } + + uint32_t getMaxDeferredSize() { + return maxDeferredIngress_; + } + + /** + * Invoked by the session when the ingress headers are complete + */ + void onIngressHeadersComplete(std::unique_ptr msg); + + /** + * Invoked by the session when some or all of the ingress entity-body has + * been parsed. + */ + void onIngressBody(std::unique_ptr chain); + + /** + * Invoked by the session when a chunk header has been parsed. + */ + void onIngressChunkHeader(size_t length); + + /** + * Invoked by the session when the CRLF terminating a chunk has been parsed. + */ + void onIngressChunkComplete(); + + /** + * Invoked by the session when the ingress trailers have been parsed. + */ + void onIngressTrailers(std::unique_ptr trailers); + + /** + * Invoked by the session when the session and transaction need to be + * upgraded to a different protocol + */ + void onIngressUpgrade(UpgradeProtocol protocol); + + /** + * Invoked by the session when the ingress message is complete. + */ + void onIngressEOM(); + + /** + * Invoked by the session when there is an error (e.g., invalid syntax, + * TCP RST) in either the ingress or egress stream. Note that this + * message is processed immediately even if this transaction normally + * would queue ingress. + * + * @param error Details for the error. This exception also has + * information about whether the error applies to the ingress, egress, + * or both directions of the transaction + */ + void onError(const HTTPException& error); + + /** + * Invoked by the session when there is a timeout on the ingress stream. + * Note that each transaction has its own timer but the session + * is the effective target of the timer. + */ + void onIngressTimeout(); + + /** + * Invoked by the session when the remote endpoint of this transaction + * signals that it has consumed 'amount' bytes. This is only for + * versions of HTTP that support per transaction flow control. + */ + void onIngressWindowUpdate(uint32_t amount); + + /** + * Invoked by the session when the remote endpoint signals that we + * should change our send window. This is only for + * versions of HTTP that support per transaction flow control. + */ + void onIngressSetSendWindow(uint32_t newWindowSize); + + /** + * Notify this transaction that it is ok to egress. Returns true if there + * is additional pending egress + */ + bool onWriteReady(uint32_t maxEgress); + + /** + * Invoked by the session when there is a timeout on the egress stream. + */ + void onEgressTimeout(); + + /** + * Invoked by the session when the first header byte is flushed. + */ + void onEgressHeaderFirstByte(); + + /** + * Invoked by the session when the first byte is flushed. + */ + void onEgressBodyFirstByte(); + + /** + * Invoked by the session when the first byte is flushed. + */ + void onEgressBodyLastByte(); + + /** + * Invoked when the ACK_LATENCY event is delivered + * + * @param latency the time between the moment when the last byte was sent + * and the moment when we received the ACK from the client + */ + void onEgressLastByteAck(std::chrono::milliseconds latency); + + /** + * Invoked by the handlers that are interested in tracking + * performance stats. + */ + void setTransportCallback(TransportCallback* cb) { + transportCallback_ = cb; + } + + /** + * @return true if egress has started on this transaction. + */ + bool isIngressStarted() const { + return ingressState_ != HTTPTransactionIngressSM::State::Start; + } + + /** + * @return true iff the ingress EOM has been queued in HTTPTransaction + * but the handler has not yet been notified of this event. + */ + bool isIngressEOMQueued() const { + return ingressState_ == HTTPTransactionIngressSM::State::EOMQueued; + } + + /** + * @return true iff the handler has been notified of the ingress EOM. + */ + bool isIngressComplete() const { + return ingressState_ == HTTPTransactionIngressSM::State::ReceivingDone; + } + + /** + * @return true iff onIngressEOM() has been called. + */ + bool isIngressEOMSeen() const { + return isIngressEOMQueued() || isIngressComplete(); + } + + /** + * @return true if egress has started on this transaction. + */ + bool isEgressStarted() const { + return egressState_ != HTTPTransactionEgressSM::State::Start; + } + + /** + * @return true iff sendEOM() has been called, but the eom has not been + * flushed to the socket yet. + */ + bool isEgressEOMQueued() const { + return egressState_ == HTTPTransactionEgressSM::State::EOMQueued; + } + + /** + * @return true iff the egress EOM has been flushed to the socket. + */ + bool isEgressComplete() const { + return egressState_ == HTTPTransactionEgressSM::State::SendingDone; + } + + /** + * @return true iff sendEOM() has been called. + */ + bool isEgressEOMSeen() const { + return isEgressEOMQueued() || isEgressComplete(); + } + + /** + * @return true if we can send headers on this transaction + */ + bool canSendHeaders() const { + return HTTPTransactionEgressSM::canTransit( + egressState_, + HTTPTransactionEgressSM::Event::sendHeaders) + && !isEgressComplete(); + } + + /** + * Send the egress message headers to the Transport. This method does + * not actually write the message out on the wire immediately. All + * writes happen at the end of the event loop at the earliest. + * Note: This method should be called once per message unless the first + * headers sent indicate a 1xx status. + * + * @param headers Message headers + */ + virtual void sendHeaders(const HTTPMessage& headers); + + /** + * Send part or all of the egress message body to the Transport. If flow + * control is enabled, the chunk boundaries may not be respected. + * This method does not actually write the message out on the wire + * immediately. All writes happen at the end of the event loop at the + * earliest. + * Note: This method may be called zero or more times per message. + * + * @param body Message body data; the Transport will take care of + * applying any necessary protocol framing, such as + * chunk headers. + */ + virtual void sendBody(std::unique_ptr body); + + /** + * Write any protocol framing required for the subsequent call(s) + * to sendBody(). This method does not actually write the message out on + * the wire immediately. All writes happen at the end of the event loop + * at the earliest. + * @param length Length in bytes of the body data to follow. + */ + virtual void sendChunkHeader(size_t length) { + CHECK(HTTPTransactionEgressSM::transit( + egressState_, HTTPTransactionEgressSM::Event::sendChunkHeader)); + // TODO: move this logic down to session/codec + if (!transport_.getCodec().supportsParallelRequests()) { + chunkHeaders_.emplace_back(Chunk(length)); + } + } + + /** + * Write any protocol syntax needed to terminate the data. This method + * does not actually write the message out on the wire immediately. All + * writes happen at the end of the event loop at the earliest. + * Frame begun by the last call to sendChunkHeader(). + */ + virtual void sendChunkTerminator() { + CHECK(HTTPTransactionEgressSM::transit( + egressState_, HTTPTransactionEgressSM::Event::sendChunkTerminator)); + } + + /** + * Send message trailers to the Transport. This method does + * not actually write the message out on the wire immediately. All + * writes happen at the end of the event loop at the earliest. + * Note: This method may be called at most once per message. + * + * @param trailers Message trailers. + */ + virtual void sendTrailers(const HTTPHeaders& trailers) { + CHECK(HTTPTransactionEgressSM::transit( + egressState_, HTTPTransactionEgressSM::Event::sendTrailers)); + if (transport_.getCodec().supportsParallelRequests()) { + // SPDY supports trailers whenever + size_t nbytes = transport_.sendTrailers(this, trailers); + if (transportCallback_) { + HTTPHeaderSize size; + size.uncompressed = nbytes; + transportCallback_->headerBytesGenerated(size); + } + } else { + // HTTP requires them to go right before EOM + trailers_.reset(new HTTPHeaders(trailers)); + } + } + + /** + * Finalize the egress message; depending on the protocol used + * by the Transport, this may involve sending an explicit "end + * of message" indicator. This method does not actually write the + * message out on the wire immediately. All writes happen at the end + * of the event loop at the earliest. + * + * If the ingress message also is complete, the transaction may + * detach itself from the Handler and Transport and delete itself + * as part of this method. + * + * Note: Either this method or sendAbort() should be called once + * per message. + */ + virtual void sendEOM(); + + /** + * Terminate the transaction. Depending on the underlying protocol, this + * may cause the connection to close or write egress bytes. This method + * does not actually write the message out on the wire immediately. All + * writes happen at the end of the event loop at the earliest. + * + * This function may also cause additional callbacks such as + * detachTransaction() to the handler either immediately or after it returns. + */ + virtual void sendAbort(); + + /** + * Pause ingress processing. Upon pause, the HTTPTransaction + * will call its Transport's pauseIngress() method. The Transport + * should make a best effort to stop invoking the HTTPTransaction's + * onIngress* callbacks. If the Transport does invoke any of those + * methods while the transaction is paused, however, the transaction + * will queue the ingress events and data and delay delivery to the + * Handler until the transaction is unpaused. + */ + virtual void pauseIngress(); + + /** + * Resume ingress processing. Only useful after a call to pauseIngress(). + */ + virtual void resumeIngress(); + + /** + * @return true iff ingress processing is paused for the handler + */ + bool isIngressPaused() const { return ingressPaused_; } + + /** + * Pause egress generation. HTTPTransaction may call its Handler's + * onEgressPaused() method if there is a state change as a result of + * this call. + * + * On receiving onEgressPaused(), the Handler should make a best effort + * to stop invoking the HTTPTransaction's egress generating methods. If + * the Handler does invoke any of those methods while the transaction is + * paused, however, the transaction will forward them anyway, unless it + * is a body event. If flow control is enabled, body events will be + * buffered for later transmission when egress is unpaused. + */ + void pauseEgress(); + + /** + * Resume egress generation. The Handler's onEgressResumed() will not be + * invoked if the HTTP/2 send window is full or there is too much + * buffered egress data on this transaction already. In that case, + * once the send window is not full or the buffer usage decreases, the + * handler will finally get onEgressResumed(). + */ + void resumeEgress(); + + /** + * @return true iff egress processing is paused for the handler + */ + bool isEgressPaused() const { return handlerEgressPaused_; } + + /** + * @return true iff this transaction can be used to push resources to + * the remote side. + */ + bool supportsPushTransactions() const { + return direction_ == TransportDirection::DOWNSTREAM && + transport_.getCodec().supportsPushTransactions(); + } + + /** + * Create a new pushed transaction associated with this transaction, + * and assign the given handler and priority. + * + * @return the new transaction for the push, or nullptr if a new push + * transaction is impossible right now. + */ + virtual HTTPTransaction* newPushedTransaction( + HTTPPushTransactionHandler* handler, uint8_t priority) { + if (isEgressEOMSeen()) { + return nullptr; + } + auto txn = transport_.newPushedTransaction(id_, handler, priority); + if (txn) { + pushedTransactions_.insert(txn->getID()); + } + return txn; + } + + /** + * Invoked by the session (upstream only) when a new pushed transaction + * arrives. The txn's handler will be notified and is responsible for + * installing a handler. If no handler is installed in the callback, + * the pushed transaction will be aborted. + */ + bool onPushedTransaction(HTTPTransaction* txn); + + /** + * True if this transaction is a server push transaction + */ + bool isPushed() const { + return assocStreamId_ != 0; + } + + /** + * Returns the associated transaction ID for pushed transactions, 0 otherwise + */ + HTTPCodec::StreamID getAssocTxnId() const { + return assocStreamId_; + } + + /** + * Get a set of server-pushed transactions associated with this transaction. + */ + const std::set& getPushedTransactions() const { + return pushedTransactions_; + } + + /** + * Remove the pushed txn ID from the set of pushed txns + * associated with this txn. + */ + void removePushedTransaction(HTTPCodec::StreamID pushStreamId) { + pushedTransactions_.erase(pushStreamId); + } + + /** + * Schedule or refresh the timeout for this transaction + */ + void refreshTimeout() { + if (transactionTimeouts_) { + transactionTimeouts_->scheduleTimeout(this); + } + } + + /** + * Tests if the first byte has already been sent, and if it + * hasn't yet then it marks it as sent. + */ + bool testAndSetFirstByteSent() { + bool ret = firstByteSent_; + firstByteSent_ = true; + return ret; + } + + bool testAndClearActive() { + bool ret = inActiveSet_; + inActiveSet_ = false; + return ret; + } + + /** + * Tests if the very first byte of Header has already been set. + * If it hasn't yet, it marks it as sent. + */ + bool testAndSetFirstHeaderByteSent() { + bool ret = firstHeaderByteSent_; + firstHeaderByteSent_ = true; + return ret; + } + + /** + * Timeout callback for this transaction. The timer is active while + * until the ingress message is complete or terminated by error. + */ + void timeoutExpired() noexcept { + transport_.transactionTimeout(this); + } + + /** + * Write a description of the transaction to a stream + */ + void describe(std::ostream& os) const; + + /** + * Set the maximum egress body size for any outbound body bytes + */ + static void setFlowControlledBodySizeLimit(uint64_t limit) { + egressBodySizeLimit_ = limit; + } + + /** + * Helper class that: + * 1. Increments callbackDepth_ to prevent destruction + * within a scope, and + * 2. calls checkForCompletion() at the end of the scope. + * + * Every method except the constructor, destructor, and + * checkForCompletion() should instantiate a CallbackGuard + * as a local variable. + */ + class CallbackGuard { + public: + explicit CallbackGuard(HTTPTransaction& txn): + txn_(txn) { + ++txn_.callbackDepth_; + } + CallbackGuard(CallbackGuard const & other): txn_(other.txn_) { + ++txn_.callbackDepth_; + } + ~CallbackGuard() { + if (0 == --txn_.callbackDepth_) { + txn_.checkForCompletion(); + } + } + + HTTPTransaction& peekTransaction() { return txn_; } + private: + HTTPTransaction& txn_; + }; + + private: + HTTPTransaction(const HTTPTransaction&) = delete; + HTTPTransaction& operator=(const HTTPTransaction&) = delete; + + /** + * Check whether the ingress and egress messages are both complete; + * if they are, detach from the Transport and Handler and delete this + * HTTPTransaction. + */ + void checkForCompletion(); + + /** + * Invokes the handler's onEgressPaused/Resumed if the handler's pause + * state needs updating + */ + void updateHandlerPauseState(); + + bool mustQueueIngress() const; + + /** + * Check if deferredIngress_ points to some queue before pushing HTTPEvent + * to it. + */ + void checkCreateDeferredIngress(); + + /** + * Implementation of sending an abort for this transaction. + */ + void sendAbort(ErrorCode statusCode); + + // Internal implementations of the ingress-related callbacks + // that work whether the ingress events are immediate or deferred. + void processIngressHeadersComplete(std::unique_ptr msg); + void processIngressBody(std::unique_ptr chain, size_t len); + void processIngressChunkHeader(size_t length); + void processIngressChunkComplete(); + void processIngressTrailers(std::unique_ptr trailers); + void processIngressUpgrade(UpgradeProtocol protocol); + void processIngressEOM(); + + void sendBodyFlowControlled(std::unique_ptr body = nullptr); + size_t sendBodyNow(std::unique_ptr body, size_t bodyLen, + bool eom); + size_t sendEOMNow(); + void onDeltaSendWindowSize(int32_t windowDelta); + + void notifyTransportPendingEgress(); + + size_t sendDeferredBody(uint32_t maxEgress); + + bool isEnqueued() const { return enqueued_; } + + void dequeue() { + DCHECK(isEnqueued()); + egressQueue_.erase(queueHandle_); + enqueued_ = false; + } + + bool hasPendingEOM() const { + return deferredEgressBody_.chainLength() == 0 && + isEgressEOMQueued(); + } + + bool isExpectingIngress() const; + + void updateReadTimeout(); + + /** + * Causes isIngressComplete() to return true, removes any queued + * ingress, and cancels the read timeout. + */ + void markIngressComplete(); + + /** + * Causes isEgressComplete() to return true, removes any queued egress, + * and cancels the write timeout. + */ + void markEgressComplete(); + + /** + * Validates the ingress state transition. Returns false and sends an + * abort with PROTOCOL_ERROR if the transition fails. Otherwise it + * returns true. + */ + bool validateIngressStateTransition(HTTPTransactionIngressSM::Event); + + /** + * Queue to hold any events that we receive from the Transaction + * while the ingress is supposed to be paused. + */ + std::unique_ptr> deferredIngress_; + + uint32_t maxDeferredIngress_{0}; + + /** + * Queue to hold any body bytes to be sent out + * while egress to the remote is supposed to be paused. + */ + folly::IOBufQueue deferredEgressBody_{folly::IOBufQueue::cacheChainLength()}; + + const TransportDirection direction_; + HTTPCodec::StreamID id_; + uint32_t seqNo_; + Handler* handler_{nullptr}; + Transport& transport_; + HTTPTransactionEgressSM::State egressState_{ + HTTPTransactionEgressSM::getNewInstance()}; + HTTPTransactionIngressSM::State ingressState_{ + HTTPTransactionIngressSM::getNewInstance()}; + apache::thrift::async::TAsyncTimeoutSet* transactionTimeouts_{nullptr}; + HTTPSessionStats* stats_{nullptr}; + + /** + * The recv window and associated data. This keeps track of how many + * bytes we are allowed to buffer. + */ + Window recvWindow_; + + /** + * The send window and associated data. This keeps track of how many + * bytes we are allowed to send and have outstanding. + */ + Window sendWindow_; + + TransportCallback* transportCallback_{nullptr}; + + /** + * Number of callbacks currently active. Used to prevent destruction + * while in a callback that might turn around and invoke some method + * of this object. + */ + unsigned callbackDepth_{0}; + + /** + * Trailers to send, if any. + */ + std::unique_ptr trailers_; + + struct Chunk { + explicit Chunk(size_t inLength) : length(inLength), headerSent(false) {} + size_t length; + bool headerSent; + }; + std::list chunkHeaders_; + + /** + * Reference to our priority queue + */ + PriorityQueue& egressQueue_; + + /** + * Handle to our position in the priority queue. Only valid when + * enqueued_ == true + */ + PriorityQueue::handle_type queueHandle_; + + /** + * bytes we need to acknowledge to the remote end using a window update + */ + int32_t recvToAck_{0}; + + /** + * ID of request transaction (for pushed txns only) + */ + HTTPCodec::StreamID assocStreamId_{0}; + + /** + * Set of all push transactions IDs associated with this transaction. + */ + std::set pushedTransactions_; + + /** + * SPDY priority in the high bits, randomness in the low bits + */ + uint32_t priority_; + + /** + * If this transaction represents a request (ie, it is backed by an + * HTTPUpstreamSession) , this field indicates the last response status + * received from the server. If this transaction represents a response, + * this field indicates the last status we've sent. For instances, this + * could take on multiple 1xx values, and then take on 200. + */ + uint16_t lastResponseStatus_{0}; + + bool ingressPaused_:1; + bool egressPaused_:1; + bool handlerEgressPaused_:1; + bool useFlowControl_:1; + bool aborted_:1; + bool deleting_:1; + bool enqueued_:1; + bool firstByteSent_:1; + bool firstHeaderByteSent_:1; + bool inResume_:1; + bool inActiveSet_:1; + + static uint64_t egressBodySizeLimit_; + static uint64_t egressBufferLimit_; +}; + +/** + * Write a description of an HTTPTransaction to an ostream + */ +std::ostream& operator<<(std::ostream& os, const HTTPTransaction& txn); + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPTransactionEgressSM.cpp b/proxygen/lib/http/session/HTTPTransactionEgressSM.cpp new file mode 100644 index 0000000000..2ce060bd94 --- /dev/null +++ b/proxygen/lib/http/session/HTTPTransactionEgressSM.cpp @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPTransactionEgressSM.h" + +namespace proxygen { + +// +--> ChunkHeaderSent -> ChunkBodySent +// | ^ v +// | | ChunkTerminatorSent -> TrailersSent +// | |__________| | | +// | | v +// Start -> HeadersSent +----> EOMQueued --> SendingDone +// | ^ +// +------------> RegularBodySent -------+ + +const HTTPTransactionEgressSMData::TransitionTable +HTTPTransactionEgressSMData::transitions = +{ + {{State::Start, Event::sendHeaders}, State::HeadersSent}, + + // For HTTP sending 100 response, then a regular response + {{State::HeadersSent, Event::sendHeaders}, State::HeadersSent}, + + {{State::HeadersSent, Event::sendBody}, State::RegularBodySent}, + {{State::HeadersSent, Event::sendChunkHeader}, State::ChunkHeaderSent}, + {{State::HeadersSent, Event::sendEOM}, State::EOMQueued}, + + {{State::RegularBodySent, Event::sendBody}, State::RegularBodySent}, + {{State::RegularBodySent, Event::sendEOM}, State::EOMQueued}, + + {{State::ChunkHeaderSent, Event::sendBody}, State::ChunkBodySent}, + + {{State::ChunkBodySent, Event::sendBody}, State::ChunkBodySent}, + {{State::ChunkBodySent, Event::sendChunkTerminator}, + State::ChunkTerminatorSent}, + + {{State::ChunkTerminatorSent, Event::sendChunkHeader}, + State::ChunkHeaderSent}, + {{State::ChunkTerminatorSent, Event::sendTrailers}, State::TrailersSent}, + {{State::ChunkTerminatorSent, Event::sendEOM}, State::EOMQueued}, + + {{State::TrailersSent, Event::sendEOM}, State::EOMQueued}, + + {{State::EOMQueued, Event::eomFlushed}, State::SendingDone}, +}; + +std::ostream& operator<<(std::ostream& os, + HTTPTransactionEgressSMData::State s) { + switch (s) { + case HTTPTransactionEgressSMData::State::Start: + os << "Start"; + break; + case HTTPTransactionEgressSMData::State::HeadersSent: + os << "HeadersSent"; + break; + case HTTPTransactionEgressSMData::State::RegularBodySent: + os << "RegularBodySent"; + break; + case HTTPTransactionEgressSMData::State::ChunkHeaderSent: + os << "ChunkHeaderSent"; + break; + case HTTPTransactionEgressSMData::State::ChunkBodySent: + os << "ChunkBodySent"; + break; + case HTTPTransactionEgressSMData::State::ChunkTerminatorSent: + os << "ChunkTerminatorSent"; + break; + case HTTPTransactionEgressSMData::State::TrailersSent: + os << "TrailersSent"; + break; + case HTTPTransactionEgressSMData::State::EOMQueued: + os << "EOMQueued"; + break; + case HTTPTransactionEgressSMData::State::SendingDone: + os << "SendingDone"; + break; + } + + return os; +} + +std::ostream& operator<<(std::ostream& os, + HTTPTransactionEgressSMData::Event e) { + switch (e) { + case HTTPTransactionEgressSMData::Event::sendHeaders: + os << "sendHeaders"; + break; + case HTTPTransactionEgressSMData::Event::sendBody: + os << "sendBody"; + break; + case HTTPTransactionEgressSMData::Event::sendChunkHeader: + os << "sendChunkHeader"; + break; + case HTTPTransactionEgressSMData::Event::sendChunkTerminator: + os << "sendChunkTerminator"; + break; + case HTTPTransactionEgressSMData::Event::sendTrailers: + os << "sendTrailers"; + break; + case HTTPTransactionEgressSMData::Event::sendEOM: + os << "sendEOM"; + break; + case HTTPTransactionEgressSMData::Event::eomFlushed: + os << "eomFlushed"; + break; + } + + return os; +} + +} diff --git a/proxygen/lib/http/session/HTTPTransactionEgressSM.h b/proxygen/lib/http/session/HTTPTransactionEgressSM.h new file mode 100644 index 0000000000..368831dc8d --- /dev/null +++ b/proxygen/lib/http/session/HTTPTransactionEgressSM.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/utils/StateMachine.h" + +#include +#include + +namespace proxygen { + +class HTTPTransactionEgressSMData { + public: + + enum class State: uint8_t { + Start, + HeadersSent, + RegularBodySent, + ChunkHeaderSent, + ChunkBodySent, + ChunkTerminatorSent, + TrailersSent, + EOMQueued, + SendingDone + }; + + enum class Event: uint8_t { + // API accessible transitions + sendHeaders, + sendBody, + sendChunkHeader, + sendChunkTerminator, + sendTrailers, + sendEOM, + // Internal state transitions + eomFlushed, + }; + + static State getInitialState() { + return State::Start; + } + + static std::pair find(State s, Event e) { + auto it = transitions.find(std::make_pair(s, e)); + if (it == transitions.end()) { + return std::make_pair(s, false); + } + + return std::make_pair(it->second, true); + } + private: + typedef std::map, State> TransitionTable; + static const TransitionTable transitions; +}; + +std::ostream& operator<<(std::ostream& os, + HTTPTransactionEgressSMData::State s); + +std::ostream& operator<<(std::ostream& os, + HTTPTransactionEgressSMData::Event e); + +typedef StateMachine HTTPTransactionEgressSM; + +} diff --git a/proxygen/lib/http/session/HTTPTransactionIngressSM.cpp b/proxygen/lib/http/session/HTTPTransactionIngressSM.cpp new file mode 100644 index 0000000000..90880ef36a --- /dev/null +++ b/proxygen/lib/http/session/HTTPTransactionIngressSM.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPTransactionIngressSM.h" + +namespace proxygen { + +// +--> ChunkHeaderReceived -> ChunkBodyReceived +// | ^ v +// | | ChunkCompleted -> TrailersReceived +// | |_______________| | | +// | v v +// Start -> HeadersReceived ---------------> EOMQueued ---> ReceivingDone +// | | ^ ^ +// | +-----> RegularBodyReceived --+ | +// | | +// +---------> UpgradeComplete --------+ + +const HTTPTransactionIngressSMData::TransitionTable +HTTPTransactionIngressSMData::transitions = +{ + {{State::Start, Event::onHeaders}, State::HeadersReceived}, + + // For HTTP receiving 100 response, then a regular response + {{State::HeadersReceived, Event::onHeaders}, State::HeadersReceived}, + + {{State::HeadersReceived, Event::onBody}, State::RegularBodyReceived}, + {{State::HeadersReceived, Event::onChunkHeader}, State::ChunkHeaderReceived}, + // special case - 0 byte body with trailers + {{State::HeadersReceived, Event::onTrailers}, State::TrailersReceived}, + {{State::HeadersReceived, Event::onUpgrade}, State::UpgradeComplete}, + {{State::HeadersReceived, Event::onEOM}, State::EOMQueued}, + + {{State::RegularBodyReceived, Event::onBody}, State::RegularBodyReceived}, + {{State::RegularBodyReceived, Event::onEOM}, State::EOMQueued}, + + {{State::ChunkHeaderReceived, Event::onBody}, State::ChunkBodyReceived}, + + {{State::ChunkBodyReceived, Event::onBody}, State::ChunkBodyReceived}, + {{State::ChunkBodyReceived, Event::onChunkComplete}, State::ChunkCompleted}, + + {{State::ChunkCompleted, Event::onChunkHeader}, State::ChunkHeaderReceived}, + // TODO: "trailers" may be received at any time due to the SPDY HEADERS + // frame coming at any time. We might want to have a + // TransactionStateMachineFactory that takes a codec and generates the + // appropriate transaction state machine from that. + {{State::ChunkCompleted, Event::onTrailers}, State::TrailersReceived}, + {{State::ChunkCompleted, Event::onEOM}, State::EOMQueued}, + + {{State::TrailersReceived, Event::onEOM}, State::EOMQueued}, + + {{State::UpgradeComplete, Event::onBody}, State::UpgradeComplete}, + {{State::UpgradeComplete, Event::onEOM}, State::EOMQueued}, + + {{State::EOMQueued, Event::eomFlushed}, State::ReceivingDone}, +}; + +std::ostream& operator<<(std::ostream& os, + HTTPTransactionIngressSMData::State s) { + switch (s) { + case HTTPTransactionIngressSMData::State::Start: + os << "Start"; + break; + case HTTPTransactionIngressSMData::State::HeadersReceived: + os << "HeadersReceived"; + break; + case HTTPTransactionIngressSMData::State::RegularBodyReceived: + os << "RegularBodyReceived"; + break; + case HTTPTransactionIngressSMData::State::ChunkHeaderReceived: + os << "ChunkHeaderReceived"; + break; + case HTTPTransactionIngressSMData::State::ChunkBodyReceived: + os << "ChunkBodyReceived"; + break; + case HTTPTransactionIngressSMData::State::ChunkCompleted: + os << "ChunkCompleted"; + break; + case HTTPTransactionIngressSMData::State::TrailersReceived: + os << "TrailersReceived"; + break; + case HTTPTransactionIngressSMData::State::UpgradeComplete: + os << "UpgradeComplete"; + break; + case HTTPTransactionIngressSMData::State::EOMQueued: + os << "EOMQueued"; + break; + case HTTPTransactionIngressSMData::State::ReceivingDone: + os << "ReceivingDone"; + break; + } + + return os; +} + +std::ostream& operator<<(std::ostream& os, + HTTPTransactionIngressSMData::Event e) { + switch (e) { + case HTTPTransactionIngressSMData::Event::onHeaders: + os << "onHeaders"; + break; + case HTTPTransactionIngressSMData::Event::onBody: + os << "onBody"; + break; + case HTTPTransactionIngressSMData::Event::onChunkHeader: + os << "onChunkHeader"; + break; + case HTTPTransactionIngressSMData::Event::onChunkComplete: + os << "onChunkComplete"; + break; + case HTTPTransactionIngressSMData::Event::onTrailers: + os << "onTrailers"; + break; + case HTTPTransactionIngressSMData::Event::onUpgrade: + os << "onUpgrade"; + break; + case HTTPTransactionIngressSMData::Event::onEOM: + os << "onEOM"; + break; + case HTTPTransactionIngressSMData::Event::eomFlushed: + os << "eomFlushed"; + break; + } + + return os; +} + +} diff --git a/proxygen/lib/http/session/HTTPTransactionIngressSM.h b/proxygen/lib/http/session/HTTPTransactionIngressSM.h new file mode 100644 index 0000000000..e88620e86e --- /dev/null +++ b/proxygen/lib/http/session/HTTPTransactionIngressSM.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/utils/StateMachine.h" + +#include +#include + +namespace proxygen { + +class HTTPTransactionIngressSMData { + public: + + enum class State: uint8_t { + Start, + HeadersReceived, + RegularBodyReceived, + ChunkHeaderReceived, + ChunkBodyReceived, + ChunkCompleted, + TrailersReceived, + UpgradeComplete, + EOMQueued, + ReceivingDone, + }; + + enum class Event: uint8_t { + // API accessible transitions + onHeaders, + onBody, + onChunkHeader, + onChunkComplete, + onTrailers, + onUpgrade, + onEOM, + // Internal state transitions + eomFlushed, + }; + + static State getInitialState() { + return State::Start; + } + + static std::pair find(State s, Event e) { + auto it = transitions.find(std::make_pair(s, e)); + if (it == transitions.end()) { + return std::make_pair(s, false); + } + + return std::make_pair(it->second, true); + } + private: + typedef std::map, State> TransitionTable; + static const TransitionTable transitions; +}; + +std::ostream& operator<<(std::ostream& os, + HTTPTransactionIngressSMData::State s); + +std::ostream& operator<<(std::ostream& os, + HTTPTransactionIngressSMData::Event e); + +typedef StateMachine HTTPTransactionIngressSM; + +} diff --git a/proxygen/lib/http/session/HTTPUpstreamSession.cpp b/proxygen/lib/http/session/HTTPUpstreamSession.cpp new file mode 100644 index 0000000000..fede00ab9f --- /dev/null +++ b/proxygen/lib/http/session/HTTPUpstreamSession.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPUpstreamSession.h" + +#include "proxygen/lib/http/session/HTTPTransaction.h" + +#include +#include + +namespace proxygen { + +HTTPUpstreamSession::~HTTPUpstreamSession() {} + +bool HTTPUpstreamSession::isReusable() const { + VLOG(4) << "isReusable: " << *this + << ", liveTransactions_=" << liveTransactions_ + << ", sock_->good()=" << sock_->good() + << ", sock_->connecting()=" << sock_->connecting() + << ", draining_=" << draining_ + << ", codec_->isReusable()=" << codec_->isReusable() + << ", codec_->isBusy()=" << codec_->isBusy() + << ", pendingWriteSize_=" << pendingWriteSize_ + << ", numActiveWrites_=" << numActiveWrites_ + << ", writeTimeout_.isScheduled()=" << writeTimeout_.isScheduled() + << ", readsShutdown_=" << readsShutdown_ + << ", writesShutdown_=" << writesShutdown_ + << ", writesDraining_=" << writesDraining_ + << ", resetAfterDrainingWrites_=" << resetAfterDrainingWrites_ + << ", ingressError_=" << ingressError_ + << ", hasMoreWrites()=" << hasMoreWrites() + << ", codec_->supportsParallelRequests()=" + << codec_->supportsParallelRequests(); + return + sock_->good() && + !sock_->connecting() && + codec_->isReusable() && + !draining_ && + !codec_->isBusy() && + !readsShutdown_ && + !writesShutdown_ && + !writesDraining_ && + !resetAfterDrainingWrites_ && + !ingressError_ && + (codec_->supportsParallelRequests() || ( + // These conditions only apply to serial codec sessions + !hasMoreWrites() && + liveTransactions_ == 0 && + !writeTimeout_.isScheduled())); +} + +HTTPTransaction* +HTTPUpstreamSession::newTransaction(HTTPTransaction::Handler* handler, + int8_t priority) { + CHECK_NOTNULL(handler); + + if (!supportsMoreTransactions() || draining_) { + // This session doesn't support any more parallel transactions + return nullptr; + } + + if (!started_) { + startNow(); + } + + HTTPCodec::StreamID streamID = codec_->createStream(); + HTTPTransaction* txn = new HTTPTransaction( + codec_->getTransportDirection(), streamID, transactionSeqNo_, *this, + txnEgressQueue_, transactionTimeouts_, sessionStats_, + codec_->supportsStreamFlowControl(), + initialReceiveWindow_, + getCodecSendWindowSize(), + priority); + + if (!addTransaction(txn)) { + delete txn; + return nullptr; + } + + transactionSeqNo_++; + + txn->setReceiveWindow(receiveStreamWindowSize_); + txn->setHandler(handler); + setNewTransactionPauseState(txn); + return txn; +} + +HTTPTransaction::Handler* +HTTPUpstreamSession::getParseErrorHandler(HTTPTransaction* txn, + const HTTPException& error) { + // No special handler for upstream requests that have a parse error + return nullptr; +} + +HTTPTransaction::Handler* +HTTPUpstreamSession::getTransactionTimeoutHandler(HTTPTransaction* txn) { + // No special handler for upstream requests that time out + return nullptr; +} + +bool HTTPUpstreamSession::allTransactionsStarted() const { + for (const auto txn: transactions_) { + if (!txn.second->isPushed() && !txn.second->isEgressStarted()) { + return false; + } + } + return true; +} + +} // proxygen diff --git a/proxygen/lib/http/session/HTTPUpstreamSession.h b/proxygen/lib/http/session/HTTPUpstreamSession.h new file mode 100644 index 0000000000..fdd114888b --- /dev/null +++ b/proxygen/lib/http/session/HTTPUpstreamSession.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPSession.h" +#include "proxygen/lib/http/session/HTTPSessionStats.h" + +#include + +namespace proxygen { + +class HTTPSessionStats; +class HTTPUpstreamSession final: public HTTPSession { + public: + /** + * @param sock An open socket on which any applicable TLS handshaking + * has been completed already. + * @param localAddr Address and port of the local end of the socket. + * @param peerAddr Address and port of the remote end of the socket. + * @param codec A codec with which to parse/generate messages in + * whatever HTTP-like wire format this session needs. + */ + HTTPUpstreamSession( + apache::thrift::async::TAsyncTimeoutSet* transactionTimeouts, + apache::thrift::async::TAsyncTransport::UniquePtr&& sock, + const folly::SocketAddress& localAddr, + const folly::SocketAddress& peerAddr, + std::unique_ptr codec, + const TransportInfo& tinfo, + InfoCallback* infoCallback): + HTTPSession(transactionTimeouts, std::move(sock), localAddr, peerAddr, + nullptr, std::move(codec), tinfo, infoCallback) { + CHECK(codec_->getTransportDirection() == TransportDirection::UPSTREAM); + } + + /** + * Creates a new transaction on this upstream session. Invoking this function + * also has the side-affect of starting reads after this event loop completes. + * + * @param handler The request handler to attach to this transaction. It must + * not be null. + * @param priority The priority to associate with this request. Lower numbers + * indicate higher priority. This parameter only has an effect + * if the session uses a parallel codec. + */ + HTTPTransaction* newTransaction(HTTPTransaction::Handler* handler, + int8_t priority = -1); + + /** + * Returns true if this session has no open transactions and the underlying + * transport can be used again in a new request. + */ + bool isReusable() const; + + /** + * Drains the current transactions and prevents new transactions from being + * created on this session. When the number of transactions reaches zero, this + * session will shutdown the transport and delete itself. + */ + void drain() { + HTTPSession::drain(); + } + + private: + ~HTTPUpstreamSession() override; + + /** + * Called by onHeadersComplete(). Currently a no-op for upstream. + */ + void setupOnHeadersComplete(HTTPTransaction* txn, + HTTPMessage* msg) override {} + + /** + * Called by processParseError() if the transaction has no handler. + */ + HTTPTransaction::Handler* getParseErrorHandler( + HTTPTransaction* txn, const HTTPException& error) override; + + /** + * Called by transactionTimeout() if the transaction has no handler. + */ + HTTPTransaction::Handler* getTransactionTimeoutHandler( + HTTPTransaction* txn) override; + + bool allTransactionsStarted() const override; +}; + +} // proxygen diff --git a/proxygen/lib/http/session/Makefile.am b/proxygen/lib/http/session/Makefile.am new file mode 100644 index 0000000000..845be43bdb --- /dev/null +++ b/proxygen/lib/http/session/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = . test diff --git a/proxygen/lib/http/session/SimpleController.cpp b/proxygen/lib/http/session/SimpleController.cpp new file mode 100644 index 0000000000..efa3d695a0 --- /dev/null +++ b/proxygen/lib/http/session/SimpleController.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/SimpleController.h" + +#include "proxygen/lib/http/session/CodecErrorResponseHandler.h" +#include "proxygen/lib/http/session/HTTPDirectResponseHandler.h" +#include "proxygen/lib/http/session/HTTPSessionAcceptor.h" + +namespace proxygen { + +SimpleController::SimpleController(HTTPSessionAcceptor* acceptor) + : acceptor_(acceptor) { +} + +HTTPTransactionHandler* SimpleController::getRequestHandler( + HTTPTransaction& txn, HTTPMessage* msg) { + return acceptor_->newHandler(txn, msg); +} + +HTTPTransactionHandler* SimpleController::getParseErrorHandler( + HTTPTransaction* txn, + const HTTPException& error, + const folly::SocketAddress& localAddress) { + + if (error.hasCodecStatusCode()) { + return new CodecErrorResponseHandler(error.getCodecStatusCode()); + } + + auto errorPage = acceptor_->getErrorPage(localAddress); + return createErrorHandler(error.hasHttpStatusCode() ? + error.getHttpStatusCode() : 400, + "Bad Request", errorPage); +} + +HTTPTransactionHandler* SimpleController::getTransactionTimeoutHandler( + HTTPTransaction* txn, + const folly::SocketAddress& localAddress) { + + auto errorPage = acceptor_->getErrorPage(localAddress); + return createErrorHandler(408, "Client timeout", errorPage); +} + +void SimpleController::attachSession(HTTPSession* sess) { +} + +void SimpleController::detachSession(const HTTPSession* sess) { +} + +HTTPTransactionHandler* SimpleController::createErrorHandler( + uint32_t statusCode, + const std::string& statusMessage, + const HTTPErrorPage* errorPage) { + + return new HTTPDirectResponseHandler(statusCode, + statusMessage, + errorPage); +} + +} diff --git a/proxygen/lib/http/session/SimpleController.h b/proxygen/lib/http/session/SimpleController.h new file mode 100644 index 0000000000..a4478a7766 --- /dev/null +++ b/proxygen/lib/http/session/SimpleController.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPSessionController.h" + +#include + +namespace proxygen { + +class HTTPErrorPage; +class HTTPSessionAcceptor; + +/** + * This simple controller provides some basic default behaviors. When + * errors occur, it will install an appropriate handler. Otherwise, it + * will install the acceptor's default handler. + */ +class SimpleController : public HTTPSessionController { + public: + explicit SimpleController(HTTPSessionAcceptor* acceptor); + + /** + * Will be invoked whenever HTTPSession successfully parses a + * request + */ + HTTPTransactionHandler* getRequestHandler(HTTPTransaction& txn, + HTTPMessage* msg) override; + + /** + * Will be invoked when HTTPSession is unable to parse a new request + * on the connection because of bad input. + * + * error contains specific information about what went wrong + */ + HTTPTransactionHandler* getParseErrorHandler( + HTTPTransaction* txn, + const HTTPException& error, + const folly::SocketAddress& localAddress) override; + + /** + * Will be invoked when HTTPSession times out parsing a new request. + */ + HTTPTransactionHandler* getTransactionTimeoutHandler( + HTTPTransaction* txn, + const folly::SocketAddress& localAddress) override; + + void attachSession(HTTPSession*) override; + void detachSession(const HTTPSession*) override; + protected: + HTTPTransactionHandler* createErrorHandler( + uint32_t statusCode, + const std::string& statusMessage, + const HTTPErrorPage* errorPage); + + HTTPSessionAcceptor* const acceptor_{nullptr}; +}; + +} diff --git a/proxygen/lib/http/session/TTLBAStats.h b/proxygen/lib/http/session/TTLBAStats.h new file mode 100644 index 0000000000..ef5cded890 --- /dev/null +++ b/proxygen/lib/http/session/TTLBAStats.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +class TTLBAStats { + public: + virtual ~TTLBAStats() noexcept {} + + virtual void recordTTLBAExceedLimit() noexcept = 0; + virtual void recordTTLBAIOBSplitByEom() noexcept = 0; + virtual void recordTTLBANotFound() noexcept = 0; + virtual void recordTTLBAReceived() noexcept = 0; + virtual void recordTTLBATimeout() noexcept = 0; + virtual void recordTTLBAEomPassed() noexcept = 0; + virtual void recordTTLBATracked() noexcept = 0; +}; + +} // namespace proxygen diff --git a/proxygen/lib/http/session/TransportFilter.cpp b/proxygen/lib/http/session/TransportFilter.cpp new file mode 100644 index 0000000000..c5e846970c --- /dev/null +++ b/proxygen/lib/http/session/TransportFilter.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/TransportFilter.h" + +using namespace apache::thrift::async; +using namespace apache::thrift::transport; +using namespace folly; + +namespace proxygen { + +// TAsyncTransport::ReadCallback methods +void PassThroughTransportFilter::getReadBuffer(void** bufReturn, + size_t* lenReturn) { + callback_->getReadBuffer(bufReturn, lenReturn); +} + +void PassThroughTransportFilter::readDataAvailable(size_t len) noexcept { + callback_->readDataAvailable(len); +} + +void PassThroughTransportFilter::readEOF() noexcept { + callback_->readEOF(); + destroy(); +} + +void PassThroughTransportFilter::readError( + const TTransportException& ex) noexcept { + callback_->readError(ex); + destroy(); +} + +// TAsyncTransport methods + +void PassThroughTransportFilter::setReadCallback( + TAsyncTransport::ReadCallback* callback) { + // Important! The filter must call setCallbackInternal in its base class and + // it must not forward the call. + setCallbackInternal(callback); +} + +TAsyncTransport::ReadCallback* +PassThroughTransportFilter::getReadCallback() const { + return call_->getReadCallback(); +} + +void PassThroughTransportFilter::write( + TAsyncTransport::WriteCallback* callback, + const void* buf, size_t bytes, WriteFlags flags) { + call_->write(callback, buf, bytes, flags); +} + +void PassThroughTransportFilter::writev( + TAsyncTransport::WriteCallback* callback, const iovec* vec, size_t count, + WriteFlags flags) { + call_->writev(callback, vec, count, flags); +} + +void PassThroughTransportFilter::writeChain( + TAsyncTransport::WriteCallback* callback, + std::unique_ptr&& iob, WriteFlags flags) { + call_->writeChain(callback, std::move(iob), flags); +} + +void PassThroughTransportFilter::close() { + call_->close(); + // wait for readEOF() to call destroy() +} + +void PassThroughTransportFilter::closeNow() { + call_->closeNow(); + // wait for readEOF() to call destroy() +} + +void PassThroughTransportFilter::closeWithReset() { + call_->closeWithReset(); + // wait for readEOF() to call destroy() +} + +void PassThroughTransportFilter::shutdownWrite() { + call_->shutdownWrite(); +} + +void PassThroughTransportFilter::shutdownWriteNow() { + call_->shutdownWriteNow(); +} + +bool PassThroughTransportFilter::good() const { + return call_->good(); +} + +bool PassThroughTransportFilter::readable() const { + return call_->readable(); +} + +bool PassThroughTransportFilter::connecting() const { + return call_->connecting(); +} + +bool PassThroughTransportFilter::error() const { + return call_->error(); +} + +void PassThroughTransportFilter::attachEventBase(EventBase* eventBase) { + call_->attachEventBase(eventBase); +} + +void PassThroughTransportFilter::detachEventBase() { + call_->detachEventBase(); +} + +bool PassThroughTransportFilter::isDetachable() const { + return call_->isDetachable(); +} + +EventBase* PassThroughTransportFilter::getEventBase() const { + return call_->getEventBase(); +} + +void PassThroughTransportFilter::setSendTimeout(uint32_t milliseconds) { + call_->setSendTimeout(milliseconds); +} + +uint32_t PassThroughTransportFilter::getSendTimeout() const { + return call_->getSendTimeout(); +} + +void PassThroughTransportFilter::getLocalAddress( + folly::SocketAddress* address) const { + call_->getLocalAddress(address); +} + +void PassThroughTransportFilter::getPeerAddress( + folly::SocketAddress* address) const { + call_->getPeerAddress(address); +} + +void PassThroughTransportFilter::setEorTracking(bool track) { + call_->setEorTracking(track); +} + +size_t PassThroughTransportFilter::getAppBytesWritten() const { + return call_->getAppBytesWritten(); +} +size_t PassThroughTransportFilter::getRawBytesWritten() const { + return call_->getRawBytesWritten(); +} +size_t PassThroughTransportFilter::getAppBytesReceived() const { + return call_->getAppBytesReceived(); +} +size_t PassThroughTransportFilter::getRawBytesReceived() const { + //new PassThroughTransportFilter(); + return call_->getRawBytesReceived(); +} + +} diff --git a/proxygen/lib/http/session/TransportFilter.h b/proxygen/lib/http/session/TransportFilter.h new file mode 100644 index 0000000000..f7d98d035e --- /dev/null +++ b/proxygen/lib/http/session/TransportFilter.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/utils/FilterChain.h" + +#include +#include + +namespace proxygen { + +typedef GenericFilter< + apache::thrift::async::TAsyncTransport, + apache::thrift::async::TAsyncTransport::ReadCallback, + &apache::thrift::async::TAsyncTransport::setReadCallback, + true, + apache::thrift::async::TAsyncTransport::Destructor> TransportFilter; + +/** + * An implementation of Transport that passes through all calls and also + * properly calls setCallback(). This is useful to subclass if you aren't + * interested in intercepting every function. See TAsyncTransport.h for + * documentation on these methods + */ +class PassThroughTransportFilter: public TransportFilter { + public: + /** + * By default, the filter gets both calls and callbacks + */ + explicit PassThroughTransportFilter(bool calls = true, + bool callbacks = true): + TransportFilter(calls, callbacks) {} + + // TAsyncTransport::ReadCallback methods + + void getReadBuffer(void** bufReturn, size_t* lenReturn) override; + + void readDataAvailable(size_t len) noexcept override; + + void readEOF() noexcept override; + + void readError(const apache::thrift::transport::TTransportException& ex) + noexcept override; + + // TAsyncTransport methods + + void setReadCallback( + apache::thrift::async::TAsyncTransport::ReadCallback* callback) override; + + apache::thrift::async::TAsyncTransport::ReadCallback* getReadCallback() + const override; + + void write(apache::thrift::async::TAsyncTransport::WriteCallback* callback, + const void* buf, size_t bytes, + apache::thrift::async::WriteFlags flags) override; + + void writev(apache::thrift::async::TAsyncTransport::WriteCallback* callback, + const iovec* vec, size_t count, + apache::thrift::async::WriteFlags flags) override; + + void writeChain( + apache::thrift::async::TAsyncTransport::WriteCallback* callback, + std::unique_ptr&& iob, + apache::thrift::async::WriteFlags flags) override; + + void close() override; + + void closeNow() override; + + void closeWithReset() override; + + void shutdownWrite() override; + + void shutdownWriteNow() override; + + bool good() const override; + + bool readable() const override; + + bool connecting() const override; + + bool error() const override; + + void attachEventBase(folly::EventBase* eventBase) override; + + void detachEventBase() override; + + bool isDetachable() const override; + + folly::EventBase* getEventBase() const override; + + void setSendTimeout(uint32_t milliseconds) override; + + uint32_t getSendTimeout() const override; + + void getLocalAddress( + folly::SocketAddress* address) const override; + + void getPeerAddress( + folly::SocketAddress* address) const override; + + void setEorTracking(bool track) override; + + size_t getAppBytesWritten() const override; + size_t getRawBytesWritten() const override; + size_t getAppBytesReceived() const override; + size_t getRawBytesReceived() const override; +}; + +typedef FilterChain< + apache::thrift::async::TAsyncTransport, + apache::thrift::async::TAsyncTransport::ReadCallback, + PassThroughTransportFilter, + &apache::thrift::async::TAsyncTransport::setReadCallback, + false> TransportFilterChain; + +} diff --git a/proxygen/lib/http/session/test/DownstreamTransactionTest.cpp b/proxygen/lib/http/session/test/DownstreamTransactionTest.cpp new file mode 100644 index 0000000000..c1873aa58b --- /dev/null +++ b/proxygen/lib/http/session/test/DownstreamTransactionTest.cpp @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/SPDYConstants.h" +#include "proxygen/lib/http/codec/test/MockHTTPCodec.h" +#include "proxygen/lib/http/codec/test/TestUtils.h" +#include "proxygen/lib/http/session/test/HTTPSessionMocks.h" +#include "proxygen/lib/http/session/test/HTTPTransactionMocks.h" +#include "proxygen/lib/test/TestAsyncTransport.h" + +#include +#include + +using namespace apache::thrift::async; +using namespace apache::thrift::transport; +using namespace proxygen; +using namespace testing; + +using folly::IOBuf; +using folly::IOBufQueue; +using std::string; +using std::unique_ptr; +using std::vector; + +class DownstreamTransactionTest : public testing::Test { + public: + DownstreamTransactionTest() {} + + void setupRequestResponseFlow(HTTPTransaction* txn, uint32_t size) { + EXPECT_CALL(transport_, describe(_)) + .WillRepeatedly(Return()); + EXPECT_CALL(handler_, setTransaction(txn)); + EXPECT_CALL(handler_, detachTransaction()); + EXPECT_CALL(transport_, detach(txn)); + EXPECT_CALL(handler_, onHeadersComplete(_)) + .WillOnce(Invoke([=](std::shared_ptr msg) { + auto response = makeResponse(200); + txn->sendHeaders(*response.get()); + txn->sendBody(makeBuf(size)); + txn->sendEOM(); + })); + EXPECT_CALL(transport_, sendHeaders(txn, _, _)) + .WillOnce(Invoke([=](Unused, const HTTPMessage& headers, Unused) { + EXPECT_EQ(headers.getStatusCode(), 200); + })); + EXPECT_CALL(transport_, sendBody(txn, _, false)) + .WillRepeatedly(Invoke([=](Unused, std::shared_ptr body, + Unused) { + auto cur = body->computeChainDataLength(); + sent_ += cur; + return cur; + })); + EXPECT_CALL(transport_, sendEOM(txn)) + .WillOnce(InvokeWithoutArgs([=]() { + CHECK_EQ(sent_, size); + txn->onIngressBody(makeBuf(size)); + txn->onIngressEOM(); + return 5; + })); + EXPECT_CALL(handler_, onBody(_)) + .WillRepeatedly(Invoke([=](std::shared_ptr body) { + received_ += body->computeChainDataLength(); + })); + EXPECT_CALL(handler_, onEOM()) + .WillOnce(InvokeWithoutArgs([=] { + CHECK_EQ(received_, size); + })); + EXPECT_CALL(transport_, notifyPendingEgress()) + .WillOnce(InvokeWithoutArgs([=] { + txn->onWriteReady(size); + })) + .WillOnce(DoDefault()); // The second call is for sending the eom + + txn->setHandler(&handler_); + } + + protected: + folly::EventBase eventBase_; + TAsyncTimeoutSet::UniquePtr transactionTimeouts_{ + new TAsyncTimeoutSet(&eventBase_, std::chrono::milliseconds(500))}; + MockHTTPTransactionTransport transport_; + StrictMock handler_; + HTTPTransaction::PriorityQueue txnEgressQueue_; + uint32_t received_{0}; + uint32_t sent_{0}; +}; + +/** + * Test that the the transaction properly forwards callbacks to the + * handler and that it interacts with its transport as expected. + */ +TEST_F(DownstreamTransactionTest, simple_callback_forwarding) { + // flow control is disabled + auto txn = new HTTPTransaction( + TransportDirection::DOWNSTREAM, + HTTPCodec::StreamID(1), 1, transport_, + txnEgressQueue_, transactionTimeouts_.get()); + setupRequestResponseFlow(txn, 100); + + txn->onIngressHeadersComplete(makeGetRequest()); + eventBase_.loop(); +} + +/** + * Testing that we're sending a window update for simple requests + */ +TEST_F(DownstreamTransactionTest, regular_window_update) { + auto txn = new HTTPTransaction( + TransportDirection::DOWNSTREAM, + HTTPCodec::StreamID(1), 1, transport_, + txnEgressQueue_, transactionTimeouts_.get(), + nullptr, + true, // flow control enabled + 400, + spdy::kInitialWindow); + uint32_t reqBodySize = 220; + setupRequestResponseFlow(txn, reqBodySize); + + // test that the window update is generated + EXPECT_CALL(transport_, sendWindowUpdate(_, reqBodySize)); + + // run the test + txn->onIngressHeadersComplete(makeGetRequest()); + eventBase_.loop(); +} + +/** + * Testing window increase using window update; we're actually using this in + * production to avoid bumping the window using the SETTINGS frame + */ +TEST_F(DownstreamTransactionTest, window_increase) { + // set initial window size higher than per-stream window + auto txn = new HTTPTransaction( + TransportDirection::DOWNSTREAM, + HTTPCodec::StreamID(1), 1, transport_, + txnEgressQueue_, transactionTimeouts_.get(), + nullptr, + true, // flow control enabled + spdy::kInitialWindow, + spdy::kInitialWindow); + uint32_t reqSize = 500; + setupRequestResponseFlow(txn, reqSize); + + // use a higher window + uint32_t perStreamWindow = spdy::kInitialWindow + 1024 * 1024; + + // we expect the difference from the per stream window and the initial window + uint32_t expectedWindowUpdate = + perStreamWindow - spdy::kInitialWindow; + EXPECT_CALL(transport_, sendWindowUpdate(_, expectedWindowUpdate)); + + txn->setReceiveWindow(perStreamWindow); + + txn->onIngressHeadersComplete(makeGetRequest()); + eventBase_.loop(); +} + +/* + * Testing window update is sent when the transaction resumed from stalling + */ +TEST_F(DownstreamTransactionTest, stalled) { + auto txn = new HTTPTransaction( + TransportDirection::DOWNSTREAM, + HTTPCodec::StreamID(1), 1, transport_, + txnEgressQueue_, transactionTimeouts_.get(), + nullptr, + true, // flow control enabled + spdy::kInitialWindow, + spdy::kInitialWindow); + + EXPECT_CALL(transport_, describe(_)) + .WillRepeatedly(Return()); + EXPECT_CALL(handler_, setTransaction(txn)); + EXPECT_CALL(handler_, detachTransaction()); + EXPECT_CALL(transport_, detach(txn)); + EXPECT_CALL(handler_, onHeadersComplete(_)); + EXPECT_CALL(transport_, sendHeaders(txn, _, _)); + EXPECT_CALL(transport_, sendEOM(txn)); + EXPECT_CALL(handler_, onBody(_)); + EXPECT_CALL(handler_, onEOM()); + EXPECT_CALL(transport_, notifyPendingEgress()); + + txn->setHandler(&handler_); + // Pause the transaction + txn->setReceiveWindow(0); + txn->onIngressHeadersComplete(makeGetRequest()); + auto response = makeResponse(200); + txn->sendHeaders(*response.get()); + txn->sendEOM(); + // Saturating the ingress window + txn->onIngressBody(makeBuf(spdy::kInitialWindow)); + + // sendWindowUpdate should be called when we setReceiveWindow back + EXPECT_CALL(transport_, sendWindowUpdate(_, spdy::kInitialWindow)); + txn->setReceiveWindow(spdy::kInitialWindow); + txn->onIngressEOM(); + eventBase_.loop(); +} + +/** + * Testing that we're not sending window update when per-stream window size is + * smaller than the initial window size + */ +TEST_F(DownstreamTransactionTest, window_decrease) { + // set initial window size higher than per-stream window + auto txn = new HTTPTransaction( + TransportDirection::DOWNSTREAM, + HTTPCodec::StreamID(1), 1, transport_, + txnEgressQueue_, transactionTimeouts_.get(), + nullptr, + true, // flow control enabled + spdy::kInitialWindow, + spdy::kInitialWindow); + setupRequestResponseFlow(txn, 500); + + // in this case, there should be no window update, as we decrease the window + // below the number of bytes we're sending + EXPECT_CALL(transport_, sendWindowUpdate(_, _)).Times(0); + + // use a smaller window + uint32_t perStreamWindow = spdy::kInitialWindow - 1000; + txn->setReceiveWindow(perStreamWindow); + + txn->onIngressHeadersComplete(makeGetRequest()); + eventBase_.loop(); +} + +TEST_F(DownstreamTransactionTest, parse_error_cbs) { + // Test where the transaction gets on parse error and then a body + // callback. This is possible because codecs are stateless between + // frames. + + auto txn = new HTTPTransaction( + TransportDirection::DOWNSTREAM, + HTTPCodec::StreamID(1), 1, transport_, + txnEgressQueue_, transactionTimeouts_.get()); + + HTTPException err(HTTPException::Direction::INGRESS); + err.setHttpStatusCode(400); + + InSequence dummy; + + EXPECT_CALL(handler_, setTransaction(txn)); + EXPECT_CALL(handler_, onError(_)) + .WillOnce(Invoke([] (const HTTPException& ex) { + ASSERT_EQ(ex.getDirection(), HTTPException::Direction::INGRESS); + })); + // onBody() is suppressed since ingress is complete after ingress onError() + // onEOM() is suppressed since ingress is complete after ingress onError() + EXPECT_CALL(transport_, sendAbort(_, _)); + EXPECT_CALL(handler_, detachTransaction()); + EXPECT_CALL(transport_, detach(txn)); + + txn->setHandler(&handler_); + txn->onError(err); + // Since the transaction is already closed for ingress, giving it + // ingress body causes the transaction to be aborted and closed + // immediately. + txn->onIngressBody(makeBuf(10)); + + eventBase_.loop(); +} diff --git a/proxygen/lib/http/session/test/HTTPDownstreamSessionTest.cpp b/proxygen/lib/http/session/test/HTTPDownstreamSessionTest.cpp new file mode 100644 index 0000000000..ad0e39ea97 --- /dev/null +++ b/proxygen/lib/http/session/test/HTTPDownstreamSessionTest.cpp @@ -0,0 +1,1367 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/test/MockHTTPCodec.h" +#include "proxygen/lib/http/codec/test/TestUtils.h" +#include "proxygen/lib/http/session/HTTPDirectResponseHandler.h" +#include "proxygen/lib/http/session/HTTPDownstreamSession.h" +#include "proxygen/lib/http/session/HTTPSession.h" +#include "proxygen/lib/http/session/test/HTTPSessionMocks.h" +#include "proxygen/lib/http/session/test/HTTPSessionTest.h" +#include "proxygen/lib/http/session/test/TestUtils.h" +#include "proxygen/lib/test/TestAsyncTransport.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace apache::thrift::async; +using namespace apache::thrift::test; +using namespace apache::thrift::transport; +using namespace folly::wangle; +using namespace folly; +using namespace proxygen; +using namespace std; +using namespace testing; + +struct HTTP1xCodecPair { + typedef HTTP1xCodec Codec; + static const int version = 1; +}; + +struct SPDY2CodecPair { + typedef SPDYCodec Codec; + static const SPDYVersion version = SPDYVersion::SPDY2; +}; + +struct SPDY3CodecPair { + typedef SPDYCodec Codec; + static const SPDYVersion version = SPDYVersion::SPDY3; +}; + +struct SPDY3_1CodecPair { + typedef SPDYCodec Codec; + static const SPDYVersion version = SPDYVersion::SPDY3_1; +}; + +template +class HTTPDownstreamTest : public testing::Test { + public: + HTTPDownstreamTest() + : eventBase_(), + transport_(new TestAsyncTransport(&eventBase_)), + transactionTimeouts_(makeTimeoutSet(&eventBase_)) { + EXPECT_CALL(mockController_, attachSession(_)); + httpSession_ = new HTTPDownstreamSession( + transactionTimeouts_.get(), + std::move(TAsyncTransport::UniquePtr(transport_)), + localAddr, peerAddr, + &mockController_, + std::move(makeServerCodec( + C::version)), + mockTransportInfo /* no stats for now */); + httpSession_->startNow(); + } + + void SetUp() { + } + + void addSingleByteReads(const char* data, int64_t delay=0) { + for (const char* p = data; *p != '\0'; ++p) { + transport_->addReadEvent(p, 1, delay); + } + } + + void testPriorities(HTTPCodec& clientCodec, uint32_t numPriorities); + + void testChunks(bool trailers); + + void parseOutput(HTTPCodec& clientCodec) { + IOBufQueue stream(IOBufQueue::cacheChainLength()); + auto writeEvents = transport_->getWriteEvents(); + for (auto event: *writeEvents) { + auto vec = event->getIoVec(); + for (auto i = 0; i < event->getCount(); i++) { + unique_ptr buf( + std::move(IOBuf::wrapBuffer(vec[i].iov_base, vec[i].iov_len))); + stream.append(std::move(buf)); + uint32_t consumed = clientCodec.onIngress(*stream.front()); + stream.split(consumed); + } + } + EXPECT_EQ(stream.chainLength(), 0); + } + protected: + EventBase eventBase_; + TestAsyncTransport* transport_; // invalid once httpSession_ is destroyed + TAsyncTimeoutSet::UniquePtr transactionTimeouts_; + StrictMock mockController_; + HTTPDownstreamSession* httpSession_; +}; + +// Uses TestAsyncTransport +typedef HTTPDownstreamTest HTTPDownstreamSessionTest; +typedef HTTPDownstreamTest SPDY2DownstreamSessionTest; +typedef HTTPDownstreamTest SPDY3DownstreamSessionTest; + +TEST_F(HTTPDownstreamSessionTest, immediate_eof) { + // Send EOF without any request data + EXPECT_CALL(mockController_, getRequestHandler(_, _)).Times(0); + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, http_1_0_no_headers) { + MockHTTPHandler* handler = new MockHTTPHandler(); + + InSequence dummy; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler)); + + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler->txn_)); + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ("/", msg->getURL()); + EXPECT_EQ("/", msg->getPath()); + EXPECT_EQ("", msg->getQueryString()); + EXPECT_EQ(1, msg->getHTTPVersion().first); + EXPECT_EQ(0, msg->getHTTPVersion().second); + })); + EXPECT_CALL(*handler, onEOM()) + .WillOnce(InvokeWithoutArgs(handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(*handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler; })); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("GET / HTTP/1.0\r\n\r\n", 0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, http_1_0_no_headers_eof) { + MockHTTPHandler* handler = new MockHTTPHandler(); + + InSequence dummy; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler)); + + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler->txn_)); + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ("http://example.com/foo?bar", msg->getURL()); + EXPECT_EQ("/foo", msg->getPath()); + EXPECT_EQ("bar", msg->getQueryString()); + EXPECT_EQ(1, msg->getHTTPVersion().first); + EXPECT_EQ(0, msg->getHTTPVersion().second); + })); + EXPECT_CALL(*handler, onEOM()) + .WillOnce(InvokeWithoutArgs(handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(*handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler; })); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("GET http://example.com/foo?bar HTTP/1.0\r\n\r\n", + 0); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, single_bytes) { + MockHTTPHandler* handler = new MockHTTPHandler(); + + InSequence dummy; + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler)); + + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler->txn_)); + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + const HTTPHeaders& hdrs = msg->getHeaders(); + EXPECT_EQ(2, hdrs.size()); + EXPECT_TRUE(hdrs.exists("host")); + EXPECT_TRUE(hdrs.exists("connection")); + + EXPECT_FALSE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ("/somepath.php?param=foo", msg->getURL()); + EXPECT_EQ("/somepath.php", msg->getPath()); + EXPECT_EQ("param=foo", msg->getQueryString()); + EXPECT_EQ(1, msg->getHTTPVersion().first); + EXPECT_EQ(1, msg->getHTTPVersion().second); + })); + EXPECT_CALL(*handler, onEOM()) + .WillOnce(InvokeWithoutArgs(handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(*handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler; })); + + EXPECT_CALL(mockController_, detachSession(_)); + + addSingleByteReads("GET /somepath.php?param=foo HTTP/1.1\r\n" + "Host: example.com\r\n" + "Connection: close\r\n" + "\r\n"); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, single_bytes_with_body) { + MockHTTPHandler* handler = new MockHTTPHandler(); + + InSequence dummy; + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler)); + + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler->txn_)); + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + const HTTPHeaders& hdrs = msg->getHeaders(); + EXPECT_EQ(3, hdrs.size()); + EXPECT_TRUE(hdrs.exists("host")); + EXPECT_TRUE(hdrs.exists("content-length")); + EXPECT_TRUE(hdrs.exists("myheader")); + + EXPECT_FALSE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ("/somepath.php?param=foo", msg->getURL()); + EXPECT_EQ("/somepath.php", msg->getPath()); + EXPECT_EQ("param=foo", msg->getQueryString()); + EXPECT_EQ(1, msg->getHTTPVersion().first); + EXPECT_EQ(1, msg->getHTTPVersion().second); + })); + EXPECT_CALL(*handler, onBody(_)) + .WillOnce(ExpectString("1")) + .WillOnce(ExpectString("2")) + .WillOnce(ExpectString("3")) + .WillOnce(ExpectString("4")) + .WillOnce(ExpectString("5")); + EXPECT_CALL(*handler, onEOM()) + .WillOnce(InvokeWithoutArgs(handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(*handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler; })); + + EXPECT_CALL(mockController_, detachSession(_)); + + addSingleByteReads("POST /somepath.php?param=foo HTTP/1.1\r\n" + "Host: example.com\r\n" + "MyHeader: FooBar\r\n" + "Content-Length: 5\r\n" + "\r\n" + "12345"); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, split_body) { + MockHTTPHandler* handler = new MockHTTPHandler(); + + InSequence dummy; + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler)); + + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler->txn_)); + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + const HTTPHeaders& hdrs = msg->getHeaders(); + EXPECT_EQ(2, hdrs.size()); + })); + EXPECT_CALL(*handler, onBody(_)) + .WillOnce(ExpectString("12345")) + .WillOnce(ExpectString("abcde")); + EXPECT_CALL(*handler, onEOM()) + .WillOnce(InvokeWithoutArgs(handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(*handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler; })); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Length: 10\r\n" + "\r\n" + "12345", 0); + transport_->addReadEvent("abcde", 5); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, post_chunked) { + MockHTTPHandler* handler = new MockHTTPHandler(); + + InSequence dummy; + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler)); + + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler->txn_)); + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + const HTTPHeaders& hdrs = msg->getHeaders(); + EXPECT_EQ(3, hdrs.size()); + EXPECT_TRUE(hdrs.exists("host")); + EXPECT_TRUE(hdrs.exists("content-type")); + EXPECT_TRUE(hdrs.exists("transfer-encoding")); + EXPECT_TRUE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ("http://example.com/cgi-bin/foo.aspx?abc&def", + msg->getURL()); + EXPECT_EQ("/cgi-bin/foo.aspx", msg->getPath()); + EXPECT_EQ("abc&def", msg->getQueryString()); + EXPECT_EQ(1, msg->getHTTPVersion().first); + EXPECT_EQ(1, msg->getHTTPVersion().second); + })); + EXPECT_CALL(*handler, onChunkHeader(3)); + EXPECT_CALL(*handler, onBody(_)) + .WillOnce(ExpectString("bar")); + EXPECT_CALL(*handler, onChunkComplete()); + EXPECT_CALL(*handler, onChunkHeader(0x22)); + EXPECT_CALL(*handler, onBody(_)) + .WillOnce(ExpectString("0123456789abcdef\nfedcba9876543210\n")); + EXPECT_CALL(*handler, onChunkComplete()); + EXPECT_CALL(*handler, onChunkHeader(3)); + EXPECT_CALL(*handler, onBody(_)) + .WillOnce(ExpectString("foo")); + EXPECT_CALL(*handler, onChunkComplete()); + EXPECT_CALL(*handler, onEOM()) + .WillOnce(InvokeWithoutArgs(handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(*handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler; })); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("POST http://example.com/cgi-bin/foo.aspx?abc&def " + "HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Type: text/pla", 0); + transport_->addReadEvent("in; charset=utf-8\r\n" + "Transfer-encoding: chunked\r\n" + "\r", 2); + transport_->addReadEvent("\n" + "3\r\n" + "bar\r\n" + "22\r\n" + "0123456789abcdef\n" + "fedcba9876543210\n" + "\r\n" + "3\r", 3); + transport_->addReadEvent("\n" + "foo\r\n" + "0\r\n\r\n", 1); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, multi_message) { + MockHTTPHandler* handler1 = new MockHTTPHandler(); + MockHTTPHandler* handler2 = new MockHTTPHandler(); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler1)) + .WillOnce(Return(handler2)); + + InSequence dummy; + EXPECT_CALL(*handler1, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler1->txn_)); + EXPECT_CALL(*handler1, onHeadersComplete(_)); + EXPECT_CALL(*handler1, onBody(_)) + .WillOnce(ExpectString("foo")) + .WillOnce(ExpectString("bar9876")); + EXPECT_CALL(*handler1, onEOM()) + .WillOnce(InvokeWithoutArgs(handler1, &MockHTTPHandler::sendReply)); + EXPECT_CALL(*handler1, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler1; })); + + EXPECT_CALL(*handler2, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler2->txn_)); + EXPECT_CALL(*handler2, onHeadersComplete(_)); + EXPECT_CALL(*handler2, onChunkHeader(0xa)); + EXPECT_CALL(*handler2, onBody(_)) + .WillOnce(ExpectString("some ")) + .WillOnce(ExpectString("data\n")); + EXPECT_CALL(*handler2, onChunkComplete()); + EXPECT_CALL(*handler2, onEOM()) + .WillOnce(InvokeWithoutArgs(handler2, &MockHTTPHandler::terminate)); + EXPECT_CALL(*handler2, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler2; })); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Length: 10\r\n" + "\r\n" + "foo", 0); + transport_->addReadEvent("bar9876" + "POST /foo HTTP/1.1\r\n" + "Host: exa", 2); + transport_->addReadEvent("mple.com\r\n" + "Connection: close\r\n" + "Trans", 0); + transport_->addReadEvent("fer-encoding: chunked\r\n" + "\r\n", 2); + transport_->addReadEvent("a\r\nsome ", 0); + transport_->addReadEvent("data\n\r\n0\r\n\r\n", 2); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, connect) { + StrictMock handler; + + InSequence dummy; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + + // Send HTTP 200 OK to accept the CONNECT request + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&handler] (std::shared_ptr msg) { + handler.sendHeaders(200, 100); + })); + + EXPECT_CALL(handler, onUpgrade(_)); + + // Data should be received using onBody + EXPECT_CALL(handler, onBody(_)) + .WillOnce(ExpectString("12345")) + .WillOnce(ExpectString("abcde")); + EXPECT_CALL(handler, onEOM()) + .WillOnce(InvokeWithoutArgs(&handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(handler, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("CONNECT test HTTP/1.1\r\n" + "\r\n" + "12345", 0); + transport_->addReadEvent("abcde", 5); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, connect_rejected) { + StrictMock handler; + + InSequence dummy; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + + // Send HTTP 400 to reject the CONNECT request + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&handler] (std::shared_ptr msg) { + handler.sendReplyCode(400); + })); + + EXPECT_CALL(handler, onEOM()) + .WillOnce(InvokeWithoutArgs(&handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(handler, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("CONNECT test HTTP/1.1\r\n" + "\r\n" + "12345", 0); + transport_->addReadEvent("abcde", 5); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, http_upgrade) { + StrictMock handler; + + InSequence dummy; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + + // Send HTTP 101 Switching Protocls to accept the upgrade request + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&handler] (std::shared_ptr msg) { + handler.sendHeaders(101, 100); + })); + + // Send the response in the new protocol after upgrade + EXPECT_CALL(handler, onUpgrade(_)) + .WillOnce(Invoke([&handler] (UpgradeProtocol protocol) { + handler.sendReplyCode(100); + })); + + EXPECT_CALL(handler, onEOM()) + .WillOnce(InvokeWithoutArgs(&handler, &MockHTTPHandler::terminate)); + EXPECT_CALL(handler, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("GET /upgrade HTTP/1.1\r\n" + "Upgrade: TEST/1.0\r\n" + "Connection: upgrade\r\n" + "\r\n", 0); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST(HTTPDownstreamTest, parse_error_no_txn) { + // 1) Get a parse error on SYN_STREAM for streamID == 1 + // 2) Expect that the codec should be asked to generate an abort on + // streamID==1 + EventBase evb; + + // Setup the controller and its expecations. + NiceMock mockController; + + // Setup the codec, its callbacks, and its expectations. + auto codec = makeDownstreamParallelCodec(); + HTTPCodec::Callback* codecCallback = nullptr; + EXPECT_CALL(*codec, setCallback(_)) + .WillRepeatedly(SaveArg<0>(&codecCallback)); + // Expect egress abort for streamID == 1 + EXPECT_CALL(*codec, generateRstStream(_, 1, _)); + + // Setup transport + bool transportGood = true; + auto transport = newMockTransport(&evb); + EXPECT_CALL(*transport, good()) + .WillRepeatedly(ReturnPointee(&transportGood)); + EXPECT_CALL(*transport, closeNow()) + .WillRepeatedly(Assign(&transportGood, false)); + EXPECT_CALL(*transport, writeChain(_, _, _)) + .WillRepeatedly(Invoke([&] (TAsyncTransport::WriteCallback* callback, + const shared_ptr iob, + WriteFlags flags) { + callback->writeSuccess(); + })); + + // Create the downstream session, thus initializing codecCallback + auto transactionTimeouts = makeInternalTimeoutSet(&evb); + auto session = new HTTPDownstreamSession( + transactionTimeouts.get(), + TAsyncTransport::UniquePtr(transport), + localAddr, peerAddr, + &mockController, std::move(codec), + mockTransportInfo); + session->startNow(); + HTTPException ex(HTTPException::Direction::INGRESS_AND_EGRESS, "foo"); + ex.setProxygenError(kErrorParseHeader); + ex.setCodecStatusCode(ErrorCode::REFUSED_STREAM); + codecCallback->onError(HTTPCodec::StreamID(1), ex, true); + + // cleanup + session->shutdownTransportWithReset(kErrorConnectionReset); + evb.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, trailers) { + testChunks(true); +} + +TEST_F(HTTPDownstreamSessionTest, explicit_chunks) { + testChunks(false); +} + +template +void HTTPDownstreamTest::testChunks(bool trailers) { + StrictMock handler; + + InSequence dummy; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + EXPECT_CALL(handler, onHeadersComplete(_)); + EXPECT_CALL(handler, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler, trailers] () { + handler.sendChunkedReplyWithBody(200, 100, 17, trailers); + })); + EXPECT_CALL(handler, detachTransaction()); + + transport_->addReadEvent("GET / HTTP/1.1\r\n" + "\r\n", 0); + transport_->addReadEOF(0); + transport_->startReadEvents(); + HTTPSession::DestructorGuard g(httpSession_); + eventBase_.loop(); + + HTTP1xCodec clientCodec(TransportDirection::UPSTREAM); + NiceMock callbacks; + + EXPECT_CALL(callbacks, onMessageBegin(1, _)) + .Times(1); + EXPECT_CALL(callbacks, onHeadersComplete(1, _)) + .Times(1); + for (int i = 0; i < 6; i++) { + EXPECT_CALL(callbacks, onChunkHeader(1, _)); + EXPECT_CALL(callbacks, onBody(1, _)); + EXPECT_CALL(callbacks, onChunkComplete(1)); + } + if (trailers) { + EXPECT_CALL(callbacks, onTrailersComplete(1, _)); + } + EXPECT_CALL(callbacks, onMessageComplete(1, _)); + + clientCodec.setCallback(&callbacks); + parseOutput(clientCodec); + EXPECT_CALL(mockController_, detachSession(_)); +} + +TEST_F(HTTPDownstreamSessionTest, http_drain) { + StrictMock handler1; + StrictMock handler2; + + InSequence dummy; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler1.txn_)); + + EXPECT_CALL(handler1, onHeadersComplete(_)) + .WillOnce(Invoke([this, &handler1] (std::shared_ptr msg) { + handler1.sendHeaders(200, 100); + httpSession_->notifyPendingShutdown(); + })); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1] { + handler1.sendBody(100); + handler1.txn_->sendEOM(); + })); + EXPECT_CALL(handler1, detachTransaction()); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler2)); + + EXPECT_CALL(handler2, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler2.txn_)); + + EXPECT_CALL(handler2, onHeadersComplete(_)) + .WillOnce(Invoke([this, &handler2] (std::shared_ptr msg) { + handler2.sendHeaders(200, 100); + })); + EXPECT_CALL(handler2, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler2] { + handler2.sendBody(100); + handler2.txn_->sendEOM(); + })); + EXPECT_CALL(handler2, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("GET / HTTP/1.1\r\n" + "\r\n", 0); + transport_->addReadEvent("GET / HTTP/1.1\r\n" + "\r\n", 0); + + transport_->startReadEvents(); + eventBase_.loop(); +} + +// 1) receive full request +// 2) notify pending shutdown +// 3) wait for session read timeout -> should be ignored +// 4) response completed +TEST_F(HTTPDownstreamSessionTest, http_drain_long_running) { + StrictMock handler; + + InSequence enforceSequence; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + // txn1, as soon as headers go out, mark set code shouldShutdown=true + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([this, &handler] (std::shared_ptr msg) { + httpSession_->notifyPendingShutdown(); + eventBase_.runAfterDelay([this] { + // simulate read timeout + httpSession_->timeoutExpired(); + }, 100); + eventBase_.runAfterDelay([&handler] { + handler.sendReplyWithBody(200, 100); + }, 200); + })); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent("GET / HTTP/1.1\r\n" + "\r\n", 0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, early_abort) { + MockHTTPHandler* handler = new MockHTTPHandler(); + + InSequence dummy; + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler)); + + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(Invoke([&] (HTTPTransaction* txn) { + handler->txn_ = txn; + handler->txn_->sendAbort(); + })); + EXPECT_CALL(*handler, onHeadersComplete(_)) + .Times(0); + EXPECT_CALL(*handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { delete handler; })); + + EXPECT_CALL(mockController_, detachSession(_)); + + addSingleByteReads("GET /somepath.php?param=foo HTTP/1.1\r\n" + "Host: example.com\r\n" + "Connection: close\r\n" + "\r\n"); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, http_writes_draining_timeout) { + IOBufQueue requests; + HTTPMessage req = getGetRequest(); + MockHTTPHandler handler1; + HTTP1xCodec clientCodec(TransportDirection::UPSTREAM); + auto streamID = HTTPCodec::StreamID(0); + clientCodec.generateHeader(requests, streamID, req); + clientCodec.generateEOM(requests, streamID); + clientCodec.generateHeader(requests, streamID, req); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + EXPECT_CALL(mockController_, detachSession(_)); + + InSequence handlerSequence; + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1, this] { + transport_->pauseWrites(); + handler1.sendHeaders(200, 1000); + })); + EXPECT_CALL(handler1, onEgressPaused()); + EXPECT_CALL(handler1, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& ex) { + ASSERT_EQ(ex.getProxygenError(), kErrorWriteTimeout); + handler1.txn_->sendAbort(); + })); + EXPECT_CALL(handler1, detachTransaction()); + + transport_->addReadEvent(requests, 10); + transport_->startReadEvents(); + eventBase_.loop(); +} + +// Send a 1.0 request, egress the EOM with the last body chunk on a paused +// socket, and let it timeout. shutdownTransportWithReset will result in a call +// to removeTransaction with writesDraining_=true +TEST_F(HTTPDownstreamSessionTest, write_timeout) { + IOBufQueue requests; + MockHTTPHandler handler1; + HTTPMessage req = getGetRequest(); + req.setHTTPVersion(1, 0); + HTTP1xCodec clientCodec(TransportDirection::UPSTREAM); + auto streamID = HTTPCodec::StreamID(0); + clientCodec.generateHeader(requests, streamID, req); + clientCodec.generateEOM(requests, streamID); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + + InSequence handlerSequence; + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1, this] { + handler1.sendHeaders(200, 100); + eventBase_.runAfterDelay([&handler1, this] { + transport_->pauseWrites(); + handler1.sendBody(100); + handler1.txn_->sendEOM(); + }, 50); + })); + EXPECT_CALL(handler1, onEgressPaused()); + EXPECT_CALL(handler1, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& ex) { + ASSERT_EQ(ex.getProxygenError(), kErrorWriteTimeout); + })); + EXPECT_CALL(handler1, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent(requests, 0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +// Send an abort from the write timeout path while pipelining +TEST_F(HTTPDownstreamSessionTest, write_timeout_pipeline) { + IOBufQueue requests; + MockHTTPHandler handler1; + + HTTPMessage req = getGetRequest(); + HTTP1xCodec clientCodec(TransportDirection::UPSTREAM); + const char* buf = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n" + "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + + InSequence handlerSequence; + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1, this] { + handler1.sendHeaders(200, 100); + eventBase_.runAfterDelay([&handler1, this] { + transport_->pauseWrites(); + handler1.sendBody(100); + handler1.txn_->sendEOM(); + }, 50); + })); + EXPECT_CALL(handler1, onEgressPaused()); + EXPECT_CALL(handler1, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& ex) { + ASSERT_EQ(ex.getProxygenError(), kErrorWriteTimeout); + handler1.txn_->sendAbort(); + })); + EXPECT_CALL(handler1, detachTransaction()); + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent(buf, 0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, body_packetization) { + IOBufQueue requests; + MockHTTPHandler handler1; + HTTPMessage req = getGetRequest(); + req.setHTTPVersion(1, 0); + req.setWantsKeepalive(false); + HTTP1xCodec clientCodec(TransportDirection::UPSTREAM); + auto streamID = HTTPCodec::StreamID(0); + clientCodec.generateHeader(requests, streamID, req); + clientCodec.generateEOM(requests, streamID); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + + InSequence handlerSequence; + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1, this] { + handler1.sendReplyWithBody(200, 100); + })); + EXPECT_CALL(handler1, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent(requests, 0); + transport_->startReadEvents(); + + // Keep the session around even after the event base loop completes so we can + // read the counters on a valid object. + HTTPSession::DestructorGuard g(httpSession_); + eventBase_.loop(); + + EXPECT_EQ(transport_->getWriteEvents()->size(), 1); +} + +TEST_F(HTTPDownstreamSessionTest, http_malformed_pkt1) { + // Create a HTTP connection and keep sending just '\n' to the HTTP1xCodec. + std::string data(90000, '\n'); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent(data.c_str(), data.length(), 0); + transport_->addReadEOF(0); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TEST_F(HTTPDownstreamSessionTest, big_explcit_chunk_write) { + // even when the handler does a massive write, the transport only gets small + // writes + IOBufQueue requests; + HTTPMessage req = getGetRequest(); + HTTP1xCodec clientCodec(TransportDirection::UPSTREAM); + auto streamID = HTTPCodec::StreamID(0); + clientCodec.generateHeader(requests, streamID, req); + clientCodec.generateEOM(requests, streamID); + MockHTTPHandler handler; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(Invoke([&handler] (HTTPTransaction* txn) { + handler.txn_ = txn; })); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&handler] (std::shared_ptr msg) { + handler.sendHeaders(200, 100, false); + size_t len = 16 * 1024 * 1024; + handler.txn_->sendChunkHeader(len); + auto chunk = makeBuf(len); + handler.txn_->sendBody(std::move(chunk)); + handler.txn_->sendChunkTerminator(); + handler.txn_->sendEOM(); + })); + EXPECT_CALL(handler, onEgressPaused()); + EXPECT_CALL(handler, onEgressResumed()); + EXPECT_CALL(handler, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent(requests, 0); + transport_->startReadEvents(); + + // Keep the session around even after the event base loop completes so we can + // read the counters on a valid object. + HTTPSession::DestructorGuard g(httpSession_); + eventBase_.loop(); + + EXPECT_GT(transport_->getWriteEvents()->size(), 3000); +} + +TEST_F(SPDY2DownstreamSessionTest, spdy_prio) { + SPDYCodec clientCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY2); + testPriorities(clientCodec, 4); +} + +TEST_F(SPDY3DownstreamSessionTest, spdy_prio) { + SPDYCodec clientCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + testPriorities(clientCodec, 8); +} + +template +void HTTPDownstreamTest::testPriorities( + HTTPCodec& clientCodec, uint32_t numPriorities) { + IOBufQueue requests; + uint32_t iterations = 10; + uint32_t maxPriority = numPriorities - 1; + HTTPMessage req = getGetRequest(); + auto streamID = HTTPCodec::StreamID(1); + for (int pri = numPriorities - 1; pri >= 0; pri--) { + req.setPriority(pri); + for (auto i = 0; i < iterations; i++) { + clientCodec.generateHeader(requests, streamID, req); + clientCodec.generateEOM(requests, streamID); + MockHTTPHandler* handler = new MockHTTPHandler(); + InSequence handlerSequence; + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler)); + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(Invoke([handler] (HTTPTransaction* txn) { + handler->txn_ = txn; })); + EXPECT_CALL(*handler, onHeadersComplete(_)); + EXPECT_CALL(*handler, onEOM()) + .WillOnce(InvokeWithoutArgs([handler] { + handler->sendReplyWithBody(200, 1000); + })); + EXPECT_CALL(*handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([handler] { delete handler; })); + streamID += 2; + } + } + + unique_ptr head = requests.move(); + head->coalesce(); + transport_->addReadEvent(head->data(), head->length(), 0); + transport_->startReadEvents(); + eventBase_.loop(); + + NiceMock callbacks; + + std::list streams; + EXPECT_CALL(callbacks, onMessageBegin(_, _)) + .Times(iterations * numPriorities); + EXPECT_CALL(callbacks, onHeadersComplete(_, _)) + .Times(iterations * numPriorities); + // body is variable and hence ignored + EXPECT_CALL(callbacks, onMessageComplete(_, _)) + .Times(iterations * numPriorities) + .WillRepeatedly(Invoke([&] (HTTPCodec::StreamID stream, bool upgrade) { + streams.push_back(stream); + })); + + clientCodec.setCallback(&callbacks); + parseOutput(clientCodec); + + // transactions finish in priority order (higher streamIDs first) + EXPECT_EQ(streams.size(), iterations * numPriorities); + auto txn = streams.begin(); + for (int band = maxPriority; band >= 0; band--) { + auto upperID = iterations * 2 * (band + 1); + auto lowerID = iterations * 2 * band; + for (auto i = 0; i < iterations; i++) { + EXPECT_LE(lowerID, (uint32_t)*txn); + EXPECT_GE(upperID, (uint32_t)*txn); + ++txn; + } + } +} + +// Verifies that the read timeout is not running when no ingress is expected/ +// required to proceed +TEST_F(SPDY3DownstreamSessionTest, spdy_timeout) { + IOBufQueue requests; + HTTPMessage req = getGetRequest(); + SPDYCodec clientCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + for (auto streamID = HTTPCodec::StreamID(1); streamID <= 3; streamID += 2) { + clientCodec.generateHeader(requests, streamID, req); + clientCodec.generateEOM(requests, streamID); + } + MockHTTPHandler* handler1 = new StrictMock(); + MockHTTPHandler* handler2 = new StrictMock(); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(handler1)) + .WillOnce(Return(handler2)); + + InSequence handlerSequence; + EXPECT_CALL(*handler1, setTransaction(_)) + .WillOnce(Invoke([handler1] (HTTPTransaction* txn) { + handler1->txn_ = txn; })); + EXPECT_CALL(*handler1, onHeadersComplete(_)) + .WillOnce(InvokeWithoutArgs([this] { transport_->pauseWrites(); })); + EXPECT_CALL(*handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([handler1] { + handler1->sendHeaders(200, 1000); + handler1->sendBody(100); + })); + EXPECT_CALL(*handler1, onEgressPaused()); + EXPECT_CALL(*handler2, setTransaction(_)) + .WillOnce(Invoke([handler2] (HTTPTransaction* txn) { + handler2->txn_ = txn; })); + EXPECT_CALL(*handler2, onEgressPaused()); + EXPECT_CALL(*handler2, onHeadersComplete(_)); + EXPECT_CALL(*handler2, onEOM()) + .WillOnce(InvokeWithoutArgs([handler2, this] { + // This transaction should start egress paused. We've received the + // EOM, so the timeout shouldn't be running delay 400ms and resume + // writes, this keeps txn1 from getting a write timeout + eventBase_.runAfterDelay([this] { + transport_->resumeWrites(); + }, 400); + })); + EXPECT_CALL(*handler1, onEgressResumed()) + .WillOnce(InvokeWithoutArgs([handler1] { handler1->txn_->sendEOM(); })); + EXPECT_CALL(*handler2, onEgressResumed()) + .WillOnce(InvokeWithoutArgs([handler2, this] { + // delay an additional 200ms. The total 600ms delay shouldn't fire + // onTimeout + eventBase_.runAfterDelay([handler2] { + handler2->sendReplyWithBody(200, 1000); }, 200 + ); + })); + EXPECT_CALL(*handler1, detachTransaction()) + .WillOnce(InvokeWithoutArgs([handler1] { delete handler1; })); + EXPECT_CALL(*handler2, detachTransaction()) + .WillOnce(InvokeWithoutArgs([handler2] { delete handler2; })); + + transport_->addReadEvent(requests, 10); + transport_->startReadEvents(); + eventBase_.loop(); +} + +// Verifies that the read timer is running while a transaction is blocked +// on a window update +TEST_F(SPDY3DownstreamSessionTest, spdy_timeout_win) { + IOBufQueue requests; + HTTPMessage req = getGetRequest(); + SPDYCodec clientCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + auto streamID = HTTPCodec::StreamID(1); + clientCodec.getEgressSettings()->setSetting(SettingsId::INITIAL_WINDOW_SIZE, + 500); + clientCodec.generateSettings(requests); + clientCodec.generateHeader(requests, streamID, req, 0, nullptr); + clientCodec.generateEOM(requests, streamID); + StrictMock handler; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + InSequence handlerSequence; + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(Invoke([&] (HTTPTransaction* txn) { + handler.txn_ = txn; })); + EXPECT_CALL(handler, onHeadersComplete(_)); + EXPECT_CALL(handler, onEOM()) + .WillOnce(InvokeWithoutArgs([&] { + handler.sendReplyWithBody(200, 1000); + })); + EXPECT_CALL(handler, onEgressPaused()); + EXPECT_CALL(handler, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& ex) { + ASSERT_EQ(ex.getProxygenError(), kErrorTimeout); + handler.terminate(); + })); + EXPECT_CALL(handler, detachTransaction()); + + transport_->addReadEvent(requests, 10); + transport_->startReadEvents(); + eventBase_.loop(); +} + +TYPED_TEST_CASE_P(HTTPDownstreamTest); + +TYPED_TEST_P(HTTPDownstreamTest, testWritesDraining) { + IOBufQueue requests; + HTTPMessage req = getGetRequest(); + auto clientCodec = + makeClientCodec(TypeParam::version); + auto badCodec = + makeServerCodec(TypeParam::version); + auto streamID = HTTPCodec::StreamID(1); + clientCodec->generateHeader(requests, streamID, req); + clientCodec->generateEOM(requests, streamID); + streamID += 1; + badCodec->generateHeader(requests, streamID, req, 1); + MockHTTPHandler handler1; + + EXPECT_CALL(this->mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + EXPECT_CALL(this->mockController_, detachSession(_)); + + InSequence handlerSequence; + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + EXPECT_CALL(handler1, onEOM()); + EXPECT_CALL(handler1, onError(_)); + EXPECT_CALL(handler1, detachTransaction()); + + this->transport_->addReadEvent(requests, 10); + this->transport_->startReadEvents(); + this->eventBase_.loop(); +} + +TYPED_TEST_P(HTTPDownstreamTest, testChromeShutdownPaused) { + IOBufQueue requests; + MockHTTPHandler handler1; + HTTPMessage req = getPostRequest(); + req.getHeaders().add(HTTP_HEADER_USER_AGENT, "Chrome/26.0"); + auto clientCodec = + makeClientCodec(TypeParam::version); + auto streamID = HTTPCodec::StreamID(1); + clientCodec->generateHeader(requests, streamID, req); + + EXPECT_CALL(this->mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + EXPECT_CALL(this->mockController_, detachSession(_)); + + InSequence handlerSequence; + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)) + .WillOnce(InvokeWithoutArgs([&handler1, this] { + this->transport_->pauseWrites(); + ((ManagedConnection *)this->httpSession_)->notifyPendingShutdown(); + // simulate forced shutdown + this->eventBase_.runAfterDelay([this] { + ((ManagedConnection *)this->httpSession_)->dropConnection(); }, + 50); + })); + EXPECT_CALL(handler1, onEgressPaused()); + EXPECT_CALL(handler1, onError(_)); + EXPECT_CALL(handler1, detachTransaction()); + + this->transport_->addReadEvent(requests, 0); + this->transport_->startReadEvents(); + this->eventBase_.loop(); +} + +// Pause writes, send reply +// Force a socket error in the same event loop cycle as the write timeout +// expiration +TYPED_TEST_P(HTTPDownstreamTest, testChromeShutdownOnError) { + IOBufQueue requests; + MockHTTPHandler handler1; + HTTPMessage req = getGetRequest(); + req.getHeaders().add(HTTP_HEADER_USER_AGENT, "Chrome/26.0"); + auto clientCodec = + makeClientCodec(TypeParam::version); + auto streamID = HTTPCodec::StreamID(1); + clientCodec->generateHeader(requests, streamID, req); + clientCodec->generateEOM(requests, streamID); + + EXPECT_CALL(this->mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + EXPECT_CALL(this->mockController_, detachSession(_)); + + InSequence handlerSequence; + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + + class SockShutdownTimeout: public TAsyncTimeoutSet::Callback { + public: + explicit SockShutdownTimeout(TestAsyncTransport* transport) + : transport_(transport) {} + + void timeoutExpired() noexcept { + VLOG(4) << "mytimeout expired"; + transport_->shutdownWriteNow(); + } + TestAsyncTransport* transport_; + }; + + SockShutdownTimeout t(this->transport_); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1, this, &t] { + this->transport_->pauseWrites(); + ((ManagedConnection *)this->httpSession_)->notifyPendingShutdown(); + // This has potential to be flaky false negative (the timeout could + // first too early, missing the test condition). + this->transactionTimeouts_->scheduleTimeout(&t); + handler1.sendReplyWithBody(200, 1000); + })); + EXPECT_CALL(handler1, onEgressPaused()); + EXPECT_CALL(handler1, detachTransaction()); + + this->transport_->addReadEvent(requests, 0); + this->transport_->startReadEvents(); + this->eventBase_.loop(); +} + +// Set max streams=1 +// send two spdy requests a few ms apart. +// Block writes +// generate a complete response for txn=1 before parsing txn=3 +// HTTPSession should allow the txn=3 to be served rather than refusing it +TEST_F(SPDY3DownstreamSessionTest, spdy_max_concurrent_streams) { + IOBufQueue requests; + StrictMock handler1; + StrictMock handler2; + HTTPMessage req = getGetRequest(); + req.setHTTPVersion(1, 0); + req.setWantsKeepalive(false); + SPDYCodec clientCodec(TransportDirection::UPSTREAM, + SPDYVersion::SPDY3); + auto streamID = HTTPCodec::StreamID(1); + clientCodec.generateHeader(requests, streamID, req); + clientCodec.generateEOM(requests, streamID); + streamID += 2; + clientCodec.generateHeader(requests, streamID, req); + clientCodec.generateEOM(requests, streamID); + + httpSession_->getCodecFilterChain()->getEgressSettings()->setSetting( + SettingsId::MAX_CONCURRENT_STREAMS, 1); + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)) + .WillOnce(Return(&handler2)); + + InSequence handlerSequence; + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1, this] { + transport_->pauseWrites(); + handler1.sendReplyWithBody(200, 100); + })); + EXPECT_CALL(handler1, onEgressPaused()); + EXPECT_CALL(handler2, setTransaction(_)) + .WillOnce(Invoke([&handler2] (HTTPTransaction* txn) { + handler2.txn_ = txn; })); + EXPECT_CALL(handler2, onEgressPaused()); + EXPECT_CALL(handler2, onHeadersComplete(_)); + EXPECT_CALL(handler2, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler2, this] { + handler2.sendReplyWithBody(200, 100); + eventBase_.runInLoop([this] { + transport_->resumeWrites(); + }); + })); + EXPECT_CALL(handler1, detachTransaction()); + EXPECT_CALL(handler2, onEgressResumed()); + EXPECT_CALL(handler2, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + transport_->addReadEvent(requests, 10); + transport_->startReadEvents(); + transport_->addReadEOF(10); + + eventBase_.loop(); +} + +REGISTER_TYPED_TEST_CASE_P(HTTPDownstreamTest, + testWritesDraining, + testChromeShutdownPaused, + testChromeShutdownOnError); + +typedef ::testing::Types ParallelCodecs; +INSTANTIATE_TYPED_TEST_CASE_P(ParallelCodecs, + HTTPDownstreamTest, + ParallelCodecs); diff --git a/proxygen/lib/http/session/test/HTTPSessionAcceptorTest.cpp b/proxygen/lib/http/session/test/HTTPSessionAcceptorTest.cpp new file mode 100644 index 0000000000..2baf337b93 --- /dev/null +++ b/proxygen/lib/http/session/test/HTTPSessionAcceptorTest.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPSessionAcceptor.h" +#include "proxygen/lib/http/session/test/HTTPSessionMocks.h" +#include "proxygen/lib/utils/TestUtils.h" + +#include +#include + +using namespace proxygen; +using namespace testing; + +using apache::thrift::async::TAsyncSocket; +using apache::thrift::test::MockTAsyncSocket; +using folly::SocketAddress; + +namespace { + +const std::string kTestDir = getContainingDirectory(__FILE__).str(); + +} + +class HTTPTargetSessionAcceptor : public HTTPSessionAcceptor { + public: + explicit HTTPTargetSessionAcceptor(const AcceptorConfiguration& accConfig) + : HTTPSessionAcceptor(accConfig) { + } + + HTTPTransaction::Handler* newHandler(HTTPTransaction& txn, + HTTPMessage* msg) noexcept override { + return new MockHTTPHandler(); + } + + void onCreate(const HTTPSession& session) override{ + EXPECT_EQ(expectedProto_, + getCodecProtocolString(session.getCodec().getProtocol())); + sessionsCreated_++; + } + + void connectionReady(TAsyncSocket::UniquePtr sock, + const SocketAddress& clientAddr, + const std::string& nextProtocolName, + TransportInfo& tinfo) { + HTTPSessionAcceptor::connectionReady(std::move(sock), + clientAddr, + nextProtocolName, + tinfo); + } + + uint32_t sessionsCreated_{0}; + std::string expectedProto_; +}; + +class HTTPSessionAcceptorTestBase : + public ::testing::TestWithParam { + public: + + virtual void setupSSL() { + sslCtxConfig_.setCertificate( + kTestDir + "test_cert1.pem", + kTestDir + "test_cert1.key", + ""); + + sslCtxConfig_.isDefault = true; + config_.sslContextConfigs.emplace_back(sslCtxConfig_); + } + + void SetUp() { + SocketAddress address("127.0.0.1", 0); + config_.bindAddress = address; + setupSSL(); + newAcceptor(); + } + + void newAcceptor() { + acceptor_ = folly::make_unique(config_); + EXPECT_CALL(mockServerSocket_, addAcceptCallback(_, _, _)); + acceptor_->init(&mockServerSocket_, &eventBase_); + } + + protected: + AcceptorConfiguration config_; + SSLContextConfig sslCtxConfig_; + std::unique_ptr acceptor_; + folly::EventBase eventBase_; + apache::thrift::test::MockTAsyncServerSocket mockServerSocket_; +}; + +class HTTPSessionAcceptorTestNPN : + public HTTPSessionAcceptorTestBase {}; +class HTTPSessionAcceptorTestNPNPlaintext : + public HTTPSessionAcceptorTestBase { + public: + void setupSSL() {} +}; +class HTTPSessionAcceptorTestNPNJunk : + public HTTPSessionAcceptorTestBase {}; + +// Verify HTTPSessionAcceptor creates the correct codec based on NPN +TEST_P(HTTPSessionAcceptorTestNPN, npn) { + std::string proto(GetParam()); + if (proto == "") { + acceptor_->expectedProto_ = "http/1.1"; + } else { + acceptor_->expectedProto_ = proto; + } + + TAsyncSocket::UniquePtr sock(new TAsyncSocket(&eventBase_)); + SocketAddress clientAddress; + TransportInfo tinfo; + acceptor_->connectionReady(std::move(sock), clientAddress, proto, tinfo); + EXPECT_EQ(acceptor_->sessionsCreated_, 1); +} + +char const* protos1[] = { "spdy/3", "spdy/2", "http/1.1", "" }; +INSTANTIATE_TEST_CASE_P(NPNPositive, + HTTPSessionAcceptorTestNPN, + ::testing::ValuesIn(protos1)); + +// Verify HTTPSessionAcceptor creates the correct plaintext codec +TEST_P(HTTPSessionAcceptorTestNPNPlaintext, plaintext_protocols) { + std::string proto(GetParam()); + config_.plaintextProtocol = proto; + newAcceptor(); + acceptor_->expectedProto_ = proto; + TAsyncSocket::UniquePtr sock(new TAsyncSocket(&eventBase_)); + SocketAddress clientAddress; + TransportInfo tinfo; + acceptor_->connectionReady(std::move(sock), clientAddress, "", tinfo); + EXPECT_EQ(acceptor_->sessionsCreated_, 1); +} + +char const* protos2[] = { "spdy/3", "spdy/2" }; +INSTANTIATE_TEST_CASE_P(NPNPlaintext, + HTTPSessionAcceptorTestNPNPlaintext, + ::testing::ValuesIn(protos2)); + +// Verify HTTPSessionAcceptor closes the socket on invalid NPN +TEST_F(HTTPSessionAcceptorTestNPNJunk, npn) { + std::string proto("/http/1.1"); + MockTAsyncSocket::UniquePtr sock(new MockTAsyncSocket(&eventBase_)); + SocketAddress clientAddress; + TransportInfo tinfo; + EXPECT_CALL(*sock.get(), closeNow()); + acceptor_->connectionReady(std::move(sock), clientAddress, proto, tinfo); + EXPECT_EQ(acceptor_->sessionsCreated_, 0); +} diff --git a/proxygen/lib/http/session/test/HTTPSessionMocks.h b/proxygen/lib/http/session/test/HTTPSessionMocks.h new file mode 100644 index 0000000000..fa9dff4953 --- /dev/null +++ b/proxygen/lib/http/session/test/HTTPSessionMocks.h @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/http/session/HTTPDownstreamSession.h" +#include "proxygen/lib/http/session/HTTPSessionController.h" +#include "proxygen/lib/http/session/HTTPTransaction.h" + +#include + +#define GMOCK_NOEXCEPT_METHOD0(m, F) GMOCK_METHOD0_(, noexcept,, m, F) +#define GMOCK_NOEXCEPT_METHOD1(m, F) GMOCK_METHOD1_(, noexcept,, m, F) + +namespace proxygen { + +class HTTPHandlerBase { + public: + HTTPHandlerBase() {} + HTTPHandlerBase(HTTPTransaction* txn, HTTPMessage* msg) : + txn_(txn), + msg_(msg) {} + + void terminate() { + txn_->sendAbort(); + } + + void sendHeaders(uint32_t code, uint32_t content_length, + bool keepalive=true) { + HTTPMessage reply; + reply.setStatusCode(code); + reply.setHTTPVersion(1, 1); + reply.setWantsKeepalive(keepalive); + reply.getHeaders().add(HTTP_HEADER_CONTENT_LENGTH, + folly::to(content_length)); + txn_->sendHeaders(reply); + } + + void sendReply() { + sendReplyCode(200); + } + + void sendReplyCode(uint32_t code) { + sendHeaders(code, 0, true); + txn_->sendEOM(); + } + + void sendBody(uint32_t content_length) { + while (content_length > 0) { + uint32_t toSend = std::min(content_length, uint32_t(4096)); + char buf[toSend]; + memset(buf, 'a', toSend); + txn_->sendBody(std::move(folly::IOBuf::copyBuffer(buf, toSend))); + content_length -= toSend; + } + } + + void sendReplyWithBody(uint32_t code, uint32_t content_length, + bool keepalive=true) { + sendHeaders(code, content_length, keepalive); + sendBody(content_length); + txn_->sendEOM(); + } + + void sendChunkedReplyWithBody(uint32_t code, uint32_t content_length, + uint32_t chunkSize, bool hasTrailers) { + HTTPMessage reply; + reply.setStatusCode(code); + reply.setHTTPVersion(1, 1); + reply.setIsChunked(true); + txn_->sendHeaders(reply); + while (content_length > 0) { + uint32_t toSend = std::min(content_length, chunkSize); + char buf[toSend]; + memset(buf, 'a', toSend); + txn_->sendChunkHeader(toSend); + txn_->sendBody(std::move(folly::IOBuf::copyBuffer(buf, toSend))); + txn_->sendChunkTerminator(); + content_length -= toSend; + } + if (hasTrailers) { + HTTPHeaders trailers; + trailers.add("X-Trailer1", "Foo"); + txn_->sendTrailers(trailers); + } + txn_->sendEOM(); + } + + HTTPTransaction* txn_{nullptr}; + std::shared_ptr msg_; +}; + +class MockHTTPHandler : public HTTPHandlerBase, + public HTTPTransaction::Handler { + public: + MockHTTPHandler() {} + MockHTTPHandler(HTTPTransaction& txn, HTTPMessage* msg, + const folly::SocketAddress&) : + HTTPHandlerBase(&txn, msg) {} + + GMOCK_NOEXCEPT_METHOD1(setTransaction, void(HTTPTransaction* txn)); + + GMOCK_NOEXCEPT_METHOD0(detachTransaction, void()); + + virtual void onHeadersComplete(std::unique_ptr msg) + noexcept override { + onHeadersComplete(std::shared_ptr(msg.release())); + } + + GMOCK_NOEXCEPT_METHOD1(onHeadersComplete, + void(std::shared_ptr msg)); + + virtual void onBody(std::unique_ptr chain) noexcept override { + onBody(std::shared_ptr(chain.release())); + } + GMOCK_NOEXCEPT_METHOD1(onBody, void(std::shared_ptr chain)); + + GMOCK_NOEXCEPT_METHOD1(onChunkHeader, void(size_t length)); + + GMOCK_NOEXCEPT_METHOD0(onChunkComplete, void()); + + virtual void onTrailers(std::unique_ptr trailers) + noexcept override { + onTrailers(std::shared_ptr(trailers.release())); + } + + GMOCK_NOEXCEPT_METHOD1(onTrailers, + void(std::shared_ptr trailers)); + + GMOCK_NOEXCEPT_METHOD0(onEOM, void()); + + GMOCK_NOEXCEPT_METHOD1(onUpgrade, void(UpgradeProtocol protocol)); + + GMOCK_NOEXCEPT_METHOD1(onError, void(const HTTPException& error)); + + GMOCK_NOEXCEPT_METHOD0(onEgressPaused, void()); + + GMOCK_NOEXCEPT_METHOD0(onEgressResumed, void()); + + GMOCK_NOEXCEPT_METHOD1(onPushedTransaction, + void(HTTPTransaction*)); +}; + +class MockHTTPPushHandler : public HTTPHandlerBase, + public HTTPTransaction::PushHandler { + public: + MockHTTPPushHandler() {} + MockHTTPPushHandler(HTTPTransaction& txn, HTTPMessage* msg, + const folly::SocketAddress&) : + HTTPHandlerBase(&txn, msg) {} + + GMOCK_NOEXCEPT_METHOD1(setTransaction, void(HTTPTransaction* txn)); + + GMOCK_NOEXCEPT_METHOD0(detachTransaction, void()); + + GMOCK_NOEXCEPT_METHOD1(onError, void(const HTTPException& error)); + + GMOCK_NOEXCEPT_METHOD0(onEgressPaused, void()); + + GMOCK_NOEXCEPT_METHOD0(onEgressResumed, void()); + + void sendPushHeaders(const std::string& path, + const std::string& host, + uint32_t content_length) { + HTTPMessage push; + push.setURL(path); + push.getHeaders().set(HTTP_HEADER_HOST, host); + push.getHeaders().add(HTTP_HEADER_CONTENT_LENGTH, + folly::to(content_length)); + txn_->sendHeaders(push); + } +}; + +class MockController : public HTTPSessionController { + public: + MOCK_METHOD2(getRequestHandler, HTTPTransactionHandler*( + HTTPTransaction&, HTTPMessage* msg)); + + MOCK_METHOD3(getParseErrorHandler, HTTPTransactionHandler*( + HTTPTransaction*, + const HTTPException&, + const folly::SocketAddress&)); + + MOCK_METHOD2(getTransactionTimeoutHandler, HTTPTransactionHandler*( + HTTPTransaction* txn, + const folly::SocketAddress&)); + + MOCK_METHOD1(attachSession, void(HTTPSession*)); + MOCK_METHOD1(detachSession, void(const HTTPSession*)); +}; + +ACTION_P(ExpectString, expected) { + std::string bodystr((const char *)arg0->data(), arg0->length()); + EXPECT_EQ(bodystr, expected); +} + +class MockHTTPSessionInfoCallback: public HTTPSession::InfoCallback { + public: + MOCK_METHOD1(onCreate, void(const HTTPSession&)); + MOCK_METHOD2(onIngressError, void(const HTTPSession&, + ProxygenError)); + MOCK_METHOD2(onRead, void(const HTTPSession&, size_t)); + MOCK_METHOD2(onWrite, void(const HTTPSession&, size_t)); + MOCK_METHOD1(onRequestBegin, void(const HTTPSession&)); + MOCK_METHOD2(onRequestEnd, void(const HTTPSession&, uint32_t)); + MOCK_METHOD1(onActivateConnection, void(const HTTPSession&)); + MOCK_METHOD1(onDeactivateConnection, void(const HTTPSession&)); + MOCK_METHOD1(onDestroy, void(const HTTPSession&)); + MOCK_METHOD2(onIngressMessage, void(const HTTPSession&, + const HTTPMessage&)); + MOCK_METHOD1(onIngressLimitExceeded, void(const HTTPSession&)); + MOCK_METHOD1(onIngressPaused, void(const HTTPSession&)); + MOCK_METHOD1(onTransactionDetached, void(const HTTPSession&)); + MOCK_METHOD1(onPingReply, void(int64_t)); + MOCK_METHOD1(onSettingsOutgoingStreamsFull, void(const HTTPSession&)); + MOCK_METHOD1(onSettingsOutgoingStreamsNotFull, void(const HTTPSession&)); +}; + +} diff --git a/proxygen/lib/http/session/test/HTTPSessionTest.h b/proxygen/lib/http/session/test/HTTPSessionTest.h new file mode 100644 index 0000000000..5124e4a008 --- /dev/null +++ b/proxygen/lib/http/session/test/HTTPSessionTest.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/HTTP1xCodec.h" +#include "proxygen/lib/http/codec/SPDYCodec.h" +#include "proxygen/lib/http/codec/TransportDirection.h" +#include "proxygen/lib/http/codec/test/MockHTTPCodec.h" +#include "proxygen/lib/http/session/test/HTTPSessionMocks.h" + +#include + +template +typename std::enable_if< + std::is_enum::value, + std::unique_ptr >::type +makeClientCodec(Version version) { + return folly::make_unique( + proxygen::TransportDirection::UPSTREAM, version); +} + +template +typename std::enable_if< + std::is_same::value, + std::unique_ptr >::type +makeClientCodec(Version version) { + return folly::make_unique( + proxygen::TransportDirection::UPSTREAM); +} + +template +typename std::enable_if< + std::is_same::value, + std::unique_ptr >::type +makeClientCodec(Version version) { + return folly::make_unique(); +} + +template +typename std::enable_if< + std::is_enum::value, + std::unique_ptr >::type +makeServerCodec(Version version) { + return folly::make_unique( + proxygen::TransportDirection::DOWNSTREAM, + (Version)version); +} + +template +typename std::enable_if< + std::is_same::value, + std::unique_ptr >::type +makeServerCodec(Version version) { + return folly::make_unique( + proxygen::TransportDirection::DOWNSTREAM); +} + +template +typename std::enable_if< + std::is_same::value, + std::unique_ptr >::type +makeServerCodec(Version version) { + return folly::make_unique(); +} diff --git a/proxygen/lib/http/session/test/HTTPTransactionMocks.h b/proxygen/lib/http/session/test/HTTPTransactionMocks.h new file mode 100644 index 0000000000..96fb27835f --- /dev/null +++ b/proxygen/lib/http/session/test/HTTPTransactionMocks.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/codec/test/MockHTTPCodec.h" +#include "proxygen/lib/http/session/HTTPTransaction.h" + +#include + +namespace proxygen { + +class MockHTTPTransactionTransport: public HTTPTransaction::Transport { + public: + MockHTTPTransactionTransport() {} + GMOCK_METHOD1_(, noexcept,, pauseIngress, void(HTTPTransaction*)); + GMOCK_METHOD1_(, noexcept,, resumeIngress, void(HTTPTransaction*)); + GMOCK_METHOD1_(, noexcept,, transactionTimeout, void(HTTPTransaction*)); + GMOCK_METHOD3_(, noexcept,, sendHeaders, void(HTTPTransaction*, + const HTTPMessage&, + HTTPHeaderSize*)); + GMOCK_METHOD3_(, noexcept,, sendBody, + size_t(HTTPTransaction*, std::shared_ptr, bool)); + + size_t sendBody(HTTPTransaction* txn, std::unique_ptr iob, + bool eom) noexcept override { + return sendBody(txn, std::shared_ptr(iob.release()), eom); + } + + GMOCK_METHOD2_(, noexcept,, sendChunkHeader, size_t(HTTPTransaction*, + size_t)); + GMOCK_METHOD1_(, noexcept,, sendChunkTerminator, size_t(HTTPTransaction*)); + GMOCK_METHOD2_(, noexcept,, sendTrailers, size_t(HTTPTransaction*, + const HTTPHeaders&)); + GMOCK_METHOD1_(, noexcept,, sendEOM, size_t(HTTPTransaction*)); + GMOCK_METHOD2_(, noexcept,, sendAbort, size_t(HTTPTransaction*, + ErrorCode)); + GMOCK_METHOD0_(, noexcept,, notifyPendingEgress, void()); + GMOCK_METHOD1_(, noexcept,, detach, void(HTTPTransaction*)); + GMOCK_METHOD2_(, noexcept,, sendWindowUpdate, size_t(HTTPTransaction*, + uint32_t)); + GMOCK_METHOD1_(, noexcept,, notifyIngressBodyProcessed, void(uint32_t)); + GMOCK_METHOD0_(, noexcept,, getLocalAddressNonConst, + const folly::SocketAddress&()); + GMOCK_METHOD3_(, noexcept,, newPushedTransaction, + HTTPTransaction*(HTTPCodec::StreamID assocStreamId, + HTTPTransaction::PushHandler* handler, + int8_t)); + const folly::SocketAddress& getLocalAddress() + const noexcept override { + return const_cast(this) + ->getLocalAddressNonConst(); + } + GMOCK_METHOD0_(, noexcept,, getPeerAddressNonConst, + const folly::SocketAddress&()); + const folly::SocketAddress& getPeerAddress() + const noexcept override { + return const_cast(this) + ->getPeerAddressNonConst(); + } + MOCK_CONST_METHOD1(describe, void(std::ostream&)); + GMOCK_METHOD0_(, noexcept,, getSetupTransportInfoNonConst, + const TransportInfo&()); + const TransportInfo& getSetupTransportInfo() const noexcept override { + return const_cast(this) + ->getSetupTransportInfoNonConst(); + } + + MOCK_METHOD1(getCurrentTransportInfo, bool(TransportInfo*)); + GMOCK_METHOD0_(, noexcept,, getCodecNonConst, const HTTPCodec&()); + const HTTPCodec& getCodec() const noexcept override { + return const_cast(this) + ->getCodecNonConst(); + } + MOCK_CONST_METHOD0(isDraining, bool()); +}; + +class MockHTTPTransaction : public HTTPTransaction { + public: + MockHTTPTransaction(TransportDirection direction, + HTTPCodec::StreamID id, + uint32_t seqNo, + PriorityQueue& egressQueue, + apache::thrift::async::TAsyncTimeoutSet* timeouts, + HTTPSessionStats* stats = nullptr, + bool useFlowControl = false, + uint32_t receiveInitialWindowSize = 0, + uint32_t sendInitialWindowSize = 0, + int8_t priority = -1) : + HTTPTransaction(direction, id, seqNo, mockTransport_, egressQueue, + timeouts, stats, useFlowControl, + receiveInitialWindowSize, + sendInitialWindowSize, + priority), + defaultAddress_("127.0.0.1", 80) { + EXPECT_CALL(mockTransport_, getLocalAddressNonConst()) + .WillRepeatedly(testing::ReturnRef(defaultAddress_)); + EXPECT_CALL(mockTransport_, getPeerAddressNonConst()) + .WillRepeatedly(testing::ReturnRef(defaultAddress_)); + EXPECT_CALL(mockTransport_, getCodecNonConst()) + .WillRepeatedly(testing::ReturnRef(mockCodec_)); + EXPECT_CALL(mockTransport_, getSetupTransportInfoNonConst()) + .WillRepeatedly(testing::ReturnRef(setupTransportInfo_)); + } + + MOCK_CONST_METHOD0(extraResponseExpected, bool()); + + MOCK_METHOD1(sendHeaders, void(const HTTPMessage& headers)); + MOCK_METHOD1(sendBody, void(std::shared_ptr)); + void sendBody(std::unique_ptr iob) noexcept override { + sendBody(std::shared_ptr(iob.release())); + } + MOCK_METHOD1(sendChunkHeader, void(size_t)); + MOCK_METHOD0(sendChunkTerminator, void()); + MOCK_METHOD1(sendTrailers, void(const HTTPHeaders& trailers)); + MOCK_METHOD0(sendEOM, void()); + MOCK_METHOD0(sendAbort, void()); + MOCK_METHOD0(drop, void()); + MOCK_METHOD0(pauseIngress, void()); + MOCK_METHOD0(resumeIngress, void()); + MOCK_CONST_METHOD0(handlerEgressPaused, bool()); + MOCK_METHOD2(newPushedTransaction, + HTTPTransaction*(HTTPPushTransactionHandler*, uint8_t)); + MOCK_METHOD1(setReceiveWindow, void(uint32_t)); + MOCK_CONST_METHOD0(getReceiveWindow, const Window&()); + + void enablePush() { + EXPECT_CALL(mockCodec_, supportsPushTransactions()) + .WillRepeatedly(testing::Return(true)); + } + + testing::NiceMock mockTransport_; + const folly::SocketAddress defaultAddress_; + MockHTTPCodec mockCodec_; + TransportInfo setupTransportInfo_; +}; + +class MockHTTPTransactionTransportCallback: + public HTTPTransaction::TransportCallback { + public: + MockHTTPTransactionTransportCallback() {} + GMOCK_METHOD0_(, noexcept,, firstByteFlushed, void()); + GMOCK_METHOD1_(, noexcept,, headerBytesGenerated, void(HTTPHeaderSize&)); + GMOCK_METHOD1_(, noexcept,, bodyBytesGenerated, void(uint32_t)); +}; + +} diff --git a/proxygen/lib/http/session/test/HTTPTransactionSMTest.cpp b/proxygen/lib/http/session/test/HTTPTransactionSMTest.cpp new file mode 100644 index 0000000000..1ce446acad --- /dev/null +++ b/proxygen/lib/http/session/test/HTTPTransactionSMTest.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/HTTPTransactionEgressSM.h" +#include "proxygen/lib/http/session/HTTPTransactionIngressSM.h" + +#include + +using namespace proxygen; + +class EgressStateMachineFixture: public ::testing::Test { + public: + EgressStateMachineFixture(): instance_( + HTTPTransactionEgressSM::getNewInstance()) {} + + void follow(HTTPTransactionEgressSM::Event e) { + EXPECT_TRUE(HTTPTransactionEgressSM::transit(instance_, e)); + } + + void fail(HTTPTransactionEgressSM::Event e) { + EXPECT_FALSE(HTTPTransactionEgressSM::transit(instance_, e)); + } + private: + HTTPTransactionEgressSM::State instance_; +}; + +class IngressStateMachineFixture: public ::testing::Test { + public: + IngressStateMachineFixture(): + instance_(HTTPTransactionIngressSM::getNewInstance()) {} + + void follow(HTTPTransactionIngressSM::Event e) { + EXPECT_TRUE(HTTPTransactionIngressSM::transit(instance_, e)); + } + + void fail(HTTPTransactionIngressSM::Event e) { + EXPECT_FALSE(HTTPTransactionIngressSM::transit(instance_, e)); + } + private: + HTTPTransactionIngressSM::State instance_; +}; + +// Egress tests + +TEST_F(EgressStateMachineFixture, BadEgressTransitions1) { + follow(HTTPTransactionEgressSM::Event::sendHeaders); + follow(HTTPTransactionEgressSM::Event::sendChunkHeader); + follow(HTTPTransactionEgressSM::Event::sendBody); + fail(HTTPTransactionEgressSM::Event::sendEOM); +} + +TEST_F(EgressStateMachineFixture, BadEgressTransitions2) { + fail(HTTPTransactionEgressSM::Event::sendBody); +} + +TEST_F(EgressStateMachineFixture, BadEgressTransitions3) { + follow(HTTPTransactionEgressSM::Event::sendHeaders); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendEOM); + fail(HTTPTransactionEgressSM::Event::sendEOM); +} + +TEST_F(EgressStateMachineFixture, BadEgressTransitions4) { + follow(HTTPTransactionEgressSM::Event::sendHeaders); + follow(HTTPTransactionEgressSM::Event::sendChunkHeader); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendChunkTerminator); + follow(HTTPTransactionEgressSM::Event::sendChunkHeader); + fail(HTTPTransactionEgressSM::Event::sendChunkTerminator); +} + +TEST_F(EgressStateMachineFixture, EgressChunkedTransitions) { + follow(HTTPTransactionEgressSM::Event::sendHeaders); + + follow(HTTPTransactionEgressSM::Event::sendChunkHeader); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendChunkTerminator); + + follow(HTTPTransactionEgressSM::Event::sendChunkHeader); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendChunkTerminator); + + follow(HTTPTransactionEgressSM::Event::sendChunkHeader); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendChunkTerminator); + + follow(HTTPTransactionEgressSM::Event::sendTrailers); + follow(HTTPTransactionEgressSM::Event::sendEOM); +} + +TEST_F(EgressStateMachineFixture, NormalEgressTransitions) { + follow(HTTPTransactionEgressSM::Event::sendHeaders); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendBody); + follow(HTTPTransactionEgressSM::Event::sendEOM); +} + +// Ingress tests + +TEST_F(IngressStateMachineFixture, BadIngressTransitions1) { + follow(HTTPTransactionIngressSM::Event::onHeaders); + follow(HTTPTransactionIngressSM::Event::onChunkHeader); + follow(HTTPTransactionIngressSM::Event::onBody); + fail(HTTPTransactionIngressSM::Event::onEOM); +} + +TEST_F(IngressStateMachineFixture, BadIngressTransitions2) { + fail(HTTPTransactionIngressSM::Event::onBody); +} + +TEST_F(IngressStateMachineFixture, BadIngressTransitions3) { + follow(HTTPTransactionIngressSM::Event::onHeaders); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onEOM); + fail(HTTPTransactionIngressSM::Event::onEOM); +} + +TEST_F(IngressStateMachineFixture, BadIngressTransitions4) { + follow(HTTPTransactionIngressSM::Event::onHeaders); + follow(HTTPTransactionIngressSM::Event::onChunkHeader); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onChunkComplete); + follow(HTTPTransactionIngressSM::Event::onChunkHeader); + fail(HTTPTransactionIngressSM::Event::onChunkComplete); +} + +TEST_F(IngressStateMachineFixture, IngressChunkedTransitions) { + follow(HTTPTransactionIngressSM::Event::onHeaders); + + follow(HTTPTransactionIngressSM::Event::onChunkHeader); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onChunkComplete); + + follow(HTTPTransactionIngressSM::Event::onChunkHeader); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onChunkComplete); + + follow(HTTPTransactionIngressSM::Event::onChunkHeader); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onChunkComplete); + + follow(HTTPTransactionIngressSM::Event::onTrailers); + follow(HTTPTransactionIngressSM::Event::onEOM); +} + +TEST_F(IngressStateMachineFixture, NormalIngressTransitions) { + follow(HTTPTransactionIngressSM::Event::onHeaders); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onBody); + follow(HTTPTransactionIngressSM::Event::onEOM); +} + +TEST_F(IngressStateMachineFixture, WeirdIngressTransitions) { + follow(HTTPTransactionIngressSM::Event::onHeaders); + follow(HTTPTransactionIngressSM::Event::onTrailers); + follow(HTTPTransactionIngressSM::Event::onEOM); +} diff --git a/proxygen/lib/http/session/test/HTTPUpstreamSessionTest.cpp b/proxygen/lib/http/session/test/HTTPUpstreamSessionTest.cpp new file mode 100644 index 0000000000..14b477293f --- /dev/null +++ b/proxygen/lib/http/session/test/HTTPUpstreamSessionTest.cpp @@ -0,0 +1,1411 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/test/MockHTTPCodec.h" +#include "proxygen/lib/http/codec/test/TestUtils.h" +#include "proxygen/lib/http/session/HTTPUpstreamSession.h" +#include "proxygen/lib/http/session/test/HTTPSessionMocks.h" +#include "proxygen/lib/http/session/test/HTTPSessionTest.h" +#include "proxygen/lib/http/session/test/TestUtils.h" +#include "proxygen/lib/test/TestAsyncTransport.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace apache::thrift::async; +using namespace apache::thrift::test; +using namespace apache::thrift::transport; +using namespace folly; +using namespace proxygen; +using namespace testing; + +using std::string; +using std::unique_ptr; +using std::vector; + +struct HTTP1xCodecPair { + typedef HTTP1xCodec Codec; + static const int version = 1; +}; + +struct SPDY2CodecPair { + typedef SPDYCodec Codec; + static const SPDYVersion version = SPDYVersion::SPDY2; +}; + +struct SPDY3CodecPair { + typedef SPDYCodec Codec; + static const SPDYVersion version = SPDYVersion::SPDY3; +}; + +struct MockHTTPCodecPair { + typedef MockHTTPCodec Codec; + static const int version = 0; +}; + +template +class HTTPUpstreamTest: public testing::Test, + public HTTPSession::InfoCallback { + public: + HTTPUpstreamTest() + : eventBase_(), + transport_(new NiceMock()), + transactionTimeouts_( + new TAsyncTimeoutSet(&eventBase_, + TimeoutManager::InternalEnum::INTERNAL, + std::chrono::milliseconds(500))) { + } + + virtual void onWriteChain(TAsyncTransport::WriteCallback* callback, + std::shared_ptr iob, + WriteFlags flags) { + if (failWrites_) { + TTransportException ex; + callback->writeError(0, ex); + } else { + if (writeInLoop_) { + eventBase_.runInLoop([&] { + callback->writeSuccess(); + }); + } else { + callback->writeSuccess(); + } + } + } + + void SetUp() override { + commonSetUp(makeClientCodec(C::version)); + } + + void commonSetUp(unique_ptr codec) { + EXPECT_CALL(*transport_, writeChain(_, _, _)) + .WillRepeatedly(Invoke(this, &HTTPUpstreamTest::onWriteChain)); + EXPECT_CALL(*transport_, setReadCallback(_)) + .WillRepeatedly(SaveArg<0>(&readCallback_)); + EXPECT_CALL(*transport_, getReadCallback()) + .WillRepeatedly(Return(readCallback_)); + EXPECT_CALL(*transport_, getEventBase()) + .WillRepeatedly(Return(&eventBase_)); + EXPECT_CALL(*transport_, good()) + .WillRepeatedly(ReturnPointee(&transportGood_)); + EXPECT_CALL(*transport_, closeNow()) + .WillRepeatedly(Assign(&transportGood_, false)); + httpSession_ = new HTTPUpstreamSession( + transactionTimeouts_.get(), + std::move(TAsyncTransport::UniquePtr(transport_)), + localAddr_, peerAddr_, + std::move(codec), + mockTransportInfo_, this); + httpSession_->startNow(); + eventBase_.loop(); + ASSERT_EQ(this->sessionDestroyed_, false); + } + + unique_ptr makeServerCodec() { + return ::makeServerCodec(C::version); + } + + void readAndLoop(const char* input) { + readAndLoop((const uint8_t *)input, strlen(input)); + } + + void readAndLoop(const uint8_t* input, size_t length) { + CHECK_NOTNULL(readCallback_); + void* buf; + size_t bufSize; + while (length > 0) { + readCallback_->getReadBuffer(&buf, &bufSize); + // This is somewhat specific to our implementation, but currently we + // always return at least some space from getReadBuffer + CHECK_GT(bufSize, 0); + bufSize = std::min(bufSize, length); + memcpy(buf, input, bufSize); + readCallback_->readDataAvailable(bufSize); + eventBase_.loop(); + length -= bufSize; + input += bufSize; + } + } + + void testBasicRequest(); + void testBasicRequestHttp10(bool keepalive); + + // HTTPSession::InfoCallback interface + void onCreate(const HTTPSession& sess) { + sessionCreated_ = true; + } + void onIngressError(const HTTPSession& sess, ProxygenError err) { + } + void onRead(const HTTPSession& sess, size_t bytesRead) { + } + void onWrite(const HTTPSession& sess, size_t bytesWritten) { + } + void onRequestBegin(const HTTPSession& sess) { + } + void onRequestEnd(const HTTPSession& sess, uint32_t maxIngressQueueSize) { + } + void onActivateConnection(const HTTPSession& sess) { + } + void onDeactivateConnection(const HTTPSession& sess) { + } + void onDestroy(const HTTPSession& sess) { + sessionDestroyed_ = true; + } + void onIngressMessage(const HTTPSession& sess, + const HTTPMessage& msg) { + } + void onIngressLimitExceeded(const HTTPSession& sess) { + } + void onIngressPaused(const HTTPSession& sess) { + } + void onTransactionDetached(const HTTPSession& sess) { + } + void onPingReply(int64_t latency) { + } + void onSettingsOutgoingStreamsFull(const HTTPSession&) { + transactionsFull_ = true; + } + void onSettingsOutgoingStreamsNotFull(const HTTPSession&) { + transactionsFull_ = false; + } + protected: + bool sessionCreated_{false}; + bool sessionDestroyed_{false}; + + bool transactionsFull_{false}; + bool transportGood_{true}; + + EventBase eventBase_; + MockTAsyncTransport* transport_; // invalid once httpSession_ is destroyed + TAsyncTransport::ReadCallback* readCallback_{nullptr}; + TAsyncTimeoutSet::UniquePtr transactionTimeouts_; + TransportInfo mockTransportInfo_; + SocketAddress localAddr_{"127.0.0.1", 80}; + SocketAddress peerAddr_{"127.0.0.1", 12345}; + HTTPUpstreamSession* httpSession_{nullptr}; + bool failWrites_{false}; + bool writeInLoop_{false}; +}; +TYPED_TEST_CASE_P(HTTPUpstreamTest); + +template +class TimeoutableHTTPUpstreamTest: public HTTPUpstreamTest { + public: + TimeoutableHTTPUpstreamTest(): HTTPUpstreamTest() { + // make it non-internal for this test class + HTTPUpstreamTest::transactionTimeouts_.reset( + new TAsyncTimeoutSet(&this->HTTPUpstreamTest::eventBase_, + std::chrono::milliseconds(500))); + } +}; + +typedef HTTPUpstreamTest HTTPUpstreamSessionTest; +typedef HTTPUpstreamTest SPDY2UpstreamSessionTest; +typedef HTTPUpstreamTest SPDY3UpstreamSessionTest; + +TEST_F(SPDY3UpstreamSessionTest, server_push) { + SPDYCodec egressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + folly::IOBufQueue output(folly::IOBufQueue::cacheChainLength()); + + HTTPMessage push; + push.getHeaders().set("HOST", "www.foo.com"); + push.setURL("https://www.foo.com/"); + egressCodec.generateHeader(output, 2, push, 1, nullptr); + auto buf = makeBuf(100); + egressCodec.generateBody(output, 2, std::move(buf), true /* eom */); + + HTTPMessage resp; + resp.setStatusCode(200); + resp.setStatusMessage("Ohai"); + egressCodec.generateHeader(output, 1, resp, 0, nullptr); + buf = makeBuf(100); + egressCodec.generateBody(output, 1, std::move(buf), true /* eom */); + + std::unique_ptr input = output.move(); + input->coalesce(); + + MockHTTPHandler handler; + MockHTTPHandler pushHandler; + HTTPTransaction* txn; + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&txn)); + EXPECT_CALL(handler, onPushedTransaction(_)) + .WillOnce(Invoke([this, &pushHandler] (HTTPTransaction* pushTxn) { + pushTxn->setHandler(&pushHandler); + })); + EXPECT_CALL(pushHandler, setTransaction(_)); + EXPECT_CALL(pushHandler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_EQ(httpSession_->getNumIncomingStreams(), 1); + EXPECT_FALSE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(msg->getPath(), "/"); + EXPECT_EQ(msg->getHeaders().getSingleOrEmpty(HTTP_HEADER_HOST), + "www.foo.com"); + })); + EXPECT_CALL(pushHandler, onBody(_)); + EXPECT_CALL(pushHandler, onEOM()); + EXPECT_CALL(pushHandler, detachTransaction()); + + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onBody(_)); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + HTTPTransaction* txn2 = httpSession_->newTransaction(&handler); + CHECK_EQ(txn, txn2); + + HTTPMessage req = getGetRequest(); + + txn->sendHeaders(req); + txn->sendEOM(); + + readAndLoop(input->data(), input->length()); + + EXPECT_EQ(httpSession_->getNumIncomingStreams(), 0); + httpSession_->destroy(); +} + +TEST_F(SPDY3UpstreamSessionTest, ingress_goaway_abort_uncreated_streams) { + // Tests whether the session aborts the streams which are not created + // at the remote end. + MockHTTPHandler handler; + HTTPTransaction* txn; + + // Create SPDY buf for GOAWAY with last good stream as 0 (no streams created) + SPDYCodec egressCodec(TransportDirection::DOWNSTREAM, + SPDYVersion::SPDY3); + folly::IOBufQueue respBuf; + egressCodec.generateGoaway(respBuf, 0, ErrorCode::NO_ERROR); + std::unique_ptr goawayFrame = respBuf.move(); + goawayFrame->coalesce(); + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&txn)); + EXPECT_CALL(handler, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& err) { + EXPECT_TRUE(err.hasProxygenError()); + EXPECT_EQ(err.getProxygenError(), kErrorStreamUnacknowledged); + })); + EXPECT_CALL(handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([this] { + // Make sure the session can't create any more transactions. + MockHTTPHandler handler2; + EXPECT_EQ(httpSession_->newTransaction(&handler2), nullptr); + })); + + // Create new transaction + HTTPTransaction* txn2 = httpSession_->newTransaction(&handler); + CHECK_EQ(txn, txn2); + + // Send the GET request + HTTPMessage req = getGetRequest(); + txn->sendHeaders(req); + txn->sendEOM(); + + // Receive GOAWAY frame while waiting for SYN_REPLY + readAndLoop(goawayFrame->data(), goawayFrame->length()); + + // Session will delete itself after the abort +} + +TYPED_TEST_P(HTTPUpstreamTest, immediate_eof) { + // Receive an EOF without any request data + this->readCallback_->readEOF(); + this->eventBase_.loop(); + EXPECT_EQ(this->sessionDestroyed_, true); +} + +template +void HTTPUpstreamTest::testBasicRequest() { + MockHTTPHandler handler; + HTTPTransaction* txn; + HTTPMessage req = getGetRequest(); + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&txn)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_TRUE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + HTTPTransaction* txn2 = httpSession_->newTransaction(&handler); + CHECK_EQ(txn, txn2); + txn->sendHeaders(req); + txn->sendEOM(); + readAndLoop("HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "0\r\n\r\n"); + + CHECK(httpSession_->supportsMoreTransactions()); + CHECK_EQ(httpSession_->getNumOutgoingStreams(), 0); +} + +TEST_F(HTTPUpstreamSessionTest, basic_request) { + testBasicRequest(); + httpSession_->destroy(); +} + +TEST_F(HTTPUpstreamSessionTest, two_requests) { + testBasicRequest(); + testBasicRequest(); + httpSession_->destroy(); +} + +TEST_F(HTTPUpstreamSessionTest, 10_requests) { + for (uint16_t i = 0; i < 10; i++) { + testBasicRequest(); + } + httpSession_->destroy(); +} + +template +void HTTPUpstreamTest::testBasicRequestHttp10(bool keepalive) { + MockHTTPHandler handler; + HTTPTransaction* txn = nullptr; + HTTPMessage req = getGetRequest(); + req.setHTTPVersion(1, 0); + if (keepalive) { + req.getHeaders().set(HTTP_HEADER_CONNECTION, "Keep-Alive"); + } + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&txn)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_EQ(200, msg->getStatusCode()); + EXPECT_EQ(keepalive ? "keep-alive" : "close", + msg->getHeaders().getSingleOrEmpty(HTTP_HEADER_CONNECTION)); + })); + EXPECT_CALL(handler, onBody(_)); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + HTTPTransaction* txn2 = httpSession_->newTransaction(&handler); + CHECK_EQ(txn, txn2); + txn->sendHeaders(req); + txn->sendEOM(); + if (keepalive) { + readAndLoop("HTTP/1.0 200 OK\r\n" + "Connection: keep-alive\r\n" + "Content-length: 7\r\n\r\n" + "content"); + } else { + readAndLoop("HTTP/1.0 200 OK\r\n" + "Connection: close\r\n" + "Content-length: 7\r\n\r\n" + "content"); + } +} + +TEST_F(HTTPUpstreamSessionTest, http10_keepalive) { + testBasicRequestHttp10(true); + testBasicRequestHttp10(false); +} + +TEST_F(HTTPUpstreamSessionTest, basic_trailers) { + MockHTTPHandler handler; + HTTPTransaction* txn; + HTTPMessage req = getGetRequest(); + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&txn)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_TRUE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onTrailers(_)); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + HTTPTransaction* txn2 = httpSession_->newTransaction(&handler); + CHECK_EQ(txn, txn2); + txn->sendHeaders(req); + txn->sendEOM(); + readAndLoop("HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "0\r\n" + "X-Trailer1: foo\r\n" + "\r\n"); + + CHECK(httpSession_->supportsMoreTransactions()); + CHECK_EQ(httpSession_->getNumOutgoingStreams(), 0); + httpSession_->destroy(); +} + +TEST_F(HTTPUpstreamSessionTest, two_requests_with_pause) { + MockHTTPHandler handler; + HTTPTransaction* txn; + HTTPMessage req = getGetRequest(); + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&txn)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_TRUE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + + HTTPTransaction* txn2 = httpSession_->newTransaction(&handler); + CHECK_EQ(txn, txn2); + + EXPECT_CALL(handler, onEOM()) + .WillOnce(InvokeWithoutArgs([&] () { + txn->pauseIngress(); + })); + EXPECT_CALL(handler, detachTransaction()); + + txn->sendHeaders(req); + txn->sendEOM(); + readAndLoop("HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "0\r\n\r\n"); + + // Even though the previous transaction paused ingress just before it + // finished up, reads resume automatically when the number of + // transactions goes to zero. This way, the second request can read + // without having to call resumeIngress() + testBasicRequest(); + httpSession_->destroy(); +} + +typedef TimeoutableHTTPUpstreamTest HTTPUpstreamTimeoutTest; +TEST_F(HTTPUpstreamTimeoutTest, write_timeout_after_response) { + // Test where the upstream session times out while writing the request + // to the server, but after the server has already sent back a full + // response. + MockHTTPHandler handler; + HTTPMessage req = getPostRequest(); + + InSequence dummy; + EXPECT_CALL(handler, setTransaction(_)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_TRUE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(*transport_, writeChain(_, _, _)) + .WillRepeatedly(Return()); // ignore write -> write timeout + EXPECT_CALL(handler, onEgressPaused()); + EXPECT_CALL(handler, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& err) { + EXPECT_TRUE(err.hasProxygenError()); + ASSERT_EQ(err.getDirection(), + HTTPException::Direction::INGRESS_AND_EGRESS); + EXPECT_EQ(err.getProxygenError(), kErrorWriteTimeout); + })); + EXPECT_CALL(handler, detachTransaction()); + + HTTPTransaction* txn = httpSession_->newTransaction(&handler); + txn->sendHeaders(req); + // Don't send the body, but get a response immediately + readAndLoop("HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "0\r\n\r\n"); +} + +TEST_F(HTTPUpstreamSessionTest, 100_continue_keepalive) { + // Test a request with 100 continue on a keepalive connection. Then make + // another request. + MockHTTPHandler handler; + HTTPMessage req = getGetRequest(); + req.getHeaders().set(HTTP_HEADER_EXPECT, "100-continue"); + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(100, msg->getStatusCode()); + })) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_TRUE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + auto txn = httpSession_->newTransaction(&handler); + txn->sendHeaders(req); + txn->sendEOM(); + readAndLoop("HTTP/1.1 100 Continue\r\n\r\n" + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + "0\r\n\r\n"); + + // Now make sure everything still works + testBasicRequest(); + httpSession_->destroy(); +} + +TEST_F(HTTPUpstreamSessionTest, 417_keepalive) { + // Test a request with 100 continue on a keepalive connection. Then make + // another request after the expectation fails. + MockHTTPHandler handler; + HTTPMessage req = getGetRequest(); + req.getHeaders().set(HTTP_HEADER_EXPECT, "100-continue"); + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsChunked()); + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(417, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + auto txn = httpSession_->newTransaction(&handler); + txn->sendHeaders(req); + txn->sendEOM(); + readAndLoop("HTTP/1.1 417 Expectation Failed\r\n" + "Content-Length: 0\r\n\r\n"); + + // Now make sure everything still works + testBasicRequest(); + EXPECT_FALSE(sessionDestroyed_); + httpSession_->destroy(); +} + +TEST_F(HTTPUpstreamSessionTest, 101_upgrade) { + // Test an upgrade request with sending 101 response. Then send + // some data and check the onBody callback contents + MockHTTPHandler handler; + HTTPMessage req = getGetRequest(); + req.getHeaders().set(HTTP_HEADER_UPGRADE, "http/2.0"); + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsChunked()); + EXPECT_EQ(101, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onUpgrade(_)); + EXPECT_CALL(handler, onBody(_)) + .WillOnce(ExpectString("Test Body\r\n")); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + auto txn = httpSession_->newTransaction(&handler); + txn->sendHeaders(req); + txn->sendEOM(); + eventBase_.loop(); + readAndLoop("HTTP/1.1 101 Switching Protocols\r\n" + "Upgrade: http/2.0\r\n\r\n" + "Test Body\r\n"); + readCallback_->readEOF(); + eventBase_.loop(); + + CHECK_EQ(httpSession_->getNumOutgoingStreams(), 0); + httpSession_->destroy(); +} + +class NoFlushUpstreamSessionTest: public HTTPUpstreamTest { + public: + void onWriteChain(TAsyncTransport::WriteCallback* callback, + std::shared_ptr iob, + WriteFlags flags) override { + if (!timesCalled_++) { + callback->writeSuccess(); + } + // do nothing -- let unacked egress build up + } + private: + uint32_t timesCalled_{0}; +}; + +TEST_F(NoFlushUpstreamSessionTest, session_paused_start_paused) { + // If the session is paused, new upstream transactions should start + // paused too. + NiceMock handler1; + NiceMock handler2; + HTTPMessage req = getGetRequest(); + + InSequence dummy; + + EXPECT_CALL(handler1, setTransaction(_)); + auto txn1 = httpSession_->newTransaction(&handler1); + txn1->sendHeaders(req); + Mock::VerifyAndClearExpectations(&handler1); + // This happens when the body write fills the txn egress queue + EXPECT_CALL(handler1, onEgressPaused()); + // This gets called once the txn egress queue is moved to the session + EXPECT_CALL(handler1, onEgressResumed()); + // The session pauses all txns since no writeSuccess for too many bytes + EXPECT_CALL(handler1, onEgressPaused()); + // Send a body big enough to pause egress + txn1->sendBody(makeBuf(HTTPSession::getPendingWriteMax())); + eventBase_.loop(); + Mock::VerifyAndClearExpectations(&handler1); + + EXPECT_CALL(handler2, setTransaction(_)); + EXPECT_CALL(handler2, onEgressPaused()); + auto txn2 = httpSession_->newTransaction(&handler2); + eventBase_.loop(); + Mock::VerifyAndClearExpectations(&handler2); + + httpSession_->shutdownTransportWithReset(kErrorTimeout); +} + +TEST_F(NoFlushUpstreamSessionTest, delete_txn_on_unpause) { + // Test where the handler gets onEgressResumed() and aborts itself and + // creates another transaction on the SPDY session inside that + // callback. This used to invalidate the transactions iterator inside + // HTTPSession and cause a crash + + NiceMock handler1; + NiceMock handler2; + NiceMock handler3; + HTTPMessage req = getGetRequest(); + + InSequence dummy; + + EXPECT_CALL(handler1, setTransaction(_)); + EXPECT_CALL(handler2, setTransaction(_)); + EXPECT_CALL(handler3, setTransaction(_)); + EXPECT_CALL(handler2, onEgressPaused()); + EXPECT_CALL(handler2, onEgressResumed()); + EXPECT_CALL(handler2, onEgressPaused()) + .WillOnce(InvokeWithoutArgs([this] { + // This time it is invoked by the session on all transactions + httpSession_->shutdownTransportWithReset(kErrorTimeout); + })); + auto txn1 = httpSession_->newTransaction(&handler1); + auto txn2 = httpSession_->newTransaction(&handler2); + auto txn3 = httpSession_->newTransaction(&handler3); + txn2->sendHeaders(req); + // This happens when the body write fills the txn egress queue + // Send a body big enough to pause egress + txn2->sendBody(makeBuf(HTTPSession::getPendingWriteMax())); + eventBase_.loop(); +} + +class MockHTTPUpstreamTest: public HTTPUpstreamTest { + public: + void SetUp() override { + auto codec = folly::make_unique>(); + codecPtr_ = codec.get(); + EXPECT_CALL(*codec, supportsParallelRequests()) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*codec, getTransportDirection()) + .WillRepeatedly(Return(TransportDirection::UPSTREAM)); + EXPECT_CALL(*codec, setCallback(_)) + .WillRepeatedly(SaveArg<0>(&codecCb_)); + EXPECT_CALL(*codec, isReusable()) + .WillRepeatedly(ReturnPointee(&reusable_)); + EXPECT_CALL(*codec, generateGoaway(_, _, _)) + .WillRepeatedly(InvokeWithoutArgs([&] { + reusable_ = false; + return 1; + })); + EXPECT_CALL(*codec, createStream()) + .WillRepeatedly(Invoke([&] { + auto ret = nextOutgoingTxn_; + nextOutgoingTxn_ += 2; + return ret; + })); + commonSetUp(std::move(codec)); + } + + void TearDown() override { + EXPECT_TRUE(sessionDestroyed_); + } + + std::unique_ptr> openTransaction() { + // Returns a mock handler with txn_ field set in it + auto handler = folly::make_unique>(); + EXPECT_CALL(*handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler->txn_)); + auto txn = httpSession_->newTransaction(handler.get()); + EXPECT_EQ(txn, handler->txn_); + return std::move(handler); + } + + MockHTTPCodec* codecPtr_{nullptr}; + HTTPCodec::Callback* codecCb_{nullptr}; + bool reusable_{true}; + uint32_t nextOutgoingTxn_{1}; +}; + +TEST_F(MockHTTPUpstreamTest, parse_error_no_txn) { + // 1) Create streamID == 1 + // 2) Send request + // 3) Detach handler + // 4) Get an ingress parse error on reply + // Expect that the codec should be asked to generate an abort on streamID==1 + + // Setup the codec expectations. + EXPECT_CALL(*codecPtr_, generateEOM(_, _)) + .WillOnce(Return(20)); + EXPECT_CALL(*codecPtr_, generateRstStream(_, 1, _)) + .WillOnce(Return(1)); + + // 1) + NiceMock handler; + auto txn = httpSession_->newTransaction(&handler); + + // 2) + auto req = getPostRequest(); + txn->sendHeaders(req); + txn->sendEOM(); + + // 3) Note this sendAbort() doesn't destroy the txn since byte events are + // enqueued + txn->sendAbort(); + + // 4) + HTTPException ex(HTTPException::Direction::INGRESS_AND_EGRESS, "foo"); + ex.setProxygenError(kErrorParseHeader); + ex.setCodecStatusCode(ErrorCode::REFUSED_STREAM); + codecCb_->onError(1, ex, true); + + // cleanup + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); + eventBase_.loop(); +} + +TEST_F(MockHTTPUpstreamTest, 0_max_outgoing_txns) { + // Test where an upstream session gets a SETTINGS frame with 0 max + // outgoing transactions. In our implementation, we jsut send a GOAWAY + // and close the connection. + + codecCb_->onSettings({{SettingsId::MAX_CONCURRENT_STREAMS, 0}}); + EXPECT_TRUE(transactionsFull_); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockHTTPUpstreamTest, outgoing_txn_settings) { + // Create 2 transactions, then receive a settings frame from + // the server indicating 1 parallel transaction at a time is allowed. + // Then get another SETTINGS frame indicating 100 max parallel + // transactions. Expect that HTTPSession invokes both info callbacks. + + NiceMock handler1; + NiceMock handler2; + httpSession_->newTransaction(&handler1); + httpSession_->newTransaction(&handler2); + + codecCb_->onSettings({{SettingsId::MAX_CONCURRENT_STREAMS, 1}}); + EXPECT_TRUE(transactionsFull_); + codecCb_->onSettings({{SettingsId::MAX_CONCURRENT_STREAMS, 100}}); + EXPECT_FALSE(transactionsFull_); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockHTTPUpstreamTest, ingress_goaway_drain) { + // Tests whether the session drains existing transactions and + // deletes itself after receiving a GOAWAY. + MockHTTPHandler handler; + HTTPTransaction* txn; + + InSequence dummy; + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&txn)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + // Create new transaction + HTTPTransaction* txn2 = httpSession_->newTransaction(&handler); + CHECK_EQ(txn, txn2); + + // Send the GET request + HTTPMessage req = getGetRequest(); + txn->sendHeaders(req); + txn->sendEOM(); + + // Receive GOAWAY frame with last good stream as 1 + codecCb_->onGoaway(1, ErrorCode::NO_ERROR); + + // New transactions cannot be created afrer goaway + EXPECT_FALSE(httpSession_->isReusable()); + EXPECT_EQ(httpSession_->newTransaction(&handler), nullptr); + + // Receive 200 OK + auto resp = makeResponse(200); + codecCb_->onMessageBegin(1, resp.get()); + codecCb_->onHeadersComplete(1, std::move(resp)); + codecCb_->onMessageComplete(1, false); + eventBase_.loop(); + + // Session will delete itself after getting the response +} + +TEST_F(MockHTTPUpstreamTest, goaway) { + // Make sure existing txns complete successfully even if we drain the + // upstream session + const unsigned numTxns = 10; + MockHTTPHandler handler[numTxns]; + + InSequence dummy; + + for (unsigned i = 0; i < numTxns; ++i) { + EXPECT_CALL(handler[i], setTransaction(_)) + .WillOnce(SaveArg<0>(&handler[i].txn_)); + EXPECT_CALL(handler[i], onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + httpSession_->newTransaction(&handler[i]); + + // Send the GET request + HTTPMessage req = getGetRequest(); + handler[i].txn_->sendHeaders(req); + handler[i].txn_->sendEOM(); + + // Receive 200 OK + auto resp = makeResponse(200); + codecCb_->onMessageBegin(handler[i].txn_->getID(), resp.get()); + codecCb_->onHeadersComplete(handler[i].txn_->getID(), std::move(resp)); + } + + codecCb_->onGoaway(numTxns * 2 + 1, ErrorCode::NO_ERROR); + for (unsigned i = 0; i < numTxns; ++i) { + EXPECT_CALL(handler[i], onEOM()); + EXPECT_CALL(handler[i], detachTransaction()); + codecCb_->onMessageComplete(i*2 + 1, false); + } + eventBase_.loop(); + + // Session will delete itself after drain completes +} + +TEST_F(MockHTTPUpstreamTest, no_window_update_on_drain) { + MockHTTPHandler handler; + HTTPTransaction* txn; + + EXPECT_CALL(*codecPtr_, supportsStreamFlowControl()) + .WillRepeatedly(Return(true)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&txn)); + + HTTPTransaction* txn2 = httpSession_->newTransaction(&handler); + + HTTPMessage req = getGetRequest(); + + txn->sendHeaders(req); + txn->sendEOM(); + httpSession_->drain(); + auto streamID = txn->getID(); + + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(handler, onBody(_)) + .Times(3); + EXPECT_CALL(handler, onEOM()); + + EXPECT_CALL(handler, detachTransaction()); + + CHECK_EQ(txn, txn2); + + uint32_t outstanding = 0; + uint32_t sendWindow = 65536; + uint32_t toSend = sendWindow * 1.55; + + // We'll get exactly one window update because we are draining + EXPECT_CALL(*codecPtr_, generateWindowUpdate(_, _, _)) + .WillOnce(Invoke([&] + (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + uint32_t delta) { + EXPECT_EQ(delta, sendWindow); + outstanding -= delta; + uint32_t len = std::min(toSend, + sendWindow - outstanding); + EXPECT_LT(len, sendWindow); + toSend -= len; + EXPECT_EQ(toSend, 0); + eventBase_.runAfterDelay([this, streamID, len] { + failWrites_ = true; + auto respBody = makeBuf(len); + codecCb_->onBody(streamID, std::move(respBody)); + codecCb_->onMessageComplete(streamID, false); + }, 50); + + const std::string dummy("window"); + writeBuf.append(dummy); + return 6; + })); + + codecCb_->onGoaway(streamID, ErrorCode::NO_ERROR); + auto resp = makeResponse(200); + codecCb_->onMessageBegin(streamID, resp.get()); + codecCb_->onHeadersComplete(streamID, std::move(resp)); + + // While there is room and the window and body to send + while (sendWindow - outstanding > 0 && toSend > 0) { + // Send up to a 36k chunk + uint32_t len = std::min(toSend, uint32_t(36000)); + // limited by the available window + len = std::min(len, sendWindow - outstanding); + auto respBody = makeBuf(len); + toSend -= len; + outstanding += len; + codecCb_->onBody(streamID, std::move(respBody)); + } + + eventBase_.loop(); +} + +TEST_F(MockHTTPUpstreamTest, get_with_body) { + // Should be allowed to send a GET request with body. + NiceMock handler; + HTTPMessage req = getGetRequest(); + req.getHeaders().set(HTTP_HEADER_CONTENT_LENGTH, "10"); + + InSequence dummy; + + EXPECT_CALL(*codecPtr_, generateHeader(_, _, _, _, _)); + EXPECT_CALL(*codecPtr_, generateBody(_, _, _, true)); + + auto txn = httpSession_->newTransaction(&handler); + txn->sendHeaders(req); + txn->sendBody(makeBuf(10)); + txn->sendEOM(); + + eventBase_.loop(); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +template +class TestAbortPost : public MockHTTPUpstreamTest { + public: + void doAbortTest() { + // Send an abort at various points while receiving the response to a GET + // The test is broken into "stages" + // Stage 0) headers received + // Stage 1) chunk header received + // Stage 2) body received + // Stage 3) chunk complete received + // Stage 4) trailers received + // Stage 5) eom received + // This test makes sure expected callbacks are received if an abort is + // sent before any of these stages. + InSequence enforceOrder; + StrictMock handler; + HTTPMessage req = getPostRequest(); + req.getHeaders().set(HTTP_HEADER_CONTENT_LENGTH, "10"); + + std::unique_ptr resp; + std::unique_ptr respBody; + std::tie(resp, respBody) = makeResponse(200, 50); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + EXPECT_CALL(*codecPtr_, generateHeader(_, _, _, _, _)); + + if (stage > 0) { + EXPECT_CALL(handler, onHeadersComplete(_)); + } + if (stage > 1) { + EXPECT_CALL(handler, onChunkHeader(_)); + } + if (stage > 2) { + EXPECT_CALL(handler, onBody(_)); + } + if (stage > 3) { + EXPECT_CALL(handler, onChunkComplete()); + } + if (stage > 4) { + EXPECT_CALL(handler, onTrailers(_)); + } + if (stage > 5) { + EXPECT_CALL(handler, onEOM()); + } + + auto txn = httpSession_->newTransaction(&handler); + auto streamID = txn->getID(); + txn->sendHeaders(req); + txn->sendBody(makeBuf(5)); // only send half the body + + auto doAbort = [&] { + EXPECT_CALL(*codecPtr_, generateRstStream(_, txn->getID(), _)); + EXPECT_CALL(handler, detachTransaction()); + const auto id = txn->getID(); + txn->sendAbort(); + EXPECT_CALL(*codecPtr_, + generateRstStream(_, id, ErrorCode::_SPDY_INVALID_STREAM)) + .Times(AtLeast(0)); + }; + + if (stage == 0) { + doAbort(); + } + codecCb_->onHeadersComplete(streamID, std::move(resp)); + if (stage == 1) { + doAbort(); + } + codecCb_->onChunkHeader(streamID, respBody->computeChainDataLength()); + if (stage == 2) { + doAbort(); + } + codecCb_->onBody(streamID, std::move(respBody)); + if (stage == 3) { + doAbort(); + } + codecCb_->onChunkComplete(streamID); + if (stage == 4) { + doAbort(); + } + codecCb_->onTrailersComplete(streamID, + folly::make_unique()); + if (stage == 5) { + doAbort(); + } + codecCb_->onMessageComplete(streamID, false); + + eventBase_.loop(); + } +}; + +typedef TestAbortPost<0> TestAbortPost0; +typedef TestAbortPost<1> TestAbortPost1; +typedef TestAbortPost<2> TestAbortPost2; +typedef TestAbortPost<3> TestAbortPost3; +typedef TestAbortPost<4> TestAbortPost4; +typedef TestAbortPost<5> TestAbortPost5; + +TEST_F(TestAbortPost1, test) { doAbortTest(); } +TEST_F(TestAbortPost2, test) { doAbortTest(); } +TEST_F(TestAbortPost3, test) { doAbortTest(); } +TEST_F(TestAbortPost4, test) { doAbortTest(); } +TEST_F(TestAbortPost5, test) { doAbortTest(); } + +TEST_F(MockHTTPUpstreamTest, abort_upgrade) { + // This is basically the same test as above, just for the upgrade path + InSequence enforceOrder; + StrictMock handler; + HTTPMessage req = getPostRequest(); + req.getHeaders().set(HTTP_HEADER_CONTENT_LENGTH, "10"); + + std::unique_ptr resp = makeResponse(200); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + EXPECT_CALL(*codecPtr_, generateHeader(_, _, _, _, _)); + + auto txn = httpSession_->newTransaction(&handler); + const auto streamID = txn->getID(); + txn->sendHeaders(req); + txn->sendBody(makeBuf(5)); // only send half the body + + EXPECT_CALL(handler, onHeadersComplete(_)); + codecCb_->onHeadersComplete(streamID, std::move(resp)); + + EXPECT_CALL(*codecPtr_, generateRstStream(_, streamID, _)); + EXPECT_CALL(handler, detachTransaction()); + EXPECT_CALL(*codecPtr_, generateRstStream(_, streamID, + ErrorCode::_SPDY_INVALID_STREAM)); + txn->sendAbort(); + codecCb_->onMessageComplete(streamID, true); // upgrade + EXPECT_CALL(*codecPtr_, generateRstStream(_, streamID, + ErrorCode::_SPDY_INVALID_STREAM)); + codecCb_->onMessageComplete(streamID, false); // eom + + eventBase_.loop(); +} + +TEST_F(MockHTTPUpstreamTest, drain_before_send_headers) { + // Test that drain on session before sendHeaders() is called on open txn + + InSequence enforceOrder; + NiceMock handler; + MockHTTPHandler pushHandler; + auto req = makeGetRequest(); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + EXPECT_CALL(*codecPtr_, generateHeader(_, _, _, _, _)); + + EXPECT_CALL(handler, onHeadersComplete(_)); + EXPECT_CALL(handler, detachTransaction()); + + auto txn = httpSession_->newTransaction(&handler); + httpSession_->drain(); + txn->sendHeaders(*req); + txn->sendEOM(); + codecCb_->onHeadersComplete(txn->getID(), makeResponse(200)); + codecCb_->onMessageComplete(txn->getID(), false); // eom + + eventBase_.loop(); +} + +TEST_F(MockHTTPUpstreamTest, receive_double_goaway) { + // Test that we handle receiving two goaways correctly + + InSequence enforceOrder; + auto req = getGetRequest(); + + // Open 2 txns but doesn't send headers yet + auto handler1 = openTransaction(); + auto handler2 = openTransaction(); + + // Get first goaway acking many un-started txns + codecCb_->onGoaway(101, ErrorCode::NO_ERROR); + + // This txn should be alive since it was ack'd by the above goaway + handler1->txn_->sendHeaders(req); + + // Second goaway acks the only the current outstanding transaction + EXPECT_CALL(*handler2, onError(_)); + EXPECT_CALL(*handler2, detachTransaction()); + codecCb_->onGoaway(handler1->txn_->getID(), ErrorCode::NO_ERROR); + + // Clean up + httpSession_->drain(); + EXPECT_CALL(*codecPtr_, generateRstStream(_, handler1->txn_->getID(), _)); + EXPECT_CALL(*handler1, detachTransaction()); + handler1->txn_->sendAbort(); +} + +TEST_F(MockHTTPUpstreamTest, server_push_invalid_assoc) { + // Test that protocol error is generated on server push + // with invalid assoc stream id + InSequence enforceOrder; + auto req = getGetRequest(); + auto handler = openTransaction(); + + int streamID = handler->txn_->getID(); + int pushID = streamID + 1; + int badAssocID = streamID + 2; + + EXPECT_CALL(*codecPtr_, + generateRstStream(_, pushID, ErrorCode::PROTOCOL_ERROR)) + .WillOnce(Return(1)); + EXPECT_CALL(*codecPtr_, + generateRstStream(_, pushID, ErrorCode::_SPDY_INVALID_STREAM)) + .Times(2); + + auto resp = makeResponse(200); + codecCb_->onPushMessageBegin(pushID, badAssocID, resp.get()); + codecCb_->onHeadersComplete(pushID, std::move(resp)); + codecCb_->onMessageComplete(pushID, false); + + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(*handler, onEOM()); + + resp = makeResponse(200); + codecCb_->onMessageBegin(streamID, resp.get()); + codecCb_->onHeadersComplete(streamID, std::move(resp)); + codecCb_->onMessageComplete(streamID, false); + + // Cleanup + EXPECT_CALL(*codecPtr_, generateRstStream(_, streamID, _)) + .Times(1); + EXPECT_CALL(*handler, detachTransaction()); + handler->terminate(); +} + +TEST_F(MockHTTPUpstreamTest, server_push_after_fin) { + // Test that protocol error is generated on server push + // after FIN is received on regular response on the stream + InSequence enforceOrder; + auto req = getGetRequest(); + auto handler = openTransaction(); + + int streamID = handler->txn_->getID(); + int pushID = streamID + 1; + + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(*handler, onEOM()); + + auto resp = makeResponse(200); + codecCb_->onMessageBegin(streamID, resp.get()); + codecCb_->onHeadersComplete(streamID, std::move(resp)); + codecCb_->onMessageComplete(streamID, false); + + EXPECT_CALL(*codecPtr_, + generateRstStream(_, pushID, ErrorCode::PROTOCOL_ERROR)) + .WillOnce(InvokeWithoutArgs([this] { + // Verify that the assoc txn is still present + EXPECT_TRUE(httpSession_->hasActiveTransactions()); + return 1; + })); + EXPECT_CALL(*codecPtr_, + generateRstStream(_, pushID, ErrorCode::_SPDY_INVALID_STREAM)) + .Times(2); + + resp = makeResponse(200); + codecCb_->onPushMessageBegin(pushID, streamID, resp.get()); + codecCb_->onHeadersComplete(pushID, std::move(resp)); + codecCb_->onMessageComplete(pushID, false); + + // Cleanup + EXPECT_CALL(*codecPtr_, generateRstStream(_, streamID, _)) + .Times(1); + EXPECT_CALL(*handler, detachTransaction()); + handler->terminate(); +} + +TEST_F(MockHTTPUpstreamTest, server_push_handler_install_fail) { + // Test that REFUSED_STREAM error is generated when the session + // fails to install the server push handler + InSequence enforceOrder; + auto req = getGetRequest(); + auto handler = openTransaction(); + + int streamID = handler->txn_->getID(); + int pushID = streamID + 1; + + EXPECT_CALL(*handler, onPushedTransaction(_)) + .WillOnce(Invoke([] (HTTPTransaction* txn) { + // Intentionally unset the handler on the upstream push txn + txn->setHandler(nullptr); + })); + EXPECT_CALL(*codecPtr_, + generateRstStream(_, pushID, ErrorCode::REFUSED_STREAM)) + .WillOnce(Return(1)); + EXPECT_CALL(*codecPtr_, + generateRstStream(_, pushID, ErrorCode::_SPDY_INVALID_STREAM)) + .Times(2); + + auto resp = folly::make_unique(); + resp->setStatusCode(200); + resp->setStatusMessage("OK"); + codecCb_->onPushMessageBegin(pushID, streamID, resp.get()); + codecCb_->onHeadersComplete(pushID, std::move(resp)); + codecCb_->onMessageComplete(pushID, false); + + EXPECT_CALL(*handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + EXPECT_FALSE(msg->getIsUpgraded()); + EXPECT_EQ(200, msg->getStatusCode()); + })); + EXPECT_CALL(*handler, onEOM()); + + resp = folly::make_unique(); + resp->setStatusCode(200); + resp->setStatusMessage("OK"); + codecCb_->onMessageBegin(streamID, resp.get()); + codecCb_->onHeadersComplete(streamID, std::move(resp)); + codecCb_->onMessageComplete(streamID, false); + + // Cleanup + EXPECT_CALL(*codecPtr_, generateRstStream(_, streamID, _)) + .Times(1); + EXPECT_CALL(*handler, detachTransaction()); + handler->terminate(); +} + +TEST_F(MockHTTPUpstreamTest, server_push_unhandled_assoc) { + // Test that REFUSED_STREAM error is generated when the assoc txn + // is unhandled + InSequence enforceOrder; + auto req = getGetRequest(); + auto handler = openTransaction(); + + int streamID = handler->txn_->getID(); + int pushID = streamID + 1; + + // Forcefully unset the handler on the assoc txn + handler->txn_->setHandler(nullptr); + + EXPECT_CALL(*codecPtr_, + generateRstStream(_, pushID, ErrorCode::REFUSED_STREAM)) + .WillOnce(Return(1)); + EXPECT_CALL(*codecPtr_, + generateRstStream(_, pushID, ErrorCode::_SPDY_INVALID_STREAM)) + .Times(2); + + auto resp = folly::make_unique(); + resp->setStatusCode(200); + resp->setStatusMessage("OK"); + codecCb_->onPushMessageBegin(pushID, streamID, resp.get()); + codecCb_->onHeadersComplete(pushID, std::move(resp)); + codecCb_->onMessageComplete(pushID, false); + + // Cleanup + EXPECT_CALL(*codecPtr_, generateRstStream(_, streamID, _)) + .Times(1); + handler->terminate(); +} + +TEST_F(MockHTTPUpstreamTest, headers_then_body_then_headers) { + HTTPMessage req = getGetRequest(); + auto handler = openTransaction(); + handler->txn_->sendHeaders(req); + + // Now receive 2 replies on the same stream (illegal!) + EXPECT_CALL(*handler, onHeadersComplete(_)); + EXPECT_CALL(*handler, onBody(_)); + // After getting the second headers, transaction will detach the handler + EXPECT_CALL(*handler, onError(_)); + EXPECT_CALL(*handler, detachTransaction()); + auto resp = makeResponse(200); + codecCb_->onMessageBegin(1, resp.get()); + codecCb_->onHeadersComplete(1, std::move(resp)); + codecCb_->onBody(1, makeBuf(20)); + codecCb_->onHeadersComplete(1, makeResponse(200)); +} + +// Register and instantiate all our type-paramterized tests +REGISTER_TYPED_TEST_CASE_P(HTTPUpstreamTest, + immediate_eof/*[, other_test]*/); + +typedef ::testing::Types AllTypes; +INSTANTIATE_TYPED_TEST_CASE_P(AllTypesPrefix, HTTPUpstreamTest, AllTypes); diff --git a/proxygen/lib/http/session/test/Makefile.am b/proxygen/lib/http/session/test/Makefile.am new file mode 100644 index 0000000000..1b76f6ac23 --- /dev/null +++ b/proxygen/lib/http/session/test/Makefile.am @@ -0,0 +1,21 @@ +SUBDIRS = . + +check_PROGRAMS = SessionTests +SessionTests_SOURCES = \ + HTTPTransactionSMTest.cpp \ + DownstreamTransactionTest.cpp \ + HTTPDownstreamSessionTest.cpp \ + HTTPSessionAcceptorTest.cpp \ + HTTPUpstreamSessionTest.cpp \ + MockCodecDownstreamTest.cpp \ + TestUtils.cpp + +SessionTests_LDADD = \ + ../../../services/libproxygenservices.la \ + ../../../test/libtestmain.la \ + ../../../test/libtesttransport.la \ + ../../../utils/libutils.la \ + ../../codec/test/libcodectestutils.la \ + ../../libproxygenhttp.la + +TESTS = SessionTests diff --git a/proxygen/lib/http/session/test/MockCodecDownstreamTest.cpp b/proxygen/lib/http/session/test/MockCodecDownstreamTest.cpp new file mode 100644 index 0000000000..7b77ae424d --- /dev/null +++ b/proxygen/lib/http/session/test/MockCodecDownstreamTest.cpp @@ -0,0 +1,1368 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/codec/test/MockHTTPCodec.h" +#include "proxygen/lib/http/codec/test/TestUtils.h" +#include "proxygen/lib/http/session/HTTPDirectResponseHandler.h" +#include "proxygen/lib/http/session/HTTPDownstreamSession.h" +#include "proxygen/lib/http/session/HTTPSession.h" +#include "proxygen/lib/http/session/test/HTTPSessionMocks.h" +#include "proxygen/lib/http/session/test/HTTPSessionTest.h" +#include "proxygen/lib/http/session/test/TestUtils.h" +#include "proxygen/lib/test/TestAsyncTransport.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace apache::thrift::async; +using namespace apache::thrift::test; +using namespace apache::thrift::transport; +using namespace folly::wangle; +using namespace folly; +using namespace proxygen; +using namespace std; +using namespace testing; + +const HTTPSettings kDefaultIngressSettings{ + { SettingsId::INITIAL_WINDOW_SIZE, 65536 } +}; + +class MockCodecDownstreamTest: public testing::Test { + public: + MockCodecDownstreamTest() + : eventBase_(), + codec_(new StrictMock()), + transport_(new NiceMock()), + transactionTimeouts_(makeInternalTimeoutSet(&eventBase_)) { + + EXPECT_CALL(*transport_, good()) + .WillRepeatedly(ReturnPointee(&transportGood_)); + EXPECT_CALL(*transport_, closeNow()) + .WillRepeatedly(Assign(&transportGood_, false)); + EXPECT_CALL(*transport_, getEventBase()) + .WillRepeatedly(Return(&eventBase_)); + EXPECT_CALL(*transport_, setReadCallback(_)) + .WillRepeatedly(SaveArg<0>(&transportCb_)); + EXPECT_CALL(mockController_, attachSession(_)); + EXPECT_CALL(*codec_, setCallback(_)) + .WillRepeatedly(SaveArg<0>(&codecCallback_)); + EXPECT_CALL(*codec_, supportsParallelRequests()) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*codec_, supportsPushTransactions()) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*codec_, getTransportDirection()) + .WillRepeatedly(Return(TransportDirection::DOWNSTREAM)); + EXPECT_CALL(*codec_, getEgressSettings()); + EXPECT_CALL(*codec_, supportsStreamFlowControl()) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*codec_, setParserPaused(_)) + .WillRepeatedly(Return()); + EXPECT_CALL(*codec_, supportsSessionFlowControl()) + .WillRepeatedly(Return(true)); // simulate spdy 3.1 + EXPECT_CALL(*codec_, getIngressSettings()) + .WillRepeatedly(Return(&kDefaultIngressSettings)); + EXPECT_CALL(*codec_, isReusable()) + .WillRepeatedly(ReturnPointee(&reusable_)); + EXPECT_CALL(*codec_, isWaitingToDrain()) + .WillRepeatedly(ReturnPointee(&drainPending_)); + EXPECT_CALL(*codec_, generateSettings(_)); + EXPECT_CALL(*codec_, createStream()) + .WillRepeatedly(InvokeWithoutArgs([&] { + return pushStreamID_ += 2; + })); + EXPECT_CALL(*codec_, enableDoubleGoawayDrain()) + .WillRepeatedly(Invoke([&] { doubleGoaway_ = true; })); + EXPECT_CALL(*codec_, generateGoaway(_, _, _)) + .WillRepeatedly(Invoke([this] (IOBufQueue& writeBuf, + HTTPCodec::StreamID lastStream, + ErrorCode code) { + if (reusable_) { + reusable_ = false; + drainPending_ = doubleGoaway_; + } else if (!drainPending_) { + return 0; + } else { + drainPending_ = false; + } + if (liveGoaways_) { + writeBuf.append(string("x")); + } + return 1; + })); + EXPECT_CALL(*codec_, generateRstStream(_, _, _)) + .WillRepeatedly(Return(1)); + + httpSession_ = new HTTPDownstreamSession( + transactionTimeouts_.get(), + std::move(TAsyncTransport::UniquePtr(transport_)), + localAddr, peerAddr, + &mockController_, + std::unique_ptr(codec_), + mockTransportInfo); + httpSession_->startNow(); + eventBase_.loop(); + } + + // Pass a function to execute inside Codec::onIngress(). This function also + // takes care of passing an empty ingress buffer to the codec. + template + void onIngressImpl(T f) { + EXPECT_CALL(*codec_, onIngress(_)) + .WillOnce(Invoke(f)); + + void* buf; + size_t bufSize; + transportCb_->getReadBuffer(&buf, &bufSize); + transportCb_->readDataAvailable(bufSize); + } + + void testGoaway(bool doubleGoaway, bool dropConnection); + + protected: + + EventBase eventBase_; + // invalid once httpSession_ is destroyed + StrictMock* codec_; + HTTPCodec::Callback* codecCallback_{nullptr}; + NiceMock* transport_; + TAsyncTransport::ReadCallback* transportCb_; + TAsyncTimeoutSet::UniquePtr transactionTimeouts_; + StrictMock mockController_; + HTTPDownstreamSession* httpSession_; + HTTPCodec::StreamID pushStreamID_{0}; + bool reusable_{true}; + bool transportGood_{true}; + bool drainPending_{false}; + bool doubleGoaway_{false}; + bool liveGoaways_{false}; +}; + +TEST_F(MockCodecDownstreamTest, on_abort_then_timeouts) { + // Test what happens when txn1 (out of many transactions) gets an abort + // followed by a transaction timeout followed by a write timeout + MockHTTPHandler handler1; + MockHTTPHandler handler2; + auto req1 = makeGetRequest(); + auto req2 = makeGetRequest(); + + fakeMockCodec(*codec_); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)) + .WillOnce(Return(&handler2)); + + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)) + .WillOnce(Invoke([&handler1] (std::shared_ptr msg) { + handler1.sendHeaders(200, 100); + handler1.sendBody(100); + })); + EXPECT_CALL(handler1, onEgressPaused()); + EXPECT_CALL(handler1, onError(_)).Times(2); + EXPECT_CALL(handler1, detachTransaction()); + EXPECT_CALL(handler2, setTransaction(_)) + .WillOnce(Invoke([&handler2] (HTTPTransaction* txn) { + handler2.txn_ = txn; })); + EXPECT_CALL(handler2, onHeadersComplete(_)) + .WillOnce(Invoke([&handler2] (std::shared_ptr msg) { + handler2.sendHeaders(200, 100); + handler2.sendBody(100); + })); + EXPECT_CALL(handler2, onEgressPaused()); + EXPECT_CALL(*transport_, writeChain(_, _, _)); + EXPECT_CALL(handler2, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& ex) { + ASSERT_EQ(ex.getProxygenError(), kErrorWriteTimeout); + })); + EXPECT_CALL(handler2, detachTransaction()); + + EXPECT_CALL(mockController_, detachSession(_)); + + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req1.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req1)); + codecCallback_->onMessageBegin(HTTPCodec::StreamID(3), req2.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(3), std::move(req2)); + // do the write, enqeue byte event + eventBase_.loop(); + + // recv an abort, detach the handler from txn1 (txn1 stays around due to the + // enqueued byte event) + codecCallback_->onAbort(HTTPCodec::StreamID(1), ErrorCode::PROTOCOL_ERROR); + // recv a transaction timeout on txn1 (used to erroneously create a direct + // response handler) + handler1.txn_->timeoutExpired(); + + // have a write timeout expire (used to cause the direct response handler to + // write out data, messing up the state machine) + httpSession_->shutdownTransportWithReset(kErrorWriteTimeout); + eventBase_.loop(); +} + +TEST_F(MockCodecDownstreamTest, server_push) { + MockHTTPHandler handler; + MockHTTPPushHandler pushHandler; + auto req = makeGetRequest(); + HTTPTransaction* pushTxn = nullptr; + + InSequence enforceOrder; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + pushTxn = handler.txn_->newPushedTransaction( + &pushHandler, handler.txn_->getPriority()); + pushHandler.sendPushHeaders("/foo", "www.foo.com", 100); + pushHandler.sendBody(100); + pushTxn->sendEOM(); + eventBase_.loop(); // flush the push txn's body + })); + EXPECT_CALL(pushHandler, setTransaction(_)) + .WillOnce(Invoke([&pushHandler] (HTTPTransaction* txn) { + pushHandler.txn_ = txn; })); + + EXPECT_CALL(*codec_, generateHeader(_, 2, _, _, _)); + EXPECT_CALL(*codec_, generateBody(_, 2, PtrBufHasLen(100), true)); + EXPECT_CALL(pushHandler, detachTransaction()); + + EXPECT_CALL(handler, onEOM()) + .WillOnce(Invoke([&] { + handler.sendReplyWithBody(200, 100); + eventBase_.loop(); // flush the response to the normal request + })); + + EXPECT_CALL(*codec_, generateHeader(_, 1, _, _, _)); + EXPECT_CALL(*codec_, generateBody(_, 1, PtrBufHasLen(100), true)); + EXPECT_CALL(handler, detachTransaction()); + + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req)); + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, server_push_after_goaway) { + // Tests if goaway + // - drains acknowledged server push transactions + // - aborts server pushed transactions not created at the client + // - prevents new transactions from being created. + MockHTTPHandler handler; + MockHTTPPushHandler pushHandler1; + MockHTTPPushHandler pushHandler2; + HTTPTransaction* pushTxn = nullptr; + + fakeMockCodec(*codec_); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(Invoke([&handler] (HTTPTransaction* txn) { + handler.txn_ = txn; })); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + // Initiate server push transactions. + pushTxn = handler.txn_->newPushedTransaction( + &pushHandler1, handler.txn_->getPriority()); + CHECK(pushTxn->getID() == HTTPCodec::StreamID(2)); + pushHandler1.sendPushHeaders("/foo", "www.foo.com", 100); + pushHandler1.sendBody(100); + pushTxn->sendEOM(); + // Initiate the second push transaction which will be aborted + pushTxn = handler.txn_->newPushedTransaction( + &pushHandler2, handler.txn_->getPriority()); + CHECK(pushTxn->getID() == HTTPCodec::StreamID(4)); + pushHandler2.sendPushHeaders("/foo", "www.foo.com", 100); + pushHandler2.sendBody(100); + pushTxn->sendEOM(); + })); + // Push transaction 1 - drained + EXPECT_CALL(pushHandler1, setTransaction(_)) + .WillOnce(Invoke([&pushHandler1] (HTTPTransaction* txn) { + pushHandler1.txn_ = txn; })); + EXPECT_CALL(pushHandler1, detachTransaction()); + // Push transaction 2 - aborted by onError after goaway + EXPECT_CALL(pushHandler2, setTransaction(_)) + .WillOnce(Invoke([&pushHandler2] (HTTPTransaction* txn) { + pushHandler2.txn_ = txn; })); + EXPECT_CALL(pushHandler2, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& err) { + EXPECT_TRUE(err.hasProxygenError()); + EXPECT_EQ(err.getProxygenError(), kErrorStreamUnacknowledged); + })); + EXPECT_CALL(pushHandler2, detachTransaction()); + + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + // Receive client request + auto req = makeGetRequest(); + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req)); + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + + // Receive goaway acknowledging only the first pushed transactions with id 2. + codecCallback_->onGoaway(2, ErrorCode::NO_ERROR); + + // New server pushed transaction cannot be created after goaway + MockHTTPPushHandler pushHandler3; + EXPECT_EQ(handler.txn_->newPushedTransaction(&pushHandler3, + handler.txn_->getPriority()), nullptr); + + // Send response to the initial client request and this destroys the session + handler.sendReplyWithBody(200, 100); + + eventBase_.loop(); + + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, server_push_abort) { + // Test that assoc txn and other push txns are not affected when client aborts + // a push txn + MockHTTPHandler handler; + MockHTTPPushHandler pushHandler1; + MockHTTPPushHandler pushHandler2; + HTTPTransaction* pushTxn1 = nullptr; + HTTPTransaction* pushTxn2 = nullptr; + + fakeMockCodec(*codec_); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(Invoke([&handler] (HTTPTransaction* txn) { + handler.txn_ = txn; })); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + // Initiate server push transactions + pushTxn1 = handler.txn_->newPushedTransaction( + &pushHandler1, handler.txn_->getPriority()); + CHECK(pushTxn1->getID() == HTTPCodec::StreamID(2)); + pushHandler1.sendPushHeaders("/foo", "www.foo.com", 100); + pushHandler1.sendBody(100); + + pushTxn2 = handler.txn_->newPushedTransaction( + &pushHandler2, handler.txn_->getPriority()); + CHECK(pushTxn2->getID() == HTTPCodec::StreamID(4)); + pushHandler2.sendPushHeaders("/bar", "www.bar.com", 200); + pushHandler2.sendBody(200); + pushTxn2->sendEOM(); + })); + + // pushTxn1 should be aborted + EXPECT_CALL(pushHandler1, setTransaction(_)) + .WillOnce(Invoke([&pushHandler1] (HTTPTransaction* txn) { + pushHandler1.txn_ = txn; })); + EXPECT_CALL(pushHandler1, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& err) { + EXPECT_TRUE(err.hasProxygenError()); + EXPECT_EQ(err.getProxygenError(), kErrorStreamAbort); + })); + EXPECT_CALL(pushHandler1, detachTransaction()); + + EXPECT_CALL(pushHandler2, setTransaction(_)) + .WillOnce(Invoke([&pushHandler2] (HTTPTransaction* txn) { + pushHandler2.txn_ = txn; })); + EXPECT_CALL(pushHandler2, detachTransaction()); + + EXPECT_CALL(handler, onEOM()); + EXPECT_CALL(handler, detachTransaction()); + + // Receive client request + auto req = makeGetRequest(); + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req)); + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + + // Send client abort on one push txn + codecCallback_->onAbort(HTTPCodec::StreamID(2), ErrorCode::CANCEL); + + handler.sendReplyWithBody(200, 100); + + eventBase_.loop(); + + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, server_push_abort_assoc) { + // Test that all associated push transactions are aborted when client aborts + // the assoc stream + MockHTTPHandler handler; + MockHTTPPushHandler pushHandler1; + MockHTTPPushHandler pushHandler2; + + fakeMockCodec(*codec_); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(Invoke([&handler] (HTTPTransaction* txn) { + handler.txn_ = txn; })); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + // Initiate server push transactions + auto pushTxn = handler.txn_->newPushedTransaction( + &pushHandler1, handler.txn_->getPriority()); + CHECK(pushTxn->getID() == HTTPCodec::StreamID(2)); + pushHandler1.sendPushHeaders("/foo", "www.foo.com", 100); + pushHandler1.sendBody(100); + eventBase_.loop(); + + pushTxn = handler.txn_->newPushedTransaction( + &pushHandler2, handler.txn_->getPriority()); + CHECK(pushTxn->getID() == HTTPCodec::StreamID(4)); + pushHandler2.sendPushHeaders("/foo", "www.foo.com", 100); + pushHandler2.sendBody(100); + eventBase_.loop(); + })); + + // Both push txns and the assoc txn should be aborted + EXPECT_CALL(pushHandler1, setTransaction(_)) + .WillOnce(Invoke([&pushHandler1] (HTTPTransaction* txn) { + pushHandler1.txn_ = txn; })); + EXPECT_CALL(pushHandler1, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& err) { + EXPECT_TRUE(err.hasProxygenError()); + EXPECT_EQ(err.getProxygenError(), kErrorStreamAbort); + })); + EXPECT_CALL(pushHandler1, detachTransaction()); + + EXPECT_CALL(pushHandler2, setTransaction(_)) + .WillOnce(Invoke([&pushHandler2] (HTTPTransaction* txn) { + pushHandler2.txn_ = txn; })); + EXPECT_CALL(pushHandler2, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& err) { + EXPECT_TRUE(err.hasProxygenError()); + EXPECT_EQ(err.getProxygenError(), kErrorStreamAbort); + })); + EXPECT_CALL(pushHandler2, detachTransaction()); + + EXPECT_CALL(handler, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& err) { + EXPECT_TRUE(err.hasProxygenError()); + EXPECT_EQ(err.getProxygenError(), kErrorStreamAbort); + })); + EXPECT_CALL(handler, detachTransaction()); + + // Receive client request + auto req = makeGetRequest(); + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req)); + + // Send client abort on assoc stream + codecCallback_->onAbort(HTTPCodec::StreamID(1), ErrorCode::CANCEL); + + eventBase_.loop(); + + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, server_push_client_message) { + // Test that error is generated when client sends data on a pushed stream + MockHTTPHandler handler; + MockHTTPPushHandler pushHandler; + auto req = makeGetRequest(); + HTTPTransaction* pushTxn = nullptr; + + InSequence enforceOrder; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + pushTxn = handler.txn_->newPushedTransaction( + &pushHandler, handler.txn_->getPriority()); + })); + EXPECT_CALL(pushHandler, setTransaction(_)) + .WillOnce(Invoke([&pushHandler] (HTTPTransaction* txn) { + pushHandler.txn_ = txn; })); + + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req)); + + EXPECT_CALL(*codec_, generateRstStream(_, 2, ErrorCode::STREAM_CLOSED)) + .WillRepeatedly(Return(1)); + EXPECT_CALL(pushHandler, onError(_)) + .WillOnce(Invoke([&] (const HTTPException& ex) { + EXPECT_TRUE(ex.hasCodecStatusCode()); + EXPECT_EQ(ex.getCodecStatusCode(), ErrorCode::STREAM_CLOSED); + })); + EXPECT_CALL(pushHandler, detachTransaction()); + + // While the assoc stream is open and pushHandler has been initialized, send + // an upstream message on the push stream causing a RST_STREAM. + req = makeGetRequest(); + codecCallback_->onMessageBegin(HTTPCodec::StreamID(2), req.get()); + + EXPECT_CALL(handler, onEOM()) + .WillOnce(InvokeWithoutArgs([&] { + handler.sendReplyWithBody(200, 100); + eventBase_.loop(); // flush the response to the assoc request + })); + EXPECT_CALL(*codec_, generateHeader(_, 1, _, _, _)); + EXPECT_CALL(*codec_, generateBody(_, 1, PtrBufHasLen(100), true)); + EXPECT_CALL(handler, detachTransaction()); + + // Complete the assoc request/response + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + + eventBase_.loop(); + + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, read_timeout) { + // Test read timeout path + MockHTTPHandler handler1; + auto req1 = makeGetRequest(); + + fakeMockCodec(*codec_); + EXPECT_CALL(*codec_, onIngressEOF()) + .WillRepeatedly(Return()); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req1.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req1)); + // force the read timeout to expire, should be a no-op because the txn is + // still expecting EOM and has its own timer. + httpSession_->timeoutExpired(); + EXPECT_EQ(httpSession_->getConnectionCloseReason(), + ConnectionCloseReason::kMAX_REASON); + + EXPECT_CALL(handler1, onEOM()) + .WillOnce(Invoke([&handler1] () { + handler1.txn_->pauseIngress(); + })); + + // send the EOM, then another timeout. Still no-op since it's waiting + // upstream + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + httpSession_->timeoutExpired(); + EXPECT_EQ(httpSession_->getConnectionCloseReason(), + ConnectionCloseReason::kMAX_REASON); + + EXPECT_CALL(*transport_, writeChain(_, _, _)) + .WillRepeatedly(Invoke([] (TAsyncTransport::WriteCallback* callback, + std::shared_ptr iob, + apache::thrift::async::WriteFlags flags) { + callback->writeSuccess(); + })); + + EXPECT_CALL(handler1, detachTransaction()); + + // Send the response, timeout. Now it's idle and should close. + handler1.txn_->resumeIngress(); + handler1.sendReplyWithBody(200, 100); + eventBase_.loop(); + + httpSession_->timeoutExpired(); + EXPECT_EQ(httpSession_->getConnectionCloseReason(), + ConnectionCloseReason::TIMEOUT); + + // tear down the test + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, ping) { + // Test ping mechanism and that we prioritize the ping reply + MockHTTPHandler handler1; + auto req1 = makeGetRequest(); + + InSequence enforceOrder; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1] () { + handler1.sendReplyWithBody(200, 100); + })); + + // Header egresses immediately + EXPECT_CALL(*codec_, generateHeader(_, _, _, _, _)); + // Ping jumps ahead of queued body in the loop callback + EXPECT_CALL(*codec_, generatePingReply(_, _)); + EXPECT_CALL(*codec_, generateBody(_, _, _, true)); + EXPECT_CALL(handler1, detachTransaction()); + + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req1.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req1)); + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + codecCallback_->onPingRequest(1); + + eventBase_.loop(); + + //EXPECT_CALL(*codec_, onIngressEOF()); + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, buffering) { + StrictMock handler; + auto req1 = makePostRequest(); + auto chunk = makeBuf(10); + auto chunkStr = chunk->clone()->moveToFbString(); + + fakeMockCodec(*codec_); + + httpSession_->setDefaultReadBufferLimit(10); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(Invoke([&handler] (HTTPTransaction* txn) { + handler.txn_ = txn; })); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(InvokeWithoutArgs([&handler] () { + handler.txn_->pauseIngress(); + })); + + EXPECT_CALL(*transport_, writeChain(_, _, _)) + .WillRepeatedly(Invoke([&] (TAsyncTransport::WriteCallback* callback, + const shared_ptr iob, + WriteFlags flags) { + callback->writeSuccess(); + })); + + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req1.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req1)); + for (int i = 0; i < 2; i++) { + codecCallback_->onBody(HTTPCodec::StreamID(1), chunk->clone()); + } + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + + EXPECT_CALL(handler, onBody(_)) + .WillOnce(ExpectString(chunkStr)) + .WillOnce(ExpectString(chunkStr)); + + EXPECT_CALL(handler, onEOM()); + + EXPECT_CALL(handler, detachTransaction()); + + eventBase_.runAfterDelay([&handler, this] { + handler.txn_->resumeIngress(); + handler.sendReplyWithBody(200, 100); + }, 30); + eventBase_.runAfterDelay([&handler, this] { + httpSession_->shutdownTransportWithReset( + ProxygenError::kErrorConnectionReset); + }, 50); + + EXPECT_CALL(mockController_, detachSession(_)); + eventBase_.loop(); +} + +TEST_F(MockCodecDownstreamTest, spdy_window) { + // Test window updates + MockHTTPHandler handler1; + auto req1 = makeGetRequest(); + + fakeMockCodec(*codec_); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)) + .WillOnce(InvokeWithoutArgs([this] () { + codecCallback_->onSettings( + {{SettingsId::INITIAL_WINDOW_SIZE, 4000}}); + })); + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1] () { + handler1.sendHeaders(200, 16000); + handler1.sendBody(12000); + // 12kb buffered -> pause upstream + })); + EXPECT_CALL(handler1, onEgressPaused()) + .WillOnce(InvokeWithoutArgs([&handler1, this] () { + eventBase_.runInLoop([this] { + codecCallback_->onWindowUpdate(1, 4000); + }); + // triggers 4k send, 8k buffered, resume + })) + .WillOnce(InvokeWithoutArgs([&handler1, this] () { + eventBase_.runInLoop([this] { + codecCallback_->onWindowUpdate(1, 8000); + }); + // triggers 8kb send + })) + .WillOnce(InvokeWithoutArgs([] () {})); + EXPECT_CALL(handler1, onEgressResumed()) + .WillOnce(InvokeWithoutArgs([&handler1, this] () { + handler1.sendBody(4000); + // 12kb buffered -> pause upstream + })) + .WillOnce(InvokeWithoutArgs([&handler1, this] () { + handler1.txn_->sendEOM(); + eventBase_.runInLoop([this] { + codecCallback_->onWindowUpdate(1, 4000); + }); + })); + + EXPECT_CALL(handler1, detachTransaction()); + + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req1.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req1)); + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + // Pad coverage numbers + std::ostrstream stream; + stream << *handler1.txn_ << httpSession_ + << httpSession_->getLocalAddress() << httpSession_->getPeerAddress(); + EXPECT_TRUE(httpSession_->isBusy()); + + EXPECT_CALL(mockController_, detachSession(_)); + + EXPECT_CALL(*transport_, writeChain(_, _, _)) + .WillRepeatedly(Invoke([] (TAsyncTransport::WriteCallback* callback, + std::shared_ptr iob, + apache::thrift::async::WriteFlags flags) { + callback->writeSuccess(); + })); + eventBase_.loop(); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, double_resume) { + // Test spdy ping mechanism and egress re-ordering + MockHTTPHandler handler1; + auto req1 = makePostRequest(); + auto buf = makeBuf(5); + auto bufStr = buf->clone()->moveToFbString(); + + fakeMockCodec(*codec_); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(Invoke([&handler1] (HTTPTransaction* txn) { + handler1.txn_ = txn; })); + EXPECT_CALL(handler1, onHeadersComplete(_)) + .WillOnce(InvokeWithoutArgs([&handler1, this] { + handler1.txn_->pauseIngress(); + eventBase_.runAfterDelay([&handler1] { + handler1.txn_->resumeIngress(); + }, 50); + })); + EXPECT_CALL(handler1, onBody(_)) + .WillOnce(Invoke([&handler1, &bufStr] ( + std::shared_ptr chain) { + EXPECT_EQ(bufStr, chain->moveToFbString()); + handler1.txn_->pauseIngress(); + handler1.txn_->resumeIngress(); + })); + + EXPECT_CALL(handler1, onEOM()) + .WillOnce(InvokeWithoutArgs([&handler1] () { + handler1.sendReplyWithBody(200, 100, false); + })); + EXPECT_CALL(handler1, detachTransaction()); + + codecCallback_->onMessageBegin(HTTPCodec::StreamID(1), req1.get()); + codecCallback_->onHeadersComplete(HTTPCodec::StreamID(1), std::move(req1)); + codecCallback_->onBody(HTTPCodec::StreamID(1), std::move(buf)); + codecCallback_->onMessageComplete(HTTPCodec::StreamID(1), false); + + EXPECT_CALL(mockController_, detachSession(_)); + + EXPECT_CALL(*transport_, writeChain(_, _, _)) + .WillRepeatedly(Invoke([] (TAsyncTransport::WriteCallback* callback, + std::shared_ptr iob, + apache::thrift::async::WriteFlags flags) { + callback->writeSuccess(); + })); + + eventBase_.loop(); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST(HTTPDownstreamTest, new_txn_egress_paused) { + // Send 1 request with prio=0 + // Have egress pause while sending the first response + // Send a second request with prio=1 + // -- the new txn should start egress paused + // Finish the body and eom both responses + // Unpause egress + // The first txn should complete first + HTTPCodec::StreamID curId(1); + std::array, 2> handlers; + TAsyncTransport::WriteCallback* delayedWrite = nullptr; + EventBase evb; + + // Setup the controller and its expecations. + NiceMock mockController; + EXPECT_CALL(mockController, getRequestHandler(_, _)) + .WillOnce(Return(&handlers[0])) + .WillOnce(Return(&handlers[1])); + + // Setup the codec, its callbacks, and its expectations. + auto codec = makeDownstreamParallelCodec(); + HTTPCodec::Callback* codecCallback = nullptr; + EXPECT_CALL(*codec, setCallback(_)) + .WillRepeatedly(SaveArg<0>(&codecCallback)); + // Let the codec generate a huge header for the first txn + static const uint64_t header1Len = HTTPSession::getPendingWriteMax(); + static const uint64_t header2Len = 20; + static const uint64_t body1Len = 30; + static const uint64_t body2Len = 40; + EXPECT_CALL(*codec, generateHeader(_, _, _, _, _)) + .WillOnce(Invoke([&] (IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + const HTTPMessage& msg, + HTTPCodec::StreamID assocStream, + HTTPHeaderSize* size) { + CHECK_EQ(stream, HTTPCodec::StreamID(1)); + writeBuf.append(makeBuf(header1Len)); + if (size) { + size->uncompressed = header1Len; + } + })) + // Let the codec generate a regular sized header for the second txn + .WillOnce(Invoke([&] (IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + const HTTPMessage& msg, + HTTPCodec::StreamID ascocStream, + HTTPHeaderSize* size) { + CHECK_EQ(stream, HTTPCodec::StreamID(2)); + writeBuf.append(makeBuf(header2Len)); + if (size) { + size->uncompressed = header2Len; + } + })); + EXPECT_CALL(*codec, generateBody(_, _, _, _)) + .WillOnce(Invoke([&] (IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + shared_ptr chain, + bool eom) { + CHECK_EQ(stream, HTTPCodec::StreamID(1)); + CHECK_EQ(chain->computeChainDataLength(), body1Len); + CHECK(eom); + writeBuf.append(chain->clone()); + return body1Len; + })) + .WillOnce(Invoke([&] (IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + shared_ptr chain, + bool eom) { + CHECK_EQ(stream, HTTPCodec::StreamID(2)); + CHECK_EQ(chain->computeChainDataLength(), body2Len); + CHECK(eom); + writeBuf.append(chain->clone()); + return body2Len; + })); + + bool transportGood = true; + auto transport = newMockTransport(&evb); + EXPECT_CALL(*transport, good()) + .WillRepeatedly(ReturnPointee(&transportGood)); + EXPECT_CALL(*transport, closeNow()) + .WillRepeatedly(Assign(&transportGood, false)); + // We expect the writes to come in this order: + // txn1 headers -> txn1 eom -> txn2 headers -> txn2 eom + EXPECT_CALL(*transport, writeChain(_, _, _)) + .WillOnce(Invoke([&] (TAsyncTransport::WriteCallback* callback, + const shared_ptr iob, + WriteFlags flags) { + CHECK_EQ(iob->computeChainDataLength(), header1Len); + delayedWrite = callback; + CHECK(delayedWrite != nullptr); + })) + .WillOnce(Invoke([&] (TAsyncTransport::WriteCallback* callback, + const shared_ptr iob, + WriteFlags flags) { + CHECK(delayedWrite == nullptr); + // Make sure the second txn has started + CHECK(handlers[1].txn_ != nullptr); + // Headers from txn 2 jump the queue and get lumped into + // this write + CHECK_EQ(iob->computeChainDataLength(), + header2Len + body1Len); + callback->writeSuccess(); + })) + .WillOnce(Invoke([&] (TAsyncTransport::WriteCallback* callback, + const shared_ptr iob, + WriteFlags flags) { + CHECK_EQ(iob->computeChainDataLength(), body2Len); + callback->writeSuccess(); + })); + + // Create the downstream session, thus initializing codecCallback + auto transactionTimeouts = makeInternalTimeoutSet(&evb); + auto session = new HTTPDownstreamSession( + transactionTimeouts.get(), + TAsyncTransport::UniquePtr(transport), + localAddr, peerAddr, + &mockController, std::move(codec), + mockTransportInfo); + session->startNow(); + + for (auto& handler: handlers) { + // Note that order of expecatations doesn't matter here + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(InvokeWithoutArgs([&] { + CHECK(handler.txn_->isEgressPaused() == + (handler.txn_->getID() == HTTPCodec::StreamID(2))); + })); + EXPECT_CALL(handler, onEOM()) + .WillOnce(InvokeWithoutArgs([&] { + CHECK(handler.txn_->isEgressPaused() == + (handler.txn_->getID() == HTTPCodec::StreamID(2))); + const HTTPMessage response; + handler.txn_->sendHeaders(response); + })); + EXPECT_CALL(handler, detachTransaction()) + .WillOnce(InvokeWithoutArgs([&] { + handler.txn_ = nullptr; + })); + EXPECT_CALL(handler, onEgressPaused()); + } + + auto p0Msg = getPriorityMessage(0); + auto p1Msg = getPriorityMessage(1); + + codecCallback->onMessageBegin(curId, p0Msg.get()); + codecCallback->onHeadersComplete(curId, std::move(p0Msg)); + codecCallback->onMessageComplete(curId, false); + ASSERT_FALSE(handlers[0].txn_->isEgressPaused()); + // looping the evb should pause egress when the huge header gets written out + evb.loop(); + // Start the second transaction + codecCallback->onMessageBegin(++curId, p1Msg.get()); + codecCallback->onHeadersComplete(curId, std::move(p1Msg)); + codecCallback->onMessageComplete(curId, false); + // Make sure both txns have egress paused + CHECK(handlers[0].txn_ != nullptr); + ASSERT_TRUE(handlers[0].txn_->isEgressPaused()); + CHECK(handlers[1].txn_ != nullptr); + ASSERT_TRUE(handlers[1].txn_->isEgressPaused()); + // Send body on the second transaction first, then 1st, but the asserts we + // have set up check that the first transaction writes out first. + handlers[1].txn_->sendBody(makeBuf(body2Len)); + handlers[1].txn_->sendEOM(); + handlers[0].txn_->sendBody(makeBuf(body1Len)); + handlers[0].txn_->sendEOM(); + // Now lets ack the first delayed write + auto tmp = delayedWrite; + delayedWrite = nullptr; + tmp->writeSuccess(); + ASSERT_TRUE(handlers[0].txn_ == nullptr); + ASSERT_TRUE(handlers[1].txn_ == nullptr); + + // Cleanup + session->shutdownTransport(); + evb.loop(); +} + +TEST_F(MockCodecDownstreamTest, conn_flow_control_blocked) { + // Let the connection level flow control window fill and then make sure + // control frames still can be processed + InSequence enforceOrder; + NiceMock handler1; + NiceMock handler2; + auto wantToWrite = spdy::kInitialWindow + 50000; + auto wantToWriteStr = folly::to(wantToWrite); + auto req1 = makeGetRequest(); + auto req2 = makeGetRequest(); + auto resp1 = makeResponse(200); + resp1->getHeaders().set(HTTP_HEADER_CONTENT_LENGTH, wantToWriteStr); + auto resp2 = makeResponse(200); + resp2->getHeaders().set(HTTP_HEADER_CONTENT_LENGTH, wantToWriteStr); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler1.txn_)); + EXPECT_CALL(handler1, onHeadersComplete(_)); + EXPECT_CALL(*codec_, generateHeader(_, 1, _, _, _)); + unsigned bodyLen = 0; + EXPECT_CALL(*codec_, generateBody(_, 1, _, false)) + .WillRepeatedly(Invoke([&] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + std::shared_ptr chain, + bool eom) { + bodyLen += chain->computeChainDataLength(); + return 0; // don't want byte events + })); + + codecCallback_->onMessageBegin(1, req1.get()); + codecCallback_->onHeadersComplete(1, std::move(req1)); + codecCallback_->onWindowUpdate(1, wantToWrite); // ensure the per-stream + // window doesn't block + handler1.txn_->sendHeaders(*resp1); + handler1.txn_->sendBody(makeBuf(wantToWrite)); // conn blocked, stream open + handler1.txn_->sendEOM(); + eventBase_.loop(); // actually send (most of) the body + CHECK_EQ(bodyLen, spdy::kInitialWindow); // should have written a full window + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler2)); + EXPECT_CALL(handler2, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler2.txn_)); + EXPECT_CALL(handler2, onHeadersComplete(_)); + EXPECT_CALL(*codec_, generateHeader(_, 3, _, _, _)); + + // Make sure we can send headers of response to a second request + codecCallback_->onMessageBegin(3, req2.get()); + codecCallback_->onHeadersComplete(3, std::move(req2)); + handler2.txn_->sendHeaders(*resp2); + + eventBase_.loop(); + + // Give a connection level window update of 10 bytes -- this should allow 10 + // bytes of the txn1 response to be written + codecCallback_->onWindowUpdate(0, 10); + EXPECT_CALL(*codec_, generateBody(_, 1, PtrBufHasLen(10), false)); + eventBase_.loop(); + + // Just tear everything down now. + EXPECT_CALL(handler1, detachTransaction()); + codecCallback_->onAbort(handler1.txn_->getID(), ErrorCode::INTERNAL_ERROR); + eventBase_.loop(); + + EXPECT_CALL(handler2, detachTransaction()); + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); + eventBase_.loop(); +} + +TEST_F(MockCodecDownstreamTest, unpaused_large_post) { + // Make sure that a large POST that streams into the handler generates + // connection level flow control so that the entire POST can be received. + InSequence enforceOrder; + NiceMock handler1; + unsigned kNumChunks = 10; + auto wantToWrite = spdy::kInitialWindow * kNumChunks; + auto wantToWriteStr = folly::to(wantToWrite); + auto req1 = makePostRequest(); + req1->getHeaders().set(HTTP_HEADER_CONTENT_LENGTH, wantToWriteStr); + auto req1Body = makeBuf(wantToWrite); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler1.txn_)); + + EXPECT_CALL(handler1, onHeadersComplete(_)); + for (unsigned i = 0; i < kNumChunks; ++i) { + EXPECT_CALL(*codec_, generateWindowUpdate(_, 0, spdy::kInitialWindow)); + EXPECT_CALL(handler1, onBody(PtrBufHasLen(spdy::kInitialWindow))); + EXPECT_CALL(*codec_, generateWindowUpdate(_, 1, spdy::kInitialWindow)); + } + EXPECT_CALL(handler1, onEOM()); + + codecCallback_->onMessageBegin(1, req1.get()); + codecCallback_->onHeadersComplete(1, std::move(req1)); + // Give kNumChunks chunks, each of the maximum window size. We should generate + // window update for each chunk + for (unsigned i = 0; i < kNumChunks; ++i) { + codecCallback_->onBody(1, makeBuf(spdy::kInitialWindow)); + } + codecCallback_->onMessageComplete(1, false); + + // Just tear everything down now. + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, ingress_paused_window_update) { + // Test sending a large response body while the handler has ingress paused. We + // should process the ingress window_updates and deliver the full body + InSequence enforceOrder; + NiceMock handler1; + auto req = makeGetRequest(); + size_t respSize = spdy::kInitialWindow * 10; + unique_ptr resp; + unique_ptr respBody; + tie(resp, respBody) = makeResponse(200, respSize); + size_t written = 0; + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler1)); + EXPECT_CALL(handler1, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler1.txn_)); + + EXPECT_CALL(handler1, onHeadersComplete(_)) + .WillOnce(InvokeWithoutArgs([&] () { + // Pause ingress. Make sure we process the window updates anyway + handler1.txn_->pauseIngress(); + })); + EXPECT_CALL(*codec_, generateHeader(_, _, _, _, _)); + EXPECT_CALL(*codec_, generateBody(_, _, _, _)) + .WillRepeatedly( + Invoke([&] (folly::IOBufQueue& writeBuf, + HTTPCodec::StreamID stream, + std::shared_ptr chain, + bool eom) { + auto len = chain->computeChainDataLength(); + written += len; + return len; + })); + + codecCallback_->onWindowUpdate(0, respSize); // open conn-level window + codecCallback_->onMessageBegin(1, req.get()); + codecCallback_->onHeadersComplete(1, std::move(req)); + EXPECT_TRUE(handler1.txn_->isIngressPaused()); + + // Unblock txn-level flow control and try to egress the body + codecCallback_->onWindowUpdate(1, respSize); + handler1.txn_->sendHeaders(*resp); + handler1.txn_->sendBody(std::move(respBody)); + + eventBase_.loop(); + EXPECT_EQ(written, respSize); + + // Just tear everything down now. + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->shutdownTransportWithReset(kErrorConnectionReset); +} + +TEST_F(MockCodecDownstreamTest, shutdown_then_send_push_headers) { + // Test that notifying session of shutdown before sendHeaders() called on a + // pushed txn lets that push txn finish. + EXPECT_CALL(*codec_, supportsPushTransactions()) + .WillRepeatedly(Return(true)); + + InSequence enforceOrder; + NiceMock handler; + MockHTTPPushHandler pushHandler; + auto req = makeGetRequest(); + + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + + EXPECT_CALL(handler, onHeadersComplete(_)) + .WillOnce(Invoke([&] (std::shared_ptr msg) { + auto pushTxn = handler.txn_->newPushedTransaction( + &pushHandler, handler.txn_->getPriority()); + // start shutdown process + httpSession_->notifyPendingShutdown(); + // we should be able to process new requests + EXPECT_TRUE(codec_->isReusable()); + pushHandler.sendPushHeaders("/foo", "www.foo.com", 0); + // we should* still* be able to process new requests + EXPECT_TRUE(codec_->isReusable()); + pushTxn->sendEOM(); + })); + EXPECT_CALL(pushHandler, setTransaction(_)) + .WillOnce(SaveArg<0>(&pushHandler.txn_)); + EXPECT_CALL(*codec_, generateHeader(_, 2, _, _, _)); + EXPECT_CALL(*codec_, generateEOM(_, 2)); + EXPECT_CALL(pushHandler, detachTransaction()); + EXPECT_CALL(handler, onEOM()) + .WillOnce(Invoke([&] { + handler.sendReply(); + })); + EXPECT_CALL(*codec_, generateHeader(_, 1, _, _, _)); + EXPECT_CALL(*codec_, generateEOM(_, 1)); + EXPECT_CALL(handler, detachTransaction()); + + codecCallback_->onMessageBegin(1, req.get()); + codecCallback_->onHeadersComplete(1, std::move(req)); + codecCallback_->onMessageComplete(1, false); + + // finish shutdown + EXPECT_CALL(*codec_, onIngressEOF()); + EXPECT_CALL(mockController_, detachSession(_)); + httpSession_->dropConnection(); + + eventBase_.loop(); +} + +TEST_F(MockCodecDownstreamTest, read_iobuf_chain_shutdown) { + // Given an ingress IOBuf chain of 2 parts, if we shutdown after reading the + // first part of the chain, we shouldn't read the second part. One way to + // simulate a 2 part chain is to put more ingress in readBuf while we are + // inside HTTPCodec::onIngress() + + InSequence enforceOrder; + + auto f = [&] () { + void* buf; + size_t bufSize; + transportCb_->getReadBuffer(&buf, &bufSize); + transportCb_->readDataAvailable(bufSize); + }; + + EXPECT_CALL(*codec_, onIngress(_)) + .WillOnce(Invoke([&] (const IOBuf& buf) { + // This first time, don't process any data. This will cause the + // ingress chain to grow in size later. + EXPECT_FALSE(buf.isChained()); + return 0; + })) + .WillOnce(Invoke([&] (const IOBuf& buf) { + // Now there should be a second buffer in the chain. + EXPECT_TRUE(buf.isChained()); + // Shutdown writes. This enough to destroy the session. + httpSession_->shutdownTransport(false, true); + return buf.length(); + })); + // We shouldn't get a third onIngress() callback. This will be enforced by the + // test framework since the codec is a strict mock. + EXPECT_CALL(mockController_, detachSession(_)); + + f(); + f(); // The first time wasn't processed, so this should make a len=2 chain. + eventBase_.loop(); +} + +void MockCodecDownstreamTest::testGoaway(bool doubleGoaway, + bool dropConnection) { + NiceMock handler; + MockHTTPHandler pushHandler; + + liveGoaways_ = true; + if (doubleGoaway) { + EXPECT_CALL(mockController_, getRequestHandler(_, _)) + .WillOnce(Return(&handler)); + EXPECT_CALL(handler, setTransaction(_)) + .WillOnce(SaveArg<0>(&handler.txn_)); + + EXPECT_CALL(handler, onHeadersComplete(_)); + EXPECT_CALL(handler, onEOM()) + .WillOnce(Invoke([&] { + handler.sendReply(); + })); + EXPECT_CALL(*codec_, generateHeader(_, 1, _, _, _)); + EXPECT_CALL(*codec_, generateEOM(_, 1)); + EXPECT_CALL(handler, detachTransaction()); + + // Turn on double GOAWAY drain + codec_->enableDoubleGoawayDrain(); + } + + // Send a GOAWAY acking uninitiated transactions + EXPECT_FALSE(drainPending_); + httpSession_->notifyPendingShutdown(); + EXPECT_EQ(drainPending_, doubleGoaway); + EXPECT_FALSE(reusable_); + + if (doubleGoaway) { + // Should be able to process new requests + auto req1 = makeGetRequest(); + codecCallback_->onMessageBegin(1, req1.get()); + codecCallback_->onHeadersComplete(1, std::move(req1)); + codecCallback_->onMessageComplete(1, false); + } + + TAsyncTransport::WriteCallback* cb = nullptr; + EXPECT_CALL(*transport_, writeChain(_, _, _)) + .WillOnce(Invoke([&] (TAsyncTransport::WriteCallback* callback, + const shared_ptr iob, + WriteFlags flags) { + // don't immediately flush the goaway + cb = callback; + })); + if (doubleGoaway || !dropConnection) { + // single goaway, drop connection doesn't get onIngressEOF + EXPECT_CALL(*codec_, onIngressEOF()); + } + eventBase_.loopOnce(); + + EXPECT_CALL(mockController_, detachSession(_)); + if (dropConnection) { + EXPECT_CALL(*transport_, closeNow()) + .WillOnce(DoAll(Assign(&transportGood_, false), + Invoke([cb] { + cb->writeError(0, TTransportException()); + }))); + + httpSession_->dropConnection(); + } else { + EXPECT_CALL(*codec_, isBusy()); + httpSession_->closeWhenIdle(); + cb->writeSuccess(); + } + EXPECT_FALSE(drainPending_); + EXPECT_FALSE(reusable_); +} + +TEST_F(MockCodecDownstreamTest, send_double_goaway_timeout) { + testGoaway(true, true); +} +TEST_F(MockCodecDownstreamTest, send_double_goaway_idle) { + testGoaway(true, false); +} +TEST_F(MockCodecDownstreamTest, send_goaway_timeout) { + testGoaway(false, true); +} +TEST_F(MockCodecDownstreamTest, send_goaway_idle) { + testGoaway(false, false); +} + +TEST_F(MockCodecDownstreamTest, shutdown_then_error) { + // Test that we ignore any errors after we shutdown the socket in HTTPSession. + onIngressImpl([&] (const IOBuf& buf) { + // This executes as the implementation of HTTPCodec::onIngress() + + InSequence dummy; + HTTPException err(HTTPException::Direction::INGRESS, "foo"); + err.setHttpStatusCode(400); + HTTPMessage req = getGetRequest(); + MockHTTPHandler handler; + + // Creates and adds a txn to the session + codecCallback_->onMessageBegin(1, &req); + + EXPECT_CALL(*codec_, closeOnEgressComplete()) + .WillOnce(Return(false)); + EXPECT_CALL(*codec_, onIngressEOF()); + EXPECT_CALL(mockController_, detachSession(_)); + + httpSession_->shutdownTransport(); + + codecCallback_->onError(1, err, false); + return buf.computeChainDataLength(); + }); +} diff --git a/proxygen/lib/http/session/test/TestUtils.cpp b/proxygen/lib/http/session/test/TestUtils.cpp new file mode 100644 index 0000000000..b2ecb6dc04 --- /dev/null +++ b/proxygen/lib/http/session/test/TestUtils.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/session/test/TestUtils.h" + +using namespace apache::thrift::async; +using namespace apache::thrift::test; +using namespace apache::thrift::transport; +using namespace folly; + +namespace proxygen { + +const TransportInfo mockTransportInfo = TransportInfo(); +const SocketAddress localAddr{"127.0.0.1", 80}; +const SocketAddress peerAddr{"127.0.0.1", 12345}; + +TAsyncTimeoutSet::UniquePtr makeInternalTimeoutSet(EventBase* evb) { + return TAsyncTimeoutSet::UniquePtr( + new TAsyncTimeoutSet(evb, TimeoutManager::InternalEnum::INTERNAL, + std::chrono::milliseconds(500))); +} + +TAsyncTimeoutSet::UniquePtr makeTimeoutSet(EventBase* evb) { + return TAsyncTimeoutSet::UniquePtr( + new TAsyncTimeoutSet(evb, std::chrono::milliseconds(500))); +} + +testing::NiceMock* newMockTransport(EventBase* evb) { + auto transport = new testing::NiceMock(); + EXPECT_CALL(*transport, getEventBase()) + .WillRepeatedly(testing::Return(evb)); + return transport; +} + +} diff --git a/proxygen/lib/http/session/test/TestUtils.h b/proxygen/lib/http/session/test/TestUtils.h new file mode 100644 index 0000000000..d52d047c5f --- /dev/null +++ b/proxygen/lib/http/session/test/TestUtils.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/session/HTTPSession.h" + +#include +#include +#include +#include + +namespace proxygen { + +extern const TransportInfo mockTransportInfo; +extern const folly::SocketAddress localAddr; +extern const folly::SocketAddress peerAddr; + +apache::thrift::async::TAsyncTimeoutSet::UniquePtr +makeInternalTimeoutSet(folly::EventBase* evb); + +apache::thrift::async::TAsyncTimeoutSet::UniquePtr +makeTimeoutSet(folly::EventBase* evb); + +testing::NiceMock* +newMockTransport(folly::EventBase* evb); + +} diff --git a/proxygen/lib/http/session/test/test_cert1.key b/proxygen/lib/http/session/test/test_cert1.key new file mode 100644 index 0000000000..a8c1eb21b8 --- /dev/null +++ b/proxygen/lib/http/session/test/test_cert1.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA5RR2meSxKy14TtIOqxxqJybf3VCzGQuWxneCFd8gKj1vUx/i +bTwlyq3M+1p8ggOuCeTTgOLJ7fCI7XeaiwQLzSQDvUju4nuiTEzdoWo5e6IL7Pih +IvED7edRsJEhHO88xMoKmHC99lIfzfReB94tSqhLFtOOu7WZygdoou07C20Imgmc +ZJCCnLSxgiZVPJk5KxHooTru0ciYDPxkcm97fHiD5BnsQsru3ZlDfGeGkN5so98K +glf8FoflQllpkljNiEHmNHBCRfY72lgDC2UJ80CIYDB532eudqB5dpW2JmUKL5KP +IrnAW4dTIN7e5OgE6nwEuo6t3B6UnZWfKK5QmwIDAQABAoIBAQC2nX1+aMSjevfx +PZrb5SbqVx8WuoJLcOWdCR/2d4yYn6jjRwOluNEXpT1HEnc23eAbmwLQN/ppZ9qA +E1xrfao6un8QoddtTMmwsYXsa1WvrRp8qicqR5+uQzzn2InhGwYCil2v8BA/EHCy +Kh/bfc31OfvCEpwyHy6d0xIi76KmE58FAXFMrXvyhHt/v3p4u6UAN7D+MvjRgH+o +zkvPHOqKjhbOde4ob29Q2vSHOymkY0macQz3OOjcdNyrtTmxrZb7iZ1MchJ+Qw8d +23IB0UPbTVIehlzsnOsVcZhipYmGoM5fnfMHaZYyvBqvi9PVePjqR+rHmVLyGpH0 +AiTWBgwBAoGBAPmY5cws6kcX5qnG+H7K3rCpo/LlUKEG6NIf4/X0EpZF8/CumcHt ++pAQYURGdUAanCLDlnA7KQm1L/fWHTOCqDmTbj0Dh9fNPfpIZ9ExfF5uE9avuAlO +MxW4gizY9k1wTPuYovnLcLIE/MJTaDefsX6qt7J5UCb/bYVAbyFCSAGBAoGBAOr0 +1CBuYSNkB+tyXmqBELodXc2kYBtRvNHvTPojnBktEUJz8Zbn1Alr0EPhLSuDgRQL +jFMgo9K9FY4fh6Bb+k+pyzWE9QsxU+VJhaXhBwDMaoGwWbqLPOifPpd33O8DEppq +lCeIf7Gge2hZZcfR9YSbN80hufUO12rnhzwj2igbAoGAWewe5jdMVD6lKId5oVx/ +bFBBs9GwAcOWf5VjsIm1ZJRhNVYCKR8OxXZRRKLJvUWY+OFUQqzdtRSk6nomXRd2 +ef/GmI68EyBrcs9BL9nPP8oStabq+9dDilDF/VhRLEcIRSCRnsMPn+MyGOSpyFht +PwBMXC1elh4yG+7B2s6hWIECgYEA1Kr2tsQyOI56fKrwuwZH1a5vEYwPUasSdhNc +Ya2JxV/9l2DK9ASy8DQeNtLCx/goBbTONw/8OOvdbt8rLAMU9raBsNe/1Xm4rnl1 +90dPx3E/llPBZBBT+VrqlL9xwwXM+OgvYfkzW9we17zTW6K30GeOgqZcY2wjoCgh +SFZLSLECgYAJ1z09ylEL709FOKd8lmFExum9tBxlsbUTnRDAwQJLX/Ms+gDNHxNz +sgktl2SiQh5bs9fDN1tyymzNAO7bFxATWBXQHF1rofkZPh01buwCpLqhqH002rNY +0e6XPuzWl6ewGUZ8m2CDFCuC+N/eeSroz6g2CeQVyhh4JwBKjaVKlQ== +-----END RSA PRIVATE KEY----- diff --git a/proxygen/lib/http/session/test/test_cert1.pem b/proxygen/lib/http/session/test/test_cert1.pem new file mode 100644 index 0000000000..02baef08dd --- /dev/null +++ b/proxygen/lib/http/session/test/test_cert1.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5zCCAc8CAQowDQYJKoZIhvcNAQEFBQAwQzELMAkGA1UEBhMCVVMxDTALBgNV +BAoMBEFzb3gxJTAjBgNVBAMMHEFzb3ggQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkw +HhcNMTQwODE5MjAwOTU5WhcNNDIwMTA0MjAwOTU5WjAwMQswCQYDVQQGEwJVUzEN +MAsGA1UECgwEQXNveDESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA5RR2meSxKy14TtIOqxxqJybf3VCzGQuWxneCFd8g +Kj1vUx/ibTwlyq3M+1p8ggOuCeTTgOLJ7fCI7XeaiwQLzSQDvUju4nuiTEzdoWo5 +e6IL7PihIvED7edRsJEhHO88xMoKmHC99lIfzfReB94tSqhLFtOOu7WZygdoou07 +C20ImgmcZJCCnLSxgiZVPJk5KxHooTru0ciYDPxkcm97fHiD5BnsQsru3ZlDfGeG +kN5so98Kglf8FoflQllpkljNiEHmNHBCRfY72lgDC2UJ80CIYDB532eudqB5dpW2 +JmUKL5KPIrnAW4dTIN7e5OgE6nwEuo6t3B6UnZWfKK5QmwIDAQABMA0GCSqGSIb3 +DQEBBQUAA4IBAQDQcEY2/fLlmpROhsJTJ7srvYA1NhpZlPrNDxOwXfOx+wWg2wjC +cYLcpfH0wQH8E/9I9SrZdxFMkt8bju4EdL8A2rk0xPrMHH0eZYzrW4q0BxABWMxH +c0joYStYQCjhoBVZI28L/s8aoP3s1ETZcFKfQNOsZTF8mlAzwkniUI4RnHaLwl8v +oKB1e0+1utBYOOwQebDHmHc/oVcodteJCivacFrxj9qk2pa5/DuG8j3GkdvOvxhN +hlN5shc7HbyJ+FXiK/0a89qeFJcjwvvPFq7N9JUbHI3GAb/aOpGXANvPyjniS8Dn +BbMzn7xVrkVQxjGuPzgKiqv8WWujtg+zXk8o +-----END CERTIFICATE----- diff --git a/proxygen/lib/http/test/HTTPMessageTest.cpp b/proxygen/lib/http/test/HTTPMessageTest.cpp new file mode 100644 index 0000000000..d17d55c1f5 --- /dev/null +++ b/proxygen/lib/http/test/HTTPMessageTest.cpp @@ -0,0 +1,463 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/HTTPMessage.h" +#include "proxygen/lib/utils/TestUtils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace proxygen; +using namespace std; + +TEST(HTTPMessage, TestParseCookiesSimple) { + HTTPMessage msg; + + msg.getHeaders().add("Cookie", "id=1256679245; data=0:1234567"); + EXPECT_EQ(msg.getCookie("id"), "1256679245"); + EXPECT_EQ(msg.getCookie("data"), "0:1234567"); + EXPECT_EQ(msg.getCookie("mising"), ""); +} + +TEST(HTTPMessage, TestParseCookiesSpaces) { + HTTPMessage msg; + + msg.getHeaders().add("Cookie", " id=1256679245 ; data=0:1234567 ;"); + EXPECT_EQ(msg.getCookie("id"), "1256679245"); + EXPECT_EQ(msg.getCookie("data"), "0:1234567"); +} + +TEST(HTTPMessage, TestParseCookiesSingleCookie) { + HTTPMessage msg; + + msg.getHeaders().add("Cookie", " user_id=1256679245 "); + EXPECT_EQ(msg.getCookie("user_id"), "1256679245"); +} + +TEST(HTTPMessage, TestParseCookiesMultipleCookies) { + HTTPMessage msg; + + msg.getHeaders().add( + "Cookie", "id=1256679245; data=0:1234567; same=Always; Name"); + msg.getHeaders().add( + "Cookie", "id2=1256679245; data2=0:1234567; same=Always; "); + EXPECT_EQ(msg.getCookie("id"), "1256679245"); + EXPECT_EQ(msg.getCookie("id2"), "1256679245"); + EXPECT_EQ(msg.getCookie("data"), "0:1234567"); + EXPECT_EQ(msg.getCookie("data2"), "0:1234567"); + EXPECT_EQ(msg.getCookie("same"), "Always"); + EXPECT_EQ(msg.getCookie("Name"), ""); +} + +TEST(HTTPMessage, TestParseQueryParamsSimple) { + HTTPMessage msg; + string url = "/test?seq=123456&userid=1256679245&dup=1&dup=2&helloWorld" + "&second=was+it+clear+%28already%29%3F"; + + msg.setURL(url); + EXPECT_EQ(msg.getQueryParam("seq"), "123456"); + EXPECT_EQ(msg.getQueryParam("userid"), "1256679245"); + EXPECT_EQ(msg.getQueryParam("dup"), "2"); + EXPECT_EQ(msg.getQueryParam("helloWorld"), ""); + EXPECT_EQ(msg.getIntQueryParam("dup", 5), 2); + EXPECT_EQ(msg.getIntQueryParam("abc", 5), 5); + EXPECT_EQ(msg.getDecodedQueryParam("second"), "was it clear (already)?"); + EXPECT_EQ(msg.getDecodedQueryParam("seq"), "123456"); + EXPECT_EQ(msg.hasQueryParam("seq"), true); + EXPECT_EQ(msg.hasQueryParam("seq1"), false); + EXPECT_EQ(msg.getIntQueryParam("dup"), 2); + EXPECT_ANY_THROW(msg.getIntQueryParam("abc")); + EXPECT_ANY_THROW(msg.getIntQueryParam("second")); +} + +TEST(HTTPMessage, TestParseQueryParamsComplex) { + HTTPMessage msg; + std::vector> input = { + {"", "", ""}, + {"key_and_equal_but_no_value", "=", ""}, + {"only_key", "", ""}, + {"key_and_value", "=", "value"}, + {"key_and_value_2", "=", "value2"}, + {"key_and_value_3", "=", "value3"} + }; + + for (int i = 0; i < (1 << input.size()); ++i) { + std::vector> sub; + for (size_t j = 0; j < input.size(); ++j) { + if ((i & (1 << j))) { + sub.push_back(input[j]); + } + } + + sort(sub.begin(), sub.end()); + do { + bool first = true; + std::string url = "/test?"; + for (const auto& val: sub) { + if (first) { + first = false; + } else { + url += "&"; + } + + url += val[0] + val[1] + val[2]; + } + + msg.setURL(url); + for (const auto& val: sub) { + if (val[0].empty()) { + continue; + } + + EXPECT_EQ(val[2], msg.getQueryParam(val[0])); + } + + } while (next_permutation(sub.begin(), sub.end())); + } +} + +TEST(HTTPMessage, TestHeaderPreservation) { + HTTPMessage msg; + HTTPHeaders& hdrs = msg.getHeaders(); + + hdrs.add("Jojo", "1"); + hdrs.add("Binky", "2"); + hdrs.add("jOJo", "3"); + hdrs.add("bINKy", "4"); + + EXPECT_EQ(hdrs.size(), 4); + EXPECT_EQ(hdrs.getNumberOfValues("jojo"), 2); + EXPECT_EQ(hdrs.getNumberOfValues("binky"), 2); +} + +TEST(HTTPMessage, TestHeaderRemove) { + HTTPMessage msg; + HTTPHeaders& hdrs = msg.getHeaders(); + + hdrs.add("Jojo", "1"); + hdrs.add("Binky", "2"); + hdrs.add("jOJo", "3"); + hdrs.add("bINKy", "4"); + hdrs.remove("jojo"); + + EXPECT_EQ(hdrs.size(), 2); + EXPECT_EQ(hdrs.getNumberOfValues("binky"), 2); +} + +TEST(HTTPMessage, TestSetHeader) { + HTTPMessage msg; + HTTPHeaders& hdrs = msg.getHeaders(); + + hdrs.set("Jojo", "1"); + EXPECT_EQ(hdrs.size(), 1); + EXPECT_EQ(hdrs.getNumberOfValues("Jojo"), 1); + + hdrs.add("jojo", "2"); + hdrs.add("jojo", "3"); + hdrs.add("bar", "4"); + EXPECT_EQ(hdrs.size(), 4); + EXPECT_EQ(hdrs.getNumberOfValues("Jojo"), 3); + + hdrs.set("joJO", "5"); + EXPECT_EQ(hdrs.size(), 2); + EXPECT_EQ(hdrs.getNumberOfValues("Jojo"), 1); +} + +TEST(HTTPMessage, TestCombine) { + HTTPMessage msg; + HTTPHeaders& headers = msg.getHeaders(); + + EXPECT_EQ(headers.combine("Combine"), ""); + + headers.add("Combine", "first value"); + EXPECT_EQ(headers.combine("Combine"), "first value"); + + headers.add("Combine", "second value"); + EXPECT_EQ(headers.combine("Combine"), "first value, second value"); + + headers.add("Combine", "third value"); + EXPECT_EQ(headers.combine("Combine"), + "first value, second value, third value"); +} + +TEST(HTTPMessage, TestProxification) { + HTTPMessage msg; + + folly::SocketAddress dstAddr("192.168.1.1", 1887); + folly::SocketAddress clientAddr("74.125.127.9", 1987); + msg.setDstAddress(dstAddr); + msg.setLocalIp("10.0.0.1"); + msg.ensureHostHeader(); + msg.setWantsKeepalive(false); + + HTTPHeaders& hdrs = msg.getHeaders(); + EXPECT_EQ("192.168.1.1", hdrs.getSingleOrEmpty("Host")); + EXPECT_FALSE(msg.wantsKeepalive()); +} + +TEST(HTTPMessage, TestKeepaliveCheck) { + { + HTTPMessage msg; + msg.setHTTPVersion(1, 0); + EXPECT_FALSE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + EXPECT_TRUE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.getHeaders().add("Connection", "close"); + EXPECT_FALSE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.getHeaders().add("Connection", "ClOsE"); + EXPECT_FALSE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.getHeaders().add("Connection", "foo,bar"); + EXPECT_TRUE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.getHeaders().add("Connection", "foo,bar"); + msg.getHeaders().add("Connection", "abc,CLOSE,def"); + msg.getHeaders().add("Connection", "xyz"); + EXPECT_FALSE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.getHeaders().add("Connection", "foo,bar"); + msg.getHeaders().add("Connection", "abc , CLOSE , def"); + msg.getHeaders().add("Connection", "xyz"); + EXPECT_FALSE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.getHeaders().add("Connection", " close "); + EXPECT_FALSE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.getHeaders().add("Connection", ", close "); + EXPECT_FALSE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 1); + msg.getHeaders().add("Connection", " close , "); + EXPECT_FALSE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 0); + msg.getHeaders().add("Connection", "Keep-Alive"); + EXPECT_TRUE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 0); + msg.getHeaders().add("Connection", "keep-alive"); + EXPECT_TRUE(msg.computeKeepalive()); + } + + { + HTTPMessage msg; + msg.setHTTPVersion(1, 0); + msg.getHeaders().add("Connection", "keep-alive"); + msg.getHeaders().add("Connection", "close"); + EXPECT_FALSE(msg.computeKeepalive()); + } +} + +TEST(HTTPMessage, TestHeaderStripPerHop) { + HTTPMessage msg; + + msg.getHeaders().add("Connection", "a, b, c"); + msg.getHeaders().add("Connection", "d"); + msg.getHeaders().add("Connection", ",,,,"); + msg.getHeaders().add("Connection", " , , , ,"); + msg.getHeaders().add("Connection", ", e"); + msg.getHeaders().add("Connection", " f ,\tg\t, \r\n\th "); + msg.getHeaders().add("Keep-Alive", "true"); + + msg.getHeaders().add("a", "1"); + msg.getHeaders().add("b", "2"); + msg.getHeaders().add("c", "3"); + msg.getHeaders().add("d", "4"); + msg.getHeaders().add("e", "5"); + msg.getHeaders().add("f", "6"); + msg.getHeaders().add("g", "7"); + msg.getHeaders().add("h", "8"); + + EXPECT_EQ(msg.getHeaders().size(), 15); + msg.stripPerHopHeaders(); + EXPECT_EQ(msg.getHeaders().size(), 0); + EXPECT_EQ(msg.getStrippedPerHopHeaders().size(), 15); +} + +TEST(HTTPMessage, TestEmptyName) { + HTTPMessage msg; + EXPECT_DEATH_NO_CORE(msg.getHeaders().set("", "empty name?!"), ".*"); +} + +TEST(HTTPMessage, TestMethod) { + HTTPMessage msg; + + msg.setMethod(HTTPMethod::GET); + EXPECT_EQ("GET", msg.getMethodString()); + EXPECT_EQ(HTTPMethod::GET, msg.getMethod()); + + msg.setMethod("FOO"); + EXPECT_EQ("FOO", msg.getMethodString()); + EXPECT_EQ(boost::none, msg.getMethod()); + + msg.setMethod(HTTPMethod::CONNECT); + EXPECT_EQ("CONNECT", msg.getMethodString()); + EXPECT_EQ(HTTPMethod::CONNECT, msg.getMethod()); +} + +void testPathAndQuery(const string& url, + const string& expectedPath, + const string& expectedQuery) { + HTTPMessage msg; + msg.setURL(url); + + EXPECT_EQ(msg.getURL(), url); + EXPECT_EQ(msg.getPath(), expectedPath); + EXPECT_EQ(msg.getQueryString(), expectedQuery); +} + +TEST(getPathAndQuery, ParseURL) { + testPathAndQuery("http://localhost:80/foo?bar#qqq", "/foo", "bar"); + testPathAndQuery("localhost:80/foo?bar#qqq", "/foo", "bar"); + testPathAndQuery("localhost", "", ""); + testPathAndQuery("/f/o/o?bar#qqq", "/f/o/o", "bar"); + testPathAndQuery("#?hello", "", ""); +} + +TEST(HTTPHeaders, AddStringPiece) { + const char foo[] = "name:value"; + HTTPHeaders headers; + + folly::StringPiece str(foo); + folly::StringPiece name = str.split_step(':'); + headers.add(name, str); + EXPECT_EQ("value", headers.getSingleOrEmpty("name")); +} + +void testRemoveQueryParam(const string& url, + const string& queryParam, + const string& expectedUrl, + const string& expectedQuery) { + HTTPMessage msg; + msg.setURL(url); + bool expectedChange = (url != expectedUrl); + EXPECT_EQ(msg.removeQueryParam(queryParam), expectedChange); + + EXPECT_EQ(msg.getURL(), expectedUrl); + EXPECT_EQ(msg.getQueryString(), expectedQuery); +} + +TEST(HTTPMessage, RemoveQueryParamTests) { + // Query param present + testRemoveQueryParam("http://localhost:80/foo?param1=a¶m2=b#qqq", + "param2", + "http://localhost:80/foo?param1=a#qqq", + "param1=a"); + // Query param not present + testRemoveQueryParam("http://localhost/foo?param1=a¶m2=b#qqq", + "param3", + "http://localhost/foo?param1=a¶m2=b#qqq", + "param1=a¶m2=b"); + // No scheme + testRemoveQueryParam("localhost:80/foo?param1=a¶m2=b#qqq", + "param2", + "localhost:80/foo?param1=a#qqq", + "param1=a"); + // Just hostname as URL and empty query param + testRemoveQueryParam("localhost", "param2", "localhost", ""); + testRemoveQueryParam("localhost", "", "localhost", ""); + // Just path as URL + testRemoveQueryParam("/f/o/o?bar#qqq", "bar", "/f/o/o#qqq", ""); +} + +void testSetQueryParam(const string& url, + const string& queryParam, + const string& paramValue, + const string& expectedUrl, + const string& expectedQuery) { + HTTPMessage msg; + msg.setURL(url); + bool expectedChange = (url != expectedUrl); + EXPECT_EQ(msg.setQueryParam(queryParam, paramValue), expectedChange); + + EXPECT_EQ(msg.getURL(), expectedUrl); + EXPECT_EQ(msg.getQueryString(), expectedQuery); +} + +TEST(HTTPMessage, SetQueryParamTests) { + // Overwrite existing parameter + testSetQueryParam("http://localhost:80/foo?param1=a¶m2=b#qqq", + "param2", + "true", + "http://localhost:80/foo?param1=a¶m2=true#qqq", + "param1=a¶m2=true"); + // Add a query parameter + testSetQueryParam("http://localhost/foo?param1=a¶m2=b#qqq", + "param3", + "true", + "http://localhost/foo?param1=a¶m2=b¶m3=true#qqq", + "param1=a¶m2=b¶m3=true"); + // Add a query parameter, should be alphabetical order + testSetQueryParam("localhost:80/foo?param1=a¶m3=c#qqq", + "param2", + "b", + "localhost:80/foo?param1=a¶m2=b¶m3=c#qqq", + "param1=a¶m2=b¶m3=c"); + // Add a query parameter when no query parameters exist + testSetQueryParam("localhost:80/foo#qqq", + "param2", + "b", + "localhost:80/foo?param2=b#qqq", + "param2=b"); +} diff --git a/proxygen/lib/http/test/Makefile.am b/proxygen/lib/http/test/Makefile.am new file mode 100644 index 0000000000..12d0eefa97 --- /dev/null +++ b/proxygen/lib/http/test/Makefile.am @@ -0,0 +1,14 @@ +SUBDIRS = . + +check_PROGRAMS = LibHTTPTests +LibHTTPTests_SOURCES = \ + HTTPMessageTest.cpp \ + RFC2616Test.cpp \ + WindowTest.cpp + +LibHTTPTests_LDADD = \ + ../libproxygenhttp.la \ + ../../utils/libutils.la \ + ../../test/libtestmain.la + +TESTS = LibHTTPTests diff --git a/proxygen/lib/http/test/MockHTTPMessageFilter.h b/proxygen/lib/http/test/MockHTTPMessageFilter.h new file mode 100644 index 0000000000..213c295280 --- /dev/null +++ b/proxygen/lib/http/test/MockHTTPMessageFilter.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPMessageFilters.h" + +namespace proxygen { + +class MockHTTPMessageFilter : public HTTPMessageFilter { + public: + GMOCK_METHOD1_(, noexcept,, onHeadersComplete, + void(std::shared_ptr)); + void onHeadersComplete(std::unique_ptr msg) noexcept override { + onHeadersComplete(std::shared_ptr(msg.release())); + } + + GMOCK_METHOD1_(, noexcept,, onBody, void(std::shared_ptr)); + void onBody(std::unique_ptr chain) noexcept override { + onBody(std::shared_ptr(chain.release())); + } + GMOCK_METHOD1_(, noexcept,, onChunkHeader, void(size_t)); + GMOCK_METHOD0_(, noexcept,, onChunkComplete, void()); + GMOCK_METHOD1_(, noexcept,, onTrailers, + void(std::shared_ptr trailers)); + void onTrailers(std::unique_ptr trailers) noexcept override { + onTrailers(std::shared_ptr(trailers.release())); + } + GMOCK_METHOD0_(, noexcept,, onEOM, void()); + GMOCK_METHOD1_(, noexcept,, onUpgrade, void(UpgradeProtocol)); + GMOCK_METHOD1_(, noexcept,, onError, void(const HTTPException&)); + + void nextOnHeadersCompletePublic(std::shared_ptr msg) { + std::unique_ptr msgU(new HTTPMessage(*msg)); + nextOnHeadersComplete(std::move(msgU)); + } + + void nextOnEOMPublic() { + nextOnEOM(); + } +}; + +} diff --git a/proxygen/lib/http/test/RFC2616Test.cpp b/proxygen/lib/http/test/RFC2616Test.cpp new file mode 100644 index 0000000000..b46f7f609d --- /dev/null +++ b/proxygen/lib/http/test/RFC2616Test.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/RFC2616.h" + +#include + +using namespace proxygen; + +using std::string; + +TEST(QvalueTest, basic) { + + std::vector output; + + string test1("iso-8859-5, unicode-1-1;q=0.8"); + EXPECT_TRUE(RFC2616::parseQvalues(test1, output)); + EXPECT_EQ(output.size(), 2); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("iso-8859-5")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 1); + EXPECT_EQ(output[1].first.compare(folly::StringPiece("unicode-1-1")), 0); + EXPECT_DOUBLE_EQ(output[1].second, 0.8); + output.clear(); + + string test2("compress, gzip"); + EXPECT_TRUE(RFC2616::parseQvalues(test2, output)); + EXPECT_EQ(output.size(), 2); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("compress")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 1); + EXPECT_EQ(output[1].first.compare(folly::StringPiece("gzip")), 0); + EXPECT_DOUBLE_EQ(output[1].second, 1); + output.clear(); + + string test3(""); + // The spec says a blank one is ok but empty headers are disallowed in SPDY? + EXPECT_FALSE(RFC2616::parseQvalues(test3, output)); + EXPECT_EQ(output.size(), 0); + + string test4("compress;q=0.5, gzip;q=1.0"); + EXPECT_TRUE(RFC2616::parseQvalues(test4, output)); + EXPECT_EQ(output.size(), 2); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("compress")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 0.5); + EXPECT_EQ(output[1].first.compare(folly::StringPiece("gzip")), 0); + EXPECT_DOUBLE_EQ(output[1].second, 1.0); + output.clear(); + + string test5("gzip;q=1.0, identity; q=0.5, *;q=0"); + EXPECT_TRUE(RFC2616::parseQvalues(test5, output)); + EXPECT_EQ(output.size(), 3); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("gzip")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 1); + EXPECT_EQ(output[1].first.compare(folly::StringPiece("identity")), 0); + EXPECT_DOUBLE_EQ(output[1].second, 0.5); + EXPECT_EQ(output[2].first.compare(folly::StringPiece("*")), 0); + EXPECT_DOUBLE_EQ(output[2].second, 0); + output.clear(); + + string test6("da, en-gb;q=0.8, en;q=0.7"); + EXPECT_TRUE(RFC2616::parseQvalues(test6, output)); + EXPECT_EQ(output.size(), 3); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("da")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 1); + EXPECT_EQ(output[1].first.compare(folly::StringPiece("en-gb")), 0); + EXPECT_DOUBLE_EQ(output[1].second, 0.8); + EXPECT_EQ(output[2].first.compare(folly::StringPiece("en")), 0); + EXPECT_DOUBLE_EQ(output[2].second, 0.7); + output.clear(); +} + +TEST(QvalueTest, extras) { + + std::vector output; + + string test1(" iso-8859-5, unicode-1-1; q=0.8 hi mom!"); + EXPECT_TRUE(RFC2616::parseQvalues(test1, output)); + EXPECT_EQ(output.size(), 2); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("iso-8859-5")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 1); + EXPECT_EQ(output[1].first.compare(folly::StringPiece("unicode-1-1")), 0); + EXPECT_DOUBLE_EQ(output[1].second, 0.8); + output.clear(); + + string test2("gzip"); + EXPECT_TRUE(RFC2616::parseQvalues(test2, output)); + EXPECT_EQ(output.size(), 1); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("gzip")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 1); + output.clear(); +} + +TEST(QvalueTest, invalids) { + + std::vector output; + + string test1(",,,"); + EXPECT_FALSE(RFC2616::parseQvalues(test1, output)); + EXPECT_EQ(output.size(), 0); + output.clear(); + + string test2(" ; q=0.1"); + EXPECT_FALSE(RFC2616::parseQvalues(test2, output)); + EXPECT_EQ(output.size(), 0); + output.clear(); + + string test3("gzip; q=uietplease"); + EXPECT_FALSE(RFC2616::parseQvalues(test3, output)); + EXPECT_EQ(output.size(), 1); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("gzip")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 1); + output.clear(); + + string test4("gzip; whoohoo, defalte"); + EXPECT_FALSE(RFC2616::parseQvalues(test4, output)); + EXPECT_EQ(output.size(), 2); + EXPECT_EQ(output[0].first.compare(folly::StringPiece("gzip")), 0); + EXPECT_DOUBLE_EQ(output[0].second, 1); + EXPECT_EQ(output[1].first.compare(folly::StringPiece("defalte")), 0); + EXPECT_DOUBLE_EQ(output[1].second, 1); + output.clear(); + +} diff --git a/proxygen/lib/http/test/WindowTest.cpp b/proxygen/lib/http/test/WindowTest.cpp new file mode 100644 index 0000000000..2167e3a785 --- /dev/null +++ b/proxygen/lib/http/test/WindowTest.cpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/http/Window.h" + +#include + +using namespace proxygen; + +TEST(WindowTest, basic) { + Window w(100); + ASSERT_TRUE(w.free(10)); + ASSERT_EQ(w.getSize(), 110); + ASSERT_EQ(w.getCapacity(), 100); + ASSERT_TRUE(w.reserve(20)); + ASSERT_EQ(w.getSize(), 90); + ASSERT_TRUE(w.reserve(90)); + ASSERT_EQ(w.getSize(), 0); + ASSERT_TRUE(w.free(5)); + ASSERT_EQ(w.getSize(), 5); + ASSERT_TRUE(w.free(0)); + ASSERT_EQ(w.getSize(), 5); +} + +TEST(WindowTest, change_capacity) { + Window w(100); + ASSERT_TRUE(w.setCapacity(10)); + ASSERT_EQ(w.getSize(), 10); + ASSERT_EQ(w.getCapacity(), 10); + ASSERT_TRUE(w.reserve(7)); + ASSERT_TRUE(w.setCapacity(5)); + ASSERT_EQ(w.getCapacity(), 5); + ASSERT_EQ(w.getSize(), -2); + ASSERT_TRUE(w.free(1)); + ASSERT_EQ(w.getSize(), -1); + ASSERT_TRUE(w.setCapacity(6)); + ASSERT_EQ(w.getSize(), 0); + ASSERT_EQ(w.getCapacity(), 6); + ASSERT_TRUE(w.setCapacity(7)); + ASSERT_EQ(w.getSize(), 1); + ASSERT_EQ(w.getCapacity(), 7); + ASSERT_EQ(w.getOutstanding(), 6); +} + +TEST(WindowTest, exceed_window) { + Window w(100); + ASSERT_TRUE(w.reserve(50)); + ASSERT_TRUE(w.reserve(40)); + ASSERT_EQ(w.getSize(), 10); + ASSERT_FALSE(w.reserve(20)); +} + +TEST(WindowTest, overflow) { + Window w(0); + ASSERT_FALSE(w.reserve(std::numeric_limits::max())); +} + +TEST(WindowTest, underflow) { + Window w(100); + ASSERT_TRUE(w.free(100)); // You can manually bump up the window + ASSERT_TRUE(w.free(100)); // You can manually bump up the window + ASSERT_EQ(w.getSize(), 300); + ASSERT_FALSE(w.free(std::numeric_limits::max())); // to a point +} + +TEST(WindowTest, huge_reserve) { + Window w(100); + ASSERT_FALSE(w.reserve(std::numeric_limits::max())); +} + +TEST(WindowTest, huge_free) { + Window w1(0); + ASSERT_TRUE(w1.free(std::numeric_limits::max())); + Window w2(1); + ASSERT_FALSE(w2.free(std::numeric_limits::max())); +} + +TEST(WindowTest, huge_free_2) { + for (unsigned i = 0; i < 10; ++i) { + Window w(i); + ASSERT_TRUE(w.free(std::numeric_limits::max() - i)); + ASSERT_FALSE(w.free(1)); + } +} + +TEST(WindowTest, bytes_outstanding) { + Window w(100); + ASSERT_EQ(w.getOutstanding(), 0); + ASSERT_TRUE(w.reserve(20)); + ASSERT_EQ(w.getOutstanding(), 20); + ASSERT_TRUE(w.free(30)); + // outstanding bytes is -10, but the API clamps this to 0 + ASSERT_EQ(w.getOutstanding(), 0); +} + +TEST(WindowTest, bytes_outstanding_after_fail) { + Window w(100); + ASSERT_EQ(w.getOutstanding(), 0); + ASSERT_FALSE(w.reserve(110)); + ASSERT_EQ(w.getOutstanding(), 0); +} + +TEST(WindowTest, non_strict) { + Window w(100); + ASSERT_TRUE(w.reserve(110, false)); + ASSERT_EQ(w.getOutstanding(), 110); + ASSERT_EQ(w.getSize(), -10); +} + +TEST(WindowTest, new_capacity_overflow) { + Window w(0); + ASSERT_TRUE(w.free(10)); + ASSERT_EQ(w.getOutstanding(), 0); + ASSERT_EQ(w.getSize(), 10); + ASSERT_TRUE(w.setCapacity(std::numeric_limits::max() - 10)); + ASSERT_EQ(w.getSize(), std::numeric_limits::max()); + ASSERT_FALSE(w.setCapacity(std::numeric_limits::max() - 9)); +} diff --git a/proxygen/lib/services/Acceptor.cpp b/proxygen/lib/services/Acceptor.cpp new file mode 100644 index 0000000000..7744f0ebd6 --- /dev/null +++ b/proxygen/lib/services/Acceptor.cpp @@ -0,0 +1,440 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/services/Acceptor.h" + +#include "proxygen/lib/ssl/SSLContextManager.h" +#include "proxygen/lib/ssl/SSLStats.h" +#include "proxygen/lib/ssl/SSLUtil.h" +#include "proxygen/lib/utils/Time.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using apache::thrift::async::TAsyncSSLSocket; +using apache::thrift::async::TAsyncServerSocket; +using apache::thrift::async::TAsyncSocket; +using apache::thrift::async::TAsyncTimeoutSet; +using apache::thrift::transport::SSLContext; +using apache::thrift::transport::SSLContext; +using apache::thrift::transport::TTransportException; +using folly::EventBase; +using folly::SocketAddress; +using folly::wangle::ConnectionManager; +using folly::wangle::ManagedConnection; +using std::chrono::microseconds; +using std::chrono::milliseconds; +using std::filebuf; +using std::ifstream; +using std::ios; +using std::shared_ptr; +using std::string; + +namespace proxygen { + +DEFINE_int32(shutdown_idle_grace_ms, 5000, "milliseconds to wait before " + "closing idle conns"); + +static const std::string empty_string; +std::atomic Acceptor::totalNumPendingSSLConns_{0}; + +/** + * Lightweight wrapper class to keep track of a newly + * accepted connection during SSL handshaking. + */ +class AcceptorHandshakeHelper : + public TAsyncSSLSocket::HandshakeCallback, + public ManagedConnection { + public: + AcceptorHandshakeHelper(TAsyncSSLSocket::UniquePtr socket, + Acceptor* acceptor, + const SocketAddress& clientAddr, + TimePoint acceptTime) + : socket_(std::move(socket)), acceptor_(acceptor), + acceptTime_(acceptTime), clientAddr_(clientAddr) { + acceptor_->downstreamConnectionManager_->addConnection(this, true); + if(acceptor_->parseClientHello_) { + socket_->enableClientHelloParsing(); + } + socket_->sslAccept(this); + } + + virtual void timeoutExpired() noexcept { + VLOG(4) << "SSL handshake timeout expired"; + sslError_ = SSLErrorEnum::TIMEOUT; + dropConnection(); + } + virtual void describe(std::ostream& os) const { + os << "pending handshake on " << clientAddr_; + } + virtual bool isBusy() const { + return true; + } + virtual void notifyPendingShutdown() {} + virtual void closeWhenIdle() {} + + virtual void dropConnection() { + VLOG(10) << "Dropping in progress handshake for " << clientAddr_; + socket_->closeNow(); + } + virtual void dumpConnectionState(uint8_t loglevel) { + } + + private: + // TAsyncSSLSocket::HandshakeCallback API + virtual void handshakeSuccess(TAsyncSSLSocket* sock) noexcept { + + const unsigned char* nextProto = nullptr; + unsigned nextProtoLength = 0; + sock->getSelectedNextProtocol(&nextProto, &nextProtoLength); + if (VLOG_IS_ON(3)) { + if (nextProto) { + VLOG(3) << "Client selected next protocol " << + string((const char*)nextProto, nextProtoLength); + } else { + VLOG(3) << "Client did not select a next protocol"; + } + } + + // fill in SSL-related fields from TransportInfo + // the other fields like RTT are filled in the Acceptor + TransportInfo tinfo; + tinfo.ssl = true; + tinfo.acceptTime = acceptTime_; + tinfo.sslSetupTime = millisecondsSince(acceptTime_); + tinfo.sslSetupBytesRead = sock->getRawBytesReceived(); + tinfo.sslSetupBytesWritten = sock->getRawBytesWritten(); + tinfo.sslServerName = sock->getSSLServerName(); + tinfo.sslCipher = sock->getNegotiatedCipherName(); + tinfo.sslVersion = sock->getSSLVersion(); + tinfo.sslCertSize = sock->getSSLCertSize(); + tinfo.sslResume = SSLUtil::getResumeState(sock); + sock->getSSLClientCiphers(tinfo.sslClientCiphers); + sock->getSSLServerCiphers(tinfo.sslServerCiphers); + tinfo.sslClientComprMethods = sock->getSSLClientComprMethods(); + tinfo.sslClientExts = sock->getSSLClientExts(); + tinfo.sslNextProtocol.assign( + reinterpret_cast(nextProto), + nextProtoLength); + + acceptor_->updateSSLStats(sock, tinfo.sslSetupTime, SSLErrorEnum::NO_ERROR); + acceptor_->downstreamConnectionManager_->removeConnection(this); + acceptor_->sslConnectionReady(std::move(socket_), clientAddr_, + nextProto ? string((const char*)nextProto, nextProtoLength) : + empty_string, tinfo); + delete this; + } + + virtual void handshakeError(TAsyncSSLSocket* sock, + const TTransportException& ex) noexcept { + auto elapsedTime = millisecondsSince(acceptTime_); + VLOG(3) << "SSL handshake error after " << elapsedTime.count() << + " ms; " << sock->getRawBytesReceived() << " bytes received & " << + sock->getRawBytesWritten() << " bytes sent: " << + ex.what(); + acceptor_->updateSSLStats(sock, elapsedTime, sslError_); + acceptor_->sslConnectionError(); + delete this; + } + + TAsyncSSLSocket::UniquePtr socket_; + Acceptor* acceptor_; + TimePoint acceptTime_; + SocketAddress clientAddr_; + SSLErrorEnum sslError_{SSLErrorEnum::NO_ERROR}; +}; + +Acceptor::Acceptor(const ServerSocketConfig& accConfig) : + accConfig_(accConfig), + socketOptions_(accConfig.getSocketOptions()) { +} + +void +Acceptor::init(TAsyncServerSocket* serverSocket, + EventBase* eventBase) { + CHECK(nullptr == this->base_); + + if (accConfig_.isSSL()) { + if (!sslCtxManager_) { + sslCtxManager_ = folly::make_unique( + eventBase, + "vip_" + getName(), + accConfig_.strictSSL, nullptr); + } + for (const auto& sslCtxConfig : accConfig_.sslContextConfigs) { + sslCtxManager_->addSSLContextConfig( + sslCtxConfig, + accConfig_.sslCacheOptions, + &accConfig_.initialTicketSeeds, + accConfig_.bindAddress, + cacheProvider_); + parseClientHello_ |= sslCtxConfig.clientHelloParsingEnabled; + } + + CHECK(sslCtxManager_->getDefaultSSLCtx()); + } + + base_ = eventBase; + state_ = State::kRunning; + downstreamConnectionManager_ = ConnectionManager::makeUnique( + eventBase, accConfig_.connectionIdleTimeout, this); + + serverSocket->addAcceptCallback(this, eventBase); + // SO_KEEPALIVE is the only setting that is inherited by accepted + // connections so only apply this setting + for (const auto& option: socketOptions_) { + if (option.first.level == SOL_SOCKET && + option.first.optname == SO_KEEPALIVE && option.second == 1) { + serverSocket->setKeepAliveEnabled(true); + break; + } + } +} + +Acceptor::~Acceptor(void) { +} + +void Acceptor::addSSLContextConfig(const SSLContextConfig& sslCtxConfig) { + sslCtxManager_->addSSLContextConfig(sslCtxConfig, + accConfig_.sslCacheOptions, + &accConfig_.initialTicketSeeds, + accConfig_.bindAddress, + cacheProvider_); +} + +void +Acceptor::drainAllConnections() { + if (downstreamConnectionManager_) { + downstreamConnectionManager_->initiateGracefulShutdown( + std::chrono::milliseconds(FLAGS_shutdown_idle_grace_ms)); + } +} + +void Acceptor::setLoadShedConfig(const LoadShedConfiguration& from, + IConnectionCounter* counter) { + loadShedConfig_ = from; + connectionCounter_ = counter; +} + +bool Acceptor::canAccept(const SocketAddress& address) { + if (!connectionCounter_) { + return true; + } + + uint64_t maxConnections = connectionCounter_->getMaxConnections(); + if (maxConnections == 0) { + return true; + } + + uint64_t currentConnections = connectionCounter_->getNumConnections(); + if (currentConnections < maxConnections) { + return true; + } + + if (loadShedConfig_.isWhitelisted(address)) { + return true; + } + + // Take care of comparing connection count against max connections across + // all acceptors. Expensive since a lock must be taken to get the counter. + auto connectionCountForLoadShedding = getConnectionCountForLoadShedding(); + if (connectionCountForLoadShedding < loadShedConfig_.getMaxConnections()) { + return true; + } + + VLOG(4) << address.describe() << " not whitelisted"; + return false; +} + +void +Acceptor::connectionAccepted( + int fd, const SocketAddress& clientAddr) noexcept { + if (!canAccept(clientAddr)) { + close(fd); + return; + } + auto acceptTime = getCurrentTime(); + for (const auto& opt: socketOptions_) { + opt.first.apply(fd, opt.second); + } + + onDoneAcceptingConnection(fd, clientAddr, acceptTime); +} + +void Acceptor::onDoneAcceptingConnection( + int fd, + const SocketAddress& clientAddr, + TimePoint acceptTime) noexcept { + processEstablishedConnection(fd, clientAddr, acceptTime); +} + +void +Acceptor::processEstablishedConnection( + int fd, + const SocketAddress& clientAddr, + TimePoint acceptTime) noexcept { + if (accConfig_.isSSL()) { + CHECK(sslCtxManager_); + TAsyncSSLSocket::UniquePtr sslSock( + new TAsyncSSLSocket(sslCtxManager_->getDefaultSSLCtx(), base_, fd)); + ++numPendingSSLConns_; + ++totalNumPendingSSLConns_; + if (totalNumPendingSSLConns_ > accConfig_.maxConcurrentSSLHandshakes) { + VLOG(2) << "dropped SSL handshake on " << accConfig_.name << + " too many handshakes in progress"; + updateSSLStats(sslSock.get(), std::chrono::milliseconds(0), + SSLErrorEnum::DROPPED); + sslConnectionError(); + return; + } + new AcceptorHandshakeHelper( + std::move(sslSock), this, clientAddr, acceptTime); + } else { + TransportInfo tinfo; + tinfo.ssl = false; + tinfo.acceptTime = acceptTime; + TAsyncSocket::UniquePtr sock(new TAsyncSocket(base_, fd)); + connectionReady(std::move(sock), clientAddr, empty_string, tinfo); + } +} + +void +Acceptor::connectionReady( + TAsyncSocket::UniquePtr sock, + const SocketAddress& clientAddr, + const string& nextProtocolName, + TransportInfo& tinfo) { + // Limit the number of reads from the socket per poll loop iteration, + // both to keep memory usage under control and to prevent one fast- + // writing client from starving other connections. + sock->setMaxReadsPerEvent(16); + tinfo.initWithSocket(sock.get()); + onNewConnection(std::move(sock), &clientAddr, nextProtocolName, tinfo); +} + +void +Acceptor::sslConnectionReady(TAsyncSocket::UniquePtr sock, + const SocketAddress& clientAddr, + const string& nextProtocol, + TransportInfo& tinfo) { + CHECK(numPendingSSLConns_ > 0); + connectionReady(std::move(sock), clientAddr, nextProtocol, tinfo); + --numPendingSSLConns_; + --totalNumPendingSSLConns_; + if (state_ == State::kDraining) { + checkDrained(); + } +} + +void +Acceptor::sslConnectionError() { + CHECK(numPendingSSLConns_ > 0); + --numPendingSSLConns_; + --totalNumPendingSSLConns_; + if (state_ == State::kDraining) { + checkDrained(); + } +} + +void +Acceptor::acceptError(const std::exception& ex) noexcept { + // An error occurred. + // The most likely error is out of FDs. TAsyncServerSocket will back off + // briefly if we are out of FDs, then continue accepting later. + // Just log a message here. + LOG(ERROR) << "error accepting on acceptor socket: " << ex.what(); +} + +void +Acceptor::acceptStopped() noexcept { + VLOG(3) << "Acceptor " << this << " acceptStopped()"; + // Drain the open client connections + drainAllConnections(); + + // If we haven't yet finished draining, begin doing so by marking ourselves + // as in the draining state. We must be sure to hit checkDrained() here, as + // if we're completely idle, we can should consider ourself drained + // immediately (as there is no outstanding work to complete to cause us to + // re-evaluate this). + if (state_ != State::kDone) { + state_ = State::kDraining; + checkDrained(); + } +} + +void +Acceptor::onEmpty(const ConnectionManager& cm) { + VLOG(3) << "Acceptor=" << this << " onEmpty()"; + if (state_ == State::kDraining) { + checkDrained(); + } +} + +void +Acceptor::checkDrained() { + CHECK(state_ == State::kDraining); + if (forceShutdownInProgress_ || + (downstreamConnectionManager_->getNumConnections() != 0) || + (numPendingSSLConns_ != 0)) { + return; + } + + VLOG(2) << "All connections drained from Acceptor=" << this << " in thread " + << base_; + + downstreamConnectionManager_.reset(); + + state_ = State::kDone; + + onConnectionsDrained(); +} + +milliseconds +Acceptor::getConnTimeout() const { + return accConfig_.connectionIdleTimeout; +} + +void Acceptor::addConnection(ManagedConnection* conn) { + // Add the socket to the timeout manager so that it can be cleaned + // up after being left idle for a long time. + downstreamConnectionManager_->addConnection(conn, true); +} + +void +Acceptor::forceStop() { + base_->runInEventBaseThread([&] { dropAllConnections(); }); +} + +void +Acceptor::dropAllConnections() { + if (downstreamConnectionManager_) { + LOG(INFO) << "Dropping all connections from Acceptor=" << this << + " in thread " << base_; + assert(base_->isInEventBaseThread()); + forceShutdownInProgress_ = true; + downstreamConnectionManager_->dropAllConnections(); + CHECK(downstreamConnectionManager_->getNumConnections() == 0); + downstreamConnectionManager_.reset(); + } + CHECK(numPendingSSLConns_ == 0); + + state_ = State::kDone; + onConnectionsDrained(); +} + +} // proxygen diff --git a/proxygen/lib/services/Acceptor.h b/proxygen/lib/services/Acceptor.h new file mode 100644 index 0000000000..3aeeed43c1 --- /dev/null +++ b/proxygen/lib/services/Acceptor.h @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/services/ConnectionCounter.h" +#include "proxygen/lib/services/LoadShedConfiguration.h" +#include "proxygen/lib/services/ServerSocketConfig.h" +#include "proxygen/lib/services/TransportInfo.h" +#include "proxygen/lib/ssl/SSLCacheProvider.h" + +#include +#include +#include +#include +#include +#include + +namespace folly { +class SocketAddress; +class SSLContext; +} + +namespace apache { namespace thrift { +namespace async { +class TAsyncTransport; +} +}} + +namespace folly { namespace wangle { +class ManagedConnection; +}} + +namespace facebook { namespace stats { +class ExportedStatMap; +}} + +namespace proxygen { + +class SSLContextManager; + +/** + * An abstract acceptor for TCP-based network services. + * + * There is one acceptor object per thread for each listening socket. When a + * new connection arrives on the listening socket, it is accepted by one of the + * acceptor objects. From that point on the connection will be processed by + * that acceptor's thread. + * + * The acceptor will call the abstract onNewConnection() method to create + * a new ManagedConnection object for each accepted socket. The acceptor + * also tracks all outstanding connections that it has accepted. + */ +class Acceptor : + public apache::thrift::async::TAsyncServerSocket::AcceptCallback, + public folly::wangle::ConnectionManager::Callback { + public: + + enum class State : uint32_t { + kInit, // not yet started + kRunning, // processing requests normally + kDraining, // processing outstanding conns, but not accepting new ones + kDone, // no longer accepting, and all connections finished + }; + + explicit Acceptor(const ServerSocketConfig& accConfig); + virtual ~Acceptor(); + + /** + * Supply an SSL cache provider + * @note Call this before init() + */ + virtual void setSSLCacheProvider( + const std::shared_ptr& cacheProvider) { + cacheProvider_ = cacheProvider; + } + + /** + * Initialize the Acceptor to run in the specified EventBase + * thread, receiving connections from the specified TAsyncServerSocket. + * + * This method will be called from the TAsyncServerSocket's primary thread, + * not the specified EventBase thread. + */ + virtual void init(apache::thrift::async::TAsyncServerSocket* serverSocket, + folly::EventBase* eventBase); + + /** + * Dynamically add a new SSLContextConfig + */ + void addSSLContextConfig(const SSLContextConfig& sslCtxConfig); + + SSLContextManager* getSSLContextManager() const { + return sslCtxManager_.get(); + } + + /** + * Return the number of outstanding connections in this service instance. + */ + uint32_t getNumConnections() const { + return downstreamConnectionManager_ ? + downstreamConnectionManager_->getNumConnections() : 0; + } + + /** + * Access the Acceptor's event base. + */ + folly::EventBase* getEventBase() { return base_; } + + /** + * Access the Acceptor's downstream (client-side) ConnectionManager + */ + virtual folly::wangle::ConnectionManager* getConnectionManager() { + return downstreamConnectionManager_.get(); + } + + /** + * Invoked when a new ManagedConnection is created. + * + * This allows the Acceptor to track the outstanding connections, + * for tracking timeouts and for ensuring that all connections have been + * drained on shutdown. + */ + void addConnection(folly::wangle::ManagedConnection* connection); + + /** + * Get this acceptor's current state. + */ + State getState() const { + return state_; + } + + /** + * Get the current connection timeout. + */ + std::chrono::milliseconds getConnTimeout() const; + + /** + * Returns the name of this VIP. + * + * Will return an empty string if no name has been configured. + */ + const std::string& getName() const { + return accConfig_.name; + } + + /** + * Force the acceptor to drop all connections and stop processing. + * + * This function may be called from any thread. The acceptor will not + * necessarily stop before this function returns: the stop will be scheduled + * to run in the acceptor's thread. + */ + virtual void forceStop(); + + bool isSSL() const { return accConfig_.isSSL(); } + + const ServerSocketConfig& getConfig() const { return accConfig_; } + + static uint64_t getTotalNumPendingSSLConns() { + return totalNumPendingSSLConns_.load(); + } + + /** + * Called right when the TCP connection has been accepted, before processing + * the first HTTP bytes (HTTP) or the SSL handshake (HTTPS) + */ + virtual void onDoneAcceptingConnection( + int fd, + const folly::SocketAddress& clientAddr, + TimePoint acceptTime + ) noexcept; + + /** + * Begins either processing HTTP bytes (HTTP) or the SSL handshake (HTTPS) + */ + void processEstablishedConnection( + int fd, + const folly::SocketAddress& clientAddr, + TimePoint acceptTime + ) noexcept; + + protected: + friend class AcceptorHandshakeHelper; + + /** + * Our event loop. + * + * Probably needs to be used to pass to a ManagedConnection + * implementation. Also visible in case a subclass wishes to do additional + * things w/ the event loop (e.g. in attach()). + */ + folly::EventBase* base_{nullptr}; + + virtual uint64_t getConnectionCountForLoadShedding(void) const { return 0; } + + /** + * Hook for subclasses to drop newly accepted connections prior + * to handshaking. + */ + virtual bool canAccept(const folly::SocketAddress&); + + /** + * Invoked when a new connection is created. This is where application starts + * processing a new downstream connection. + * + * NOTE: Application should add the new connection to + * downstreamConnectionManager so that it can be garbage collected after + * certain period of idleness. + * + * @param sock the socket connected to the client + * @param address the address of the client + * @param nextProtocolName the name of the L6 or L7 protocol to be + * spoken on the connection, if known (e.g., + * from TLS NPN during secure connection setup), + * or an empty string if unknown + */ + virtual void onNewConnection( + apache::thrift::async::TAsyncSocket::UniquePtr sock, + const folly::SocketAddress* address, + const std::string& nextProtocolName, + const TransportInfo& tinfo) = 0; + + /** + * Hook for subclasses to record stats about SSL connection establishment. + */ + virtual void updateSSLStats( + const apache::thrift::async::TAsyncSSLSocket* sock, + std::chrono::milliseconds acceptLatency, + SSLErrorEnum error) noexcept {} + + /** + * Drop all connections. + * + * forceStop() schedules dropAllConnections() to be called in the acceptor's + * thread. + */ + void dropAllConnections(); + + /** + * Drains all open connections of their outstanding transactions. When + * a connection's transaction count reaches zero, the connection closes. + */ + void drainAllConnections(); + + /** + * onConnectionsDrained() will be called once all connections have been + * drained while the acceptor is stopping. + * + * Subclasses can override this method to perform any subclass-specific + * cleanup. + */ + virtual void onConnectionsDrained() {} + + // TAsyncServerSocket::AcceptCallback methods + void connectionAccepted(int fd, + const folly::SocketAddress& clientAddr) + noexcept; + void acceptError(const std::exception& ex) noexcept; + void acceptStopped() noexcept; + + // ConnectionManager::Callback methods + void onEmpty(const folly::wangle::ConnectionManager& cm); + void onConnectionAdded(const folly::wangle::ConnectionManager& cm) {} + void onConnectionRemoved(const folly::wangle::ConnectionManager& cm) {} + + /** + * Process a connection that is to ready to receive L7 traffic. + * This method is called immediately upon accept for plaintext + * connections and upon completion of SSL handshaking or resumption + * for SSL connections. + */ + void connectionReady( + apache::thrift::async::TAsyncSocket::UniquePtr sock, + const folly::SocketAddress& clientAddr, + const std::string& nextProtocolName, + TransportInfo& tinfo); + + const LoadShedConfiguration& getLoadShedConfiguration() const { + return loadShedConfig_; + } + + protected: + const ServerSocketConfig accConfig_; + void setLoadShedConfig(const LoadShedConfiguration& from, + IConnectionCounter* counter); + + /** + * Socket options to apply to the client socket + */ + apache::thrift::async::TAsyncSocket::OptionMap socketOptions_; + + std::unique_ptr sslCtxManager_; + + /** + * Whether we want to enable client hello parsing in the handshake helper + * to get list of supported client ciphers. + */ + bool parseClientHello_{false}; + + folly::wangle::ConnectionManager::UniquePtr downstreamConnectionManager_; + + private: + + // Forbidden copy constructor and assignment opererator + Acceptor(Acceptor const &) = delete; + Acceptor& operator=(Acceptor const &) = delete; + + /** + * Wrapper for connectionReady() that decrements the count of + * pending SSL connections. + */ + void sslConnectionReady(apache::thrift::async::TAsyncSocket::UniquePtr sock, + const folly::SocketAddress& clientAddr, + const std::string& nextProtocol, + TransportInfo& tinfo); + + /** + * Notification callback for SSL handshake failures. + */ + void sslConnectionError(); + + void checkDrained(); + + State state_{State::kInit}; + uint64_t numPendingSSLConns_{0}; + + static std::atomic totalNumPendingSSLConns_; + + bool forceShutdownInProgress_{false}; + LoadShedConfiguration loadShedConfig_; + IConnectionCounter* connectionCounter_{nullptr}; + std::shared_ptr cacheProvider_; +}; + +} // proxygen diff --git a/proxygen/lib/services/AcceptorConfiguration.h b/proxygen/lib/services/AcceptorConfiguration.h new file mode 100644 index 0000000000..fd83ff6110 --- /dev/null +++ b/proxygen/lib/services/AcceptorConfiguration.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/services/ServerSocketConfig.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace proxygen { + +/** + * Configuration for a single Acceptor. + * + * This configures not only accept behavior, but also some types of SSL + * behavior that may make sense to configure on a per-VIP basis (e.g. which + * cert(s) we use, etc). + */ +struct AcceptorConfiguration : public ServerSocketConfig { + /** + * Determines if the VIP should accept traffic from only internal or + * external clients. Internal VIPs have different behavior + * (e.g. Via headers, etc). + */ + bool internal{false}; + + /** + * The number of milliseconds a transaction can be idle before we close it. + */ + std::chrono::milliseconds transactionIdleTimeout{600000}; + + /** + * The compression level to use for SPDY headers with responses from + * this Acceptor. + */ + int spdyCompressionLevel{Z_NO_COMPRESSION}; + + /** + * The name of the protocol to use on non-TLS connections. + */ + std::string plaintextProtocol; + +}; + +} // proxygen diff --git a/proxygen/lib/services/ConnectionCounter.h b/proxygen/lib/services/ConnectionCounter.h new file mode 100644 index 0000000000..176390a918 --- /dev/null +++ b/proxygen/lib/services/ConnectionCounter.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +class IConnectionCounter { + public: + virtual uint64_t getNumConnections() const = 0; + + /** + * Get the maximum number of non-whitelisted client-side connections + * across all Acceptors managed by this. A value + * of zero means "unlimited." + */ + virtual uint64_t getMaxConnections() const = 0; + + /** + * Increment the count of client-side connections. + */ + virtual void onConnectionAdded() = 0; + + /** + * Decrement the count of client-side connections. + */ + virtual void onConnectionRemoved() = 0; + virtual ~IConnectionCounter() {} +}; + +class SimpleConnectionCounter: public IConnectionCounter { + public: + uint64_t getNumConnections() const override { return numConnections_; } + uint64_t getMaxConnections() const override { return maxConnections_; } + void setMaxConnections(uint64_t maxConnections) { + maxConnections_ = maxConnections; + } + + void onConnectionAdded() override { numConnections_++; } + void onConnectionRemoved() override { numConnections_--; } + virtual ~SimpleConnectionCounter() {} + + protected: + uint64_t maxConnections_{0}; + uint64_t numConnections_{0}; +}; + +} diff --git a/proxygen/lib/services/HTTPAcceptor.h b/proxygen/lib/services/HTTPAcceptor.h new file mode 100644 index 0000000000..536fe168af --- /dev/null +++ b/proxygen/lib/services/HTTPAcceptor.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/services/Acceptor.h" +#include "proxygen/lib/services/AcceptorConfiguration.h" + +namespace proxygen { + +class HTTPAcceptor : public Acceptor { + public: + explicit HTTPAcceptor(const AcceptorConfiguration& accConfig) + : Acceptor(accConfig) + , accConfig_(accConfig) {} + + /** + * Returns true if this server is internal to facebook + */ + bool isInternal() const { + return accConfig_.internal; + } + + /** + * Access the general-purpose timeout manager for transactions. + */ + virtual apache::thrift::async::TAsyncTimeoutSet* getTransactionTimeoutSet() { + return transactionTimeouts_.get(); + } + + virtual void init(apache::thrift::async::TAsyncServerSocket* serverSocket, + folly::EventBase* eventBase) { + Acceptor::init(serverSocket, eventBase); + transactionTimeouts_.reset(new apache::thrift::async::TAsyncTimeoutSet( + eventBase, accConfig_.transactionIdleTimeout)); + + } + + const AcceptorConfiguration& getConfig() const { return accConfig_; } + + protected: + AcceptorConfiguration accConfig_; + private: + apache::thrift::async::TAsyncTimeoutSet::UniquePtr transactionTimeouts_; + apache::thrift::async::TAsyncTimeoutSet::UniquePtr tcpEventsTimeouts_; +}; + +} diff --git a/proxygen/lib/services/LoadShedConfiguration.cpp b/proxygen/lib/services/LoadShedConfiguration.cpp new file mode 100644 index 0000000000..ffc3eb9ada --- /dev/null +++ b/proxygen/lib/services/LoadShedConfiguration.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/services/LoadShedConfiguration.h" + +#include +#include + +using folly::SocketAddress; +using std::string; + +namespace proxygen { + +void LoadShedConfiguration::addWhitelistAddr(folly::StringPiece input) { + auto addr = input.str(); + size_t separator = addr.find_first_of('/'); + if (separator == string::npos) { + whitelistAddrs_.insert(SocketAddress(addr, 0)); + } else { + unsigned prefixLen = folly::to(addr.substr(separator + 1)); + addr.erase(separator); + whitelistNetworks_.insert( + NetworkAddress(SocketAddress(addr, 0), prefixLen)); + } +} + +bool LoadShedConfiguration::isWhitelisted(const SocketAddress& address) const { + if (whitelistAddrs_.find(address) != whitelistAddrs_.end()) { + return true; + } + for (auto& network : whitelistNetworks_) { + if (network.contains(address)) { + return true; + } + } + return false; +} + +} diff --git a/proxygen/lib/services/LoadShedConfiguration.h b/proxygen/lib/services/LoadShedConfiguration.h new file mode 100644 index 0000000000..fb101a20ec --- /dev/null +++ b/proxygen/lib/services/LoadShedConfiguration.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/services/NetworkAddress.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace proxygen { + +/** + * Class that holds an LoadShed configuration for a service + */ +class LoadShedConfiguration { + public: + + // Comparison function for SocketAddress that disregards the port + struct AddressOnlyCompare { + bool operator()( + const folly::SocketAddress& addr1, + const folly::SocketAddress& addr2) const { + return addr1.getIPAddress() < addr2.getIPAddress(); + } + }; + + typedef std::set AddressSet; + typedef std::set NetworkSet; + + LoadShedConfiguration() {} + + ~LoadShedConfiguration() {} + + void addWhitelistAddr(folly::StringPiece); + + /** + * Set/get the set of IPs that should be whitelisted through even when we're + * trying to shed load. + */ + void setWhitelistAddrs(const AddressSet& addrs) { whitelistAddrs_ = addrs; } + const AddressSet& getWhitelistAddrs() const { return whitelistAddrs_; } + + /** + * Set/get the set of networks that should be whitelisted through even + * when we're trying to shed load. + */ + void setWhitelistNetworks(const NetworkSet& networks) { + whitelistNetworks_ = networks; + } + const NetworkSet& getWhitelistNetworks() const { return whitelistNetworks_; } + + /** + * Set/get the maximum number of downstream connections across all VIPs. + */ + void setMaxConnections(uint64_t maxConns) { maxConnections_ = maxConns; } + uint64_t getMaxConnections() const { return maxConnections_; } + + /** + * Set/get the maximum cpu usage. + */ + void setMaxMemUsage(double max) { + CHECK(max >= 0); + CHECK(max <= 1); + maxMemUsage_ = max; + } + double getMaxMemUsage() const { return maxMemUsage_; } + + /** + * Set/get the maximum memory usage. + */ + void setMaxCpuUsage(double max) { + CHECK(max >= 0); + CHECK(max <= 1); + maxCpuUsage_ = max; + } + double getMaxCpuUsage() const { return maxCpuUsage_; } + + void setLoadUpdatePeriod(std::chrono::milliseconds period) { + period_ = period; + } + std::chrono::milliseconds getLoadUpdatePeriod() const { return period_; } + + bool isWhitelisted( + const folly::SocketAddress& addr) const; + + private: + + AddressSet whitelistAddrs_; + NetworkSet whitelistNetworks_; + uint64_t maxConnections_{0}; + double maxMemUsage_; + double maxCpuUsage_; + std::chrono::milliseconds period_; +}; + +} diff --git a/proxygen/lib/services/Makefile.am b/proxygen/lib/services/Makefile.am new file mode 100644 index 0000000000..65f17e64a9 --- /dev/null +++ b/proxygen/lib/services/Makefile.am @@ -0,0 +1,29 @@ +SUBDIRS = . + +noinst_LTLIBRARIES = libproxygenservices.la + +libproxygenservicesdir = $(includedir)/proxygen/lib/services +nobase_libproxygenservices_HEADERS = \ + Acceptor.h \ + AcceptorConfiguration.h \ + ConnectionCounter.h \ + HTTPAcceptor.h \ + LoadShedConfiguration.h \ + NetworkAddress.h \ + RequestWorker.h \ + ServerSocketConfig.h \ + Service.h \ + ServiceConfiguration.h \ + ServiceWorker.h \ + TransportInfo.h \ + WorkerThread.h + +libproxygenservices_la_SOURCES = \ + Acceptor.cpp \ + LoadShedConfiguration.cpp \ + RequestWorker.cpp \ + Service.cpp \ + TransportInfo.cpp \ + WorkerThread.cpp + +libproxygenservices_la_LIBADD = ../ssl/libproxygenssl.la diff --git a/proxygen/lib/services/NetworkAddress.h b/proxygen/lib/services/NetworkAddress.h new file mode 100644 index 0000000000..02c7aa4eb8 --- /dev/null +++ b/proxygen/lib/services/NetworkAddress.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +/** + * A simple wrapper around SocketAddress that represents + * a network in CIDR notation + */ +class NetworkAddress { +public: + /** + * Create a NetworkAddress for an addr/prefixLen + * @param addr IPv4 or IPv6 address of the network + * @param prefixLen Prefix length, in bits + */ + NetworkAddress(const folly::SocketAddress& addr, + unsigned prefixLen): + addr_(addr), prefixLen_(prefixLen) {} + + /** Get the network address */ + const folly::SocketAddress& getAddress() const { + return addr_; + } + + /** Get the prefix length in bits */ + unsigned getPrefixLength() const { return prefixLen_; } + + /** Check whether a given address lies within the network */ + bool contains(const folly::SocketAddress& addr) const { + return addr_.prefixMatch(addr, prefixLen_); + } + + /** Comparison operator to enable use in ordered collections */ + bool operator<(const NetworkAddress& other) const { + if (addr_ < other.addr_) { + return true; + } else if (other.addr_ < addr_) { + return false; + } else { + return (prefixLen_ < other.prefixLen_); + } + } + +private: + folly::SocketAddress addr_; + unsigned prefixLen_; +}; + +} // proxygen diff --git a/proxygen/lib/services/RequestWorker.cpp b/proxygen/lib/services/RequestWorker.cpp new file mode 100644 index 0000000000..a6cb5e9be7 --- /dev/null +++ b/proxygen/lib/services/RequestWorker.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/services/RequestWorker.h" + +#include "proxygen/lib/services/ServiceWorker.h" + +#include + +namespace proxygen { + +RequestWorker::RequestWorker(FinishCallback& callback, uint8_t threadId) + : WorkerThread(folly::EventBaseManager::get()), + nextRequestId_(static_cast(threadId) << 56), + callback_(callback) { +} + +uint64_t RequestWorker::nextRequestId() { + return getRequestWorker()->nextRequestId_++; +} + +void RequestWorker::flushStats() { + CHECK(getEventBase()->isInEventBaseThread()); + FOR_EACH_KV (service, worker, serviceWorkers_) { + worker->flushStats(); + } +} + +void RequestWorker::cleanup() { + WorkerThread::cleanup(); + callback_.workerFinished(this); +} + +} // proxygen diff --git a/proxygen/lib/services/RequestWorker.h b/proxygen/lib/services/RequestWorker.h new file mode 100644 index 0000000000..0e2c300fc9 --- /dev/null +++ b/proxygen/lib/services/RequestWorker.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/services/WorkerThread.h" + +#include +#include + +namespace proxygen { + +class Service; +class ServiceWorker; + +/** + * RequestWorker extends WorkerThread, and also contains a list of + * ServiceWorkers running in this thread. + */ +class RequestWorker : public WorkerThread { + public: + class FinishCallback { + public: + virtual ~FinishCallback() noexcept {} + virtual void workerFinished(RequestWorker*) = 0; + }; + + /** + * Create a new RequestWorker. + * + * @param proxygen The object to notify when this worker finishes. + * @param threadId A unique ID for this worker. + */ + RequestWorker(FinishCallback& callback, uint8_t threadId); + + static uint64_t nextRequestId(); + + static RequestWorker* getRequestWorker() { + RequestWorker* self = dynamic_cast( + WorkerThread::getCurrentWorkerThread()); + CHECK_NOTNULL(self); + return self; + } + + /** + * Track the ServiceWorker objects in-use by this worker. + */ + void addServiceWorker(Service* service, ServiceWorker* sw) { + CHECK(serviceWorkers_.find(service) == serviceWorkers_.end()); + serviceWorkers_[service] = sw; + } + + /** + * For a given service, returns the ServiceWorker associated with this + * RequestWorker + */ + ServiceWorker* getServiceWorker(Service* service) const { + auto it = serviceWorkers_.find(service); + CHECK(it != serviceWorkers_.end()); + return it->second; + } + + /** + * Flush any thread-local stats being tracked by our ServiceWorkers. + * + * This must be invoked from within worker's thread. + */ + void flushStats(); + + private: + void cleanup() override; + + // The next request id within this thread. The id has its highest byte set to + // the thread id, so is unique across the process. + uint64_t nextRequestId_; + + // The ServiceWorkers executing in this worker + std::map serviceWorkers_; + + FinishCallback& callback_; +}; + +} diff --git a/proxygen/lib/services/ServerSocketConfig.h b/proxygen/lib/services/ServerSocketConfig.h new file mode 100644 index 0000000000..29e8a53f2a --- /dev/null +++ b/proxygen/lib/services/ServerSocketConfig.h @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/ssl/SSLCacheOptions.h" +#include "proxygen/lib/ssl/SSLContextConfig.h" +#include "proxygen/lib/ssl/SSLUtil.h" +#include "proxygen/lib/ssl/TLSTicketKeySeeds.h" +#include "proxygen/lib/utils/SocketOptions.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace proxygen { + +/** + * Configuration for a single Acceptor. + * + * This configures not only accept behavior, but also some types of SSL + * behavior that may make sense to configure on a per-VIP basis (e.g. which + * cert(s) we use, etc). + */ +struct ServerSocketConfig { + ServerSocketConfig() { + // generate a single random current seed + uint8_t seed[32]; + folly::Random::secureRandom(seed, sizeof(seed)); + initialTicketSeeds.currentSeeds.push_back( + SSLUtil::hexlify(std::string((char *)seed, sizeof(seed)))); + } + + bool isSSL() const { return !(sslContextConfigs.empty()); } + + /** + * Set/get the socket options to apply on all downstream connections. + */ + void setSocketOptions( + const apache::thrift::async::TAsyncSocket::OptionMap& opts) { + socketOptions_ = filterIPSocketOptions(opts, bindAddress.getFamily()); + } + apache::thrift::async::TAsyncSocket::OptionMap& + getSocketOptions() { + return socketOptions_; + } + const apache::thrift::async::TAsyncSocket::OptionMap& + getSocketOptions() const { + return socketOptions_; + } + + bool hasExternalPrivateKey() const { + for (const auto& cfg : sslContextConfigs) { + if (!cfg.isLocalPrivateKey) { + return true; + } + } + return false; + } + + /** + * The name of this acceptor; used for stats/reporting purposes. + */ + std::string name; + + /** + * The depth of the accept queue backlog. + */ + uint32_t acceptBacklog{1024}; + + /** + * The number of milliseconds a connection can be idle before we close it. + */ + std::chrono::milliseconds connectionIdleTimeout{600000}; + + /** + * The address to bind to. + */ + folly::SocketAddress bindAddress; + + /** + * Options for controlling the SSL cache. + */ + SSLCacheOptions sslCacheOptions{std::chrono::seconds(600), 20480, 200}; + + /** + * The initial TLS ticket seeds. + */ + TLSTicketKeySeeds initialTicketSeeds; + + /** + * The configs for all the SSL_CTX for use by this Acceptor. + */ + std::vector sslContextConfigs; + + /** + * Determines if the Acceptor does strict checking when loading the SSL + * contexts. + */ + bool strictSSL{true}; + + /** + * Maximum number of concurrent pending SSL handshakes + */ + uint32_t maxConcurrentSSLHandshakes{30720}; + + private: + apache::thrift::async::TAsyncSocket::OptionMap socketOptions_; +}; + +} // proxygen diff --git a/proxygen/lib/services/Service.cpp b/proxygen/lib/services/Service.cpp new file mode 100644 index 0000000000..1bbbdcdae6 --- /dev/null +++ b/proxygen/lib/services/Service.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/services/Service.h" + +#include "proxygen/lib/services/RequestWorker.h" +#include "proxygen/lib/services/ServiceWorker.h" + +namespace proxygen { + +Service::Service() { +} + +Service::~Service() { +} + +void Service::addServiceWorker(std::unique_ptr worker, + RequestWorker* reqWorker) { + reqWorker->addServiceWorker(this, worker.get()); + workers_.emplace_back(std::move(worker)); +} + +void Service::clearServiceWorkers() { + workers_.clear(); +} + +} // proxygen diff --git a/proxygen/lib/services/Service.h b/proxygen/lib/services/Service.h new file mode 100644 index 0000000000..0434743789 --- /dev/null +++ b/proxygen/lib/services/Service.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +namespace proxygen { + +class ServiceWorker; +class RequestWorker; + +/* + * A Service object represents a network service running in proxygen. + * + * The Service object is primarily a construct used for managing startup and + * shutdown. The RunProxygen() call will accept a list of Service objects, and + * will invoke start() on each service to start them. When shutdown has been + * requested, it will invoke failHealthChecks() on each service followed (after + * some period of time) by stopAccepting(). + * + * All of these methods are invoked from the main thread. + */ +class Service { + public: + Service(); + virtual ~Service(); + + /** + * Start the service. + * + * start() will be invoked from proxygen's main thread, before the worker + * threads have started processing their event loops. + * + * The return value indicates if the service is enabled or not. Return true + * if the service is enabled and was started successfully, and false if the + * service is disabled and is intentionally not started. Throw an exception + * if the service is enabled and is supposed to be running, but an error + * occurred starting it. + */ + virtual void start(folly::EventBase* mainEventBase, + const std::list& workers) = 0; + + /** + * Mark the service as about to stop; invoked from main thread. + * + * This indicates that the service will be told to stop at some later time + * and should continue to service requests but tell the healthchecker that it + * is dying. + */ + virtual void failHealthChecks() {} + + /** + * Stop accepting all new work; invoked from proxygen's main thread. + * + * This should cause the service to stop accepting new work, and begin to + * fully shut down. stop() may return before all work has completed, but it + * should eventually cause all events for this service to be removed from the + * main EventBase and from the worker threads. + */ + virtual void stopAccepting() = 0; + + /** + * Forcibly stop the service. + * + * If the service does not stop on its own after stopAccepting() is called, + * forceStop() will eventually be called to forcibly stop all processing. + * + * (At the moment this isn't pure virtual simply because I haven't had the + * time to update all existing services to implement forceStop(). Proxygen + * will forcibly terminate the event loop even if a service does not stop + * processing when forceStop() is called, so properly implementing + * forceStop() isn't strictly required.) + */ + virtual void forceStop() {} + + /** + * Perform per-thread cleanup. + * + * This method will be called once for each RequestWorker thread, just before + * that thread is about to exit. Note that this method is called from the + * worker thread itself, not from the main thread. + * + * failHealthChecks() and stopAccepting() will always be called in the main + * thread before cleanupWorkerState() is called in any of the worker threads. + * + * forceStop() may be called in the main thread at any point during shutdown. + * (i.e., Some worker threads may already have finished and called + * cleanupWorkerState(). Once forceStop() is invoked, the remaining threads + * will forcibly exit and then call cleanupWorkerState().) + */ + virtual void cleanupWorkerState(RequestWorker* worker) {} + + /** + * Add a new ServiceWorker (subclasses should create one ServiceWorker + * per worker thread) + */ + void addServiceWorker(std::unique_ptr worker, + RequestWorker* reqWorker); + + /** + * List of workers + */ + const std::list>& getServiceWorkers() const { + return workers_; + } + + /** + * Delete all the workers + */ + void clearServiceWorkers(); + + /** + * Start even when config_test_only is set - default to false + */ + virtual bool startWithConfigTest(bool configTestOnly) { + return !configTestOnly; + } + + private: + // Forbidden copy constructor and assignment opererator + Service(Service const &) = delete; + Service& operator=(Service const &) = delete; + + // Workers + std::list> workers_; +}; + +} // proxygen diff --git a/proxygen/lib/services/ServiceConfiguration.h b/proxygen/lib/services/ServiceConfiguration.h new file mode 100644 index 0000000000..49481554ff --- /dev/null +++ b/proxygen/lib/services/ServiceConfiguration.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/services/AcceptorConfiguration.h" + +namespace proxygen { + +/** + * Base class for all the Configuration objects + */ +class ServiceConfiguration { + public: + ServiceConfiguration() + : writeBufferLimit_(4096) +, takeoverEnabled_(false) {} + + virtual ~ServiceConfiguration() {} + + /** + * Set/get the list of acceptors that will be receiving traffic. + */ + void setAcceptors( + const std::list &acceptors) { + acceptors_.clear(); + acceptors_.insert(acceptors_.begin(), acceptors.begin(), acceptors.end()); + } + const std::list &getAcceptors() const { + return acceptors_; + } + + /** + * Set/get the amount of data that we're allowed to buffer in-memory before + * back-pressuring the other end of an HTTP connection. + */ + void setWriteBufferLimit(uint64_t size) { writeBufferLimit_ = size; } + uint64_t getWriteBufferLimit() const { return writeBufferLimit_; } + + /** + * Set/get whether or not we should enable socket takeover + */ + void setTakeoverEnabled(bool enabled) { takeoverEnabled_ = enabled; } + bool takeoverEnabled() const { return takeoverEnabled_; } + + private: + std::list acceptors_; + uint64_t writeBufferLimit_; + bool takeoverEnabled_; +}; + +} diff --git a/proxygen/lib/services/ServiceWorker.h b/proxygen/lib/services/ServiceWorker.h new file mode 100644 index 0000000000..02da03259a --- /dev/null +++ b/proxygen/lib/services/ServiceWorker.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/services/Acceptor.h" +#include "proxygen/lib/services/ConnectionCounter.h" + +#include +#include +#include + +namespace proxygen { + +class Service; +class RequestWorker; + +/** + * ServiceWorker contains all of the per-thread information for a Service. + * + * ServiceWorker contains a pointer back to the single global Service. + * It contains a list of ProxyAcceptor objects for this worker thread, + * one per VIP. + * + * ServiceWorker fits into the Proxygen object hierarchy as follows: + * + * - Service: one instance (globally across the entire program) + * - ServiceWorker: one instance per thread + * - ServiceAcceptor: one instance per VIP per thread + */ +class ServiceWorker { + public: + ServiceWorker(Service* service, RequestWorker* worker) + : service_(service), worker_(worker) { + } + + virtual ~ServiceWorker() {} + + Service* getService() const { + return service_; + } + + void addServiceAcceptor(std::unique_ptr acceptor) { + acceptors_.emplace_back(std::move(acceptor)); + } + + RequestWorker* getRequestWorker() const { + return worker_; + } + + const std::list>& getAcceptors() { + return acceptors_; + } + + // Flush any thread-local stats that the service is tracking + virtual void flushStats() { + } + + // Destruct all the acceptors + void clearAcceptors() { + acceptors_.clear(); + } + + IConnectionCounter* getConnectionCounter() { + return &connectionCounter_; + } + + virtual void forceStop() {} + + protected: + SimpleConnectionCounter connectionCounter_; + + private: + // Forbidden copy constructor and assignment operator + ServiceWorker(ServiceWorker const &) = delete; + ServiceWorker& operator=(ServiceWorker const &) = delete; + + /** + * The global Service object. + */ + Service* service_; + + /** + * The RequestWorker that is actually responsible for running the EventBase + * loop in this thread. + */ + RequestWorker* worker_; + + /** + * A list of the Acceptor objects specific to this worker thread, one + * Acceptor per VIP. + */ + std::list> acceptors_; +}; + +} // proxygen diff --git a/proxygen/lib/services/TransportInfo.cpp b/proxygen/lib/services/TransportInfo.cpp new file mode 100644 index 0000000000..2e9ef426fd --- /dev/null +++ b/proxygen/lib/services/TransportInfo.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/services/TransportInfo.h" + +#include +#include +#include + +using apache::thrift::async::TAsyncSocket; +using std::chrono::microseconds; +using std::map; +using std::string; + +namespace proxygen { + +bool TransportInfo::initWithSocket(const TAsyncSocket* sock) { +#if defined(__linux__) || defined(__FreeBSD__) + if (!TransportInfo::readTcpInfo(&tcpinfo, sock)) { + tcpinfoErrno = errno; + return false; + } + rtt = microseconds(tcpinfo.tcpi_rtt); + validTcpinfo = true; +#else + tcpinfoErrno = EINVAL; + rtt = microseconds(-1); +#endif + return true; +} + +int64_t TransportInfo::readRTT(const TAsyncSocket* sock) { +#if defined(__linux__) || defined(__FreeBSD__) + struct tcp_info tcpinfo; + if (!TransportInfo::readTcpInfo(&tcpinfo, sock)) { + return -1; + } + return tcpinfo.tcpi_rtt; +#else + return -1; +#endif +} + +#if defined(__linux__) || defined(__FreeBSD__) +bool TransportInfo::readTcpInfo(struct tcp_info* tcpinfo, + const TAsyncSocket* sock) { + socklen_t len = sizeof(struct tcp_info); + if (!sock) { + return false; + } + if (getsockopt(sock->getFd(), IPPROTO_TCP, + TCP_INFO, (void*) tcpinfo, &len) < 0) { + VLOG(4) << "Error calling getsockopt(): " << strerror(errno); + return false; + } + return true; +} +#endif + +} // proxygen diff --git a/proxygen/lib/services/TransportInfo.h b/proxygen/lib/services/TransportInfo.h new file mode 100644 index 0000000000..17fe8d027f --- /dev/null +++ b/proxygen/lib/services/TransportInfo.h @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/http/HTTPHeaderSize.h" +#include "proxygen/lib/ssl/SSLUtil.h" +#include "proxygen/lib/utils/Time.h" + +#include +#include +#include + +namespace apache { namespace thrift { namespace async { +class TAsyncSocket; +}}} + +namespace proxygen { + +struct TransportInfo { + /* + * timestamp of when the connection handshake was completed + */ + TimePoint acceptTime{}; + + /* + * connection RTT (Round-Trip Time) + */ + std::chrono::microseconds rtt{0}; + +#if defined(__linux__) || defined(__FreeBSD__) + /* + * TCP information as fetched from getsockopt(2) + */ + tcp_info tcpinfo { +#if __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 17 + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 // 32 +#else + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 // 29 +#endif // __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 17 + }; +#endif // defined(__linux__) || defined(__FreeBSD__) + + /* + * time for setting the connection, from the moment in was accepted until it + * is established. + */ + std::chrono::milliseconds setupTime{0}; + + /* + * time for setting up the SSL connection or SSL handshake + */ + std::chrono::milliseconds sslSetupTime{0}; + + /* + * The name of the SSL ciphersuite used by the transaction's + * transport. Returns null if the transport is not SSL. + */ + const char* sslCipher{nullptr}; + + /* + * The SSL server name used by the transaction's + * transport. Returns null if the transport is not SSL. + */ + const char* sslServerName{nullptr}; + + /* + * list of ciphers sent by the client + */ + std::string sslClientCiphers{}; + + /* + * list of compression methods sent by the client + */ + std::string sslClientComprMethods{}; + + /* + * list of TLS extensions sent by the client + */ + std::string sslClientExts{}; + + /* + * hash of all the SSL parameters sent by the client + */ + std::string sslSignature{}; + + /* + * list of ciphers supported by the server + */ + std::string sslServerCiphers{}; + + /* + * guessed "(os) (browser)" based on SSL Signature + */ + std::string guessedUserAgent{}; + + /** + * The result of SSL NPN negotiation. + */ + std::string sslNextProtocol{}; + + /* + * total number of bytes sent over the connection + */ + int64_t totalBytes{0}; + + /** + * header bytes read + */ + HTTPHeaderSize ingressHeader; + + /* + * header bytes written + */ + HTTPHeaderSize egressHeader; + + /* + * Here is how the timeToXXXByte variables are planned out: + * 1. All timeToXXXByte variables are measuring the ByteEvent from reqStart_ + * 2. You can get the timing between two ByteEvents by calculating their + * differences. For example: + * timeToLastBodyByteAck - timeToFirstByte + * => Total time to deliver the body + * 3. The calculation in point (2) is typically done outside proxygen + * + * Future plan: + * We should log the timestamps (TimePoints) and allow + * the consumer to calculate the latency whenever it + * wants instead of calculating them in proxygen, for the sake of flexibility. + * For example: + * 1. TimePoint reqStartTimestamp; + * 2. TimePoint firstHeaderByteSentTimestamp; + * 3. TimePoint firstBodyByteTimestamp; + * 3. TimePoint lastBodyByteTimestamp; + * 4. TimePoint lastBodyByteAckTimestamp; + */ + + /* + * time to first header byte written to the kernel send buffer + * NOTE: It is not 100% accurate since TAsyncSocket does not do + * do callback on partial write. + */ + int32_t timeToFirstHeaderByte{-1}; + + /* + * time to first body byte written to the kernel send buffer + */ + int32_t timeToFirstByte{-1}; + + /* + * time to last body byte written to the kernel send buffer + */ + int32_t timeToLastByte{-1}; + + /* + * time to TCP Ack received for the last written body byte + */ + int32_t timeToLastBodyByteAck{-1}; + + /* + * time it took the client to ACK the last byte, from the moment when the + * kernel sent the last byte to the client and until it received the ACK + * for that byte + */ + int32_t lastByteAckLatency{-1}; + + /* + * time spent inside proxygen + */ + int32_t proxyLatency{-1}; + + /* + * time between connection accepted and client message headers completed + */ + int32_t clientLatency{-1}; + + /* + * latency for communication with the server + */ + int32_t serverLatency{-1}; + + /* + * time used to get a usable connection. + */ + int32_t connectLatency{-1}; + + /* + * body bytes written + */ + uint32_t egressBodySize{0}; + + /* + * value of errno in case of getsockopt() error + */ + int tcpinfoErrno{0}; + + /* + * bytes read & written during SSL Setup + */ + uint32_t sslSetupBytesWritten{0}; + uint32_t sslSetupBytesRead{0}; + + /** + * SSL error detail + */ + uint32_t sslError{0}; + + /** + * body bytes read + */ + uint32_t ingressBodySize{0}; + + /* + * The SSL version used by the transaction's transport, in + * OpenSSL's format: 4 bits for the major version, followed by 4 bits + * for the minor version. Returns zero for non-SSL. + */ + uint16_t sslVersion{0}; + + /* + * The SSL certificate size. + */ + uint16_t sslCertSize{0}; + + /** + * response status code + */ + uint16_t statusCode{0}; + + /* + * The SSL mode for the transaction's transport: new session, + * resumed session, or neither (non-SSL). + */ + SSLResumeEnum sslResume{SSLResumeEnum::NA}; + + /* + * true if the tcpinfo was successfully read from the kernel + */ + bool validTcpinfo{false}; + + /* + * true if the connection is SSL, false otherwise + */ + bool ssl{false}; + + /* + * get the RTT value in milliseconds + */ + std::chrono::milliseconds getRttMs() const { + return std::chrono::duration_cast(rtt); + } + + /* + * initialize the fields related with tcp_info + */ + bool initWithSocket(const apache::thrift::async::TAsyncSocket* sock); + + /* + * Get the kernel's estimate of round-trip time (RTT) to the transport's peer + * in microseconds. Returns -1 on error. + */ + static int64_t readRTT(const apache::thrift::async::TAsyncSocket* sock); + +#if defined(__linux__) || defined(__FreeBSD__) + /* + * perform the getsockopt(2) syscall to fetch TCP info for a given socket + */ + static bool readTcpInfo(struct tcp_info* tcpinfo, + const apache::thrift::async::TAsyncSocket* sock); +#endif +}; + +} // proxygen diff --git a/proxygen/lib/services/WorkerThread.cpp b/proxygen/lib/services/WorkerThread.cpp new file mode 100644 index 0000000000..209b61aea7 --- /dev/null +++ b/proxygen/lib/services/WorkerThread.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/services/WorkerThread.h" + +#include +#include +#include +#include +#include + +namespace proxygen { + +__thread WorkerThread* WorkerThread::currentWorker_ = nullptr; + +std::atomic_uint WorkerThread::objectCounter_; + +WorkerThread::WorkerThread(folly::EventBaseManager* eventBaseManager) + : eventBaseManager_(eventBaseManager) { + //eventBase_.setName(folly::to("WorkerThread", + // objectCounter_.fetch_add(1))); +} + +WorkerThread::~WorkerThread() { + CHECK(state_ == State::IDLE); +} + +void WorkerThread::start() { + CHECK(state_ == State::IDLE); + state_ = State::STARTING; + + auto barrier = std::make_shared(2); + { + // because you could theoretically call wait in parallel with start, + // why are you in such a hurry anyways? + std::lock_guard guard(joinLock_); + thread_ = std::thread([&, barrier]() mutable { + this->setup(); + barrier->wait(); + barrier.reset(); + this->runLoop(); + this->cleanup(); + }); + } + barrier->wait(); + // The server has been set up and is now in the loop implementation +} + +void WorkerThread::stopWhenIdle() { + // Call runInEventBaseThread() to perform all of the work in the actual + // worker thread. + // + // This way we don't have to synchronize access to state_. + eventBase_.runInEventBaseThread([this] { + if (state_ == State::RUNNING) { + state_ = State::STOP_WHEN_IDLE; + eventBase_.terminateLoopSoon(); + } else if (state_ != State::STOP_WHEN_IDLE) { + LOG(FATAL) << "stopWhenIdle() called in unexpected state " << + static_cast(state_); + } + }); +} + +void WorkerThread::forceStop() { + // Call runInEventBaseThread() to perform all of the work in the actual + // worker thread. + // + // This way we don't have to synchronize access to state_. + // + // This also has the benefit of preserving ordering between functions already + // scheduled with runInEventBaseThread() and the actual stop. Functions + // already scheduled before forceStop() was called are guaranteed to be run + // before the thread stops. (If we called terminateLoopSoon() from the + // current thread, the worker thread may stop before running functions + // already scheduled via runInEventBaseThread().) + eventBase_.runInEventBaseThread([this] { + if (state_ == State::RUNNING || state_ == State::STOP_WHEN_IDLE) { + state_ = State::FORCE_STOP; + eventBase_.terminateLoopSoon(); + } else { + LOG(FATAL) << "forceStop() called in unexpected state " << + static_cast(state_); + } + }); +} + +void WorkerThread::wait() { + std::lock_guard guard(joinLock_); + if (thread_.joinable()) { + thread_.join(); + } +} + +void WorkerThread::setup() { + sigset_t ss; + + // Ignore some signals + sigemptyset(&ss); + sigaddset(&ss, SIGHUP); + sigaddset(&ss, SIGINT); + sigaddset(&ss, SIGQUIT); + sigaddset(&ss, SIGUSR1); + sigaddset(&ss, SIGUSR2); + sigaddset(&ss, SIGPIPE); + sigaddset(&ss, SIGALRM); + sigaddset(&ss, SIGTERM); + sigaddset(&ss, SIGCHLD); + sigaddset(&ss, SIGIO); + PCHECK(pthread_sigmask(SIG_BLOCK, &ss, nullptr) == 0); + + // Update the currentWorker_ thread-local pointer + CHECK(nullptr == currentWorker_); + currentWorker_ = this; + + // Update the manager with the event base this worker runs on + if (eventBaseManager_) { + eventBaseManager_->setEventBase(&eventBase_, false); + } +} + +void WorkerThread::cleanup() { + currentWorker_ = nullptr; + if (eventBaseManager_) { + eventBaseManager_->clearEventBase(); + } +} + +void WorkerThread::runLoop() { + // Update state_ + CHECK(state_ == State::STARTING); + state_ = State::RUNNING; + + VLOG(1) << "WorkerThread " << this << " starting"; + + // Call loopForever(). This will only return after stopWhenIdle() or + // forceStop() has been called. + eventBase_.loopForever(); + + if (state_ == State::STOP_WHEN_IDLE) { + // We have been asked to stop when there are no more events left. + // Call loop() to finish processing events. This will return when there + // are no more events to process, or after forceStop() has been called. + VLOG(1) << "WorkerThread " << this << " finishing non-internal events"; + eventBase_.loop(); + } + + CHECK(state_ == State::STOP_WHEN_IDLE || state_ == State::FORCE_STOP); + state_ = State::IDLE; + + VLOG(1) << "WorkerThread " << this << " terminated"; +} + +} // proxygen diff --git a/proxygen/lib/services/WorkerThread.h b/proxygen/lib/services/WorkerThread.h new file mode 100644 index 0000000000..19cf452907 --- /dev/null +++ b/proxygen/lib/services/WorkerThread.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include +#include +#include + +namespace folly { +class EventBaseManager; +} + +namespace proxygen { + +/** + * A WorkerThread represents an independent event loop that runs in its own + * thread. + */ +class WorkerThread { + public: + explicit WorkerThread(folly::EventBaseManager* ebm); + virtual ~WorkerThread(); + + /** + * Begin execution of the worker. + * + * This starts the worker thread, and returns immediately. + */ + void start(); + + /** + * Request that the worker thread stop when there are no more events to + * process. + * + * Normally each worker thread runs forever, even if it is idle with no + * events to process. This function requests that the worker thread return + * when it becomes idle. + * + * This is used for graceful shutdown: Once the services have been asked to + * shutdown, stopWhenIdle() can be called on the WorkerThread so that it will + * return as soon as the services in this thread no longer have any events to + * process. + * + * Typically you will still want to call forceStop() after a timeout, in case + * some of the services take too long to shut down gracefully. + */ + void stopWhenIdle(); + + /** + * Request that the worker stop executing as soon as possible. + * + * This will terminate the worker thread's event loop, and cause the thread + * to return. If there are any services still running in the worker thread, + * their events will no longer be processed. + * + * This function is asynchronous: it signals the worker thread to stop, and + * returns without waiting for the thread to actually terminate. The wait() + * method must be called to wait for the thread to terminate. + */ + void forceStop(); + + /** + * Synchronously wait for termination of the worker thread. + * + * Note that the worker thread will only terminate after stopWhenIdle() or + * forceStop() has been called, so you typically should only call wait() + * after first using one of these functions. + */ + void wait(); + + /** + * Get the EventBase used to drive the events in this worker thread. + */ + folly::EventBase* getEventBase() { + return &eventBase_; + } + + /** + * Get the current WorkerThread running this thread. + * + * Returns nullptr if called from a thread that is not running inside + * WorkerThread. + */ + static WorkerThread* getCurrentWorkerThread() { + return currentWorker_; + } + + protected: + virtual void setup(); + virtual void cleanup(); + + private: + enum class State : uint8_t { + IDLE, // Not yet started + STARTING, // start() called, thread not fully started yet + RUNNING, // Thread running normally + STOP_WHEN_IDLE, // stopWhenIdle() called, not stopped yet + FORCE_STOP, // forceStop() called, but the loop is still cleaning up + }; + + // Forbidden copy constructor and assignment operator + WorkerThread(WorkerThread const &) = delete; + WorkerThread& operator=(WorkerThread const &) = delete; + + void runLoop(); + + State state_{State::IDLE}; + std::thread thread_; + std::mutex joinLock_; + folly::EventBase eventBase_; + folly::EventBaseManager* eventBaseManager_{nullptr}; + + // A thread-local pointer to the current WorkerThread for this thread + static __thread WorkerThread* currentWorker_; + + // A count of the number of WorkerThreads that have been constructed + static std::atomic_uint objectCounter_; +}; + +} // proxygen diff --git a/proxygen/lib/services/test/AcceptorTest.cpp b/proxygen/lib/services/test/AcceptorTest.cpp new file mode 100644 index 0000000000..20501d8d04 --- /dev/null +++ b/proxygen/lib/services/test/AcceptorTest.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include +#include +#include +#include + +using namespace apache::thrift::async; +using namespace folly; +using namespace proxygen; + +class TestConnection : public folly::wangle::ManagedConnection { + public: + void timeoutExpired() noexcept { + } + void describe(std::ostream& os) const { + } + bool isBusy() const { + return false; + } + void notifyPendingShutdown() { + } + void closeWhenIdle() { + } + void dropConnection() { + } + void dumpConnectionState(uint8_t loglevel) { + } +}; + +class TestAcceptor : public Acceptor { + public: + explicit TestAcceptor(const ServerSocketConfig& accConfig) + : Acceptor(accConfig) {} + + virtual void onNewConnection( + apache::thrift::async::TAsyncSocket::UniquePtr sock, + const folly::SocketAddress* address, + const std::string& nextProtocolName, + const TransportInfo& tinfo) { + addConnection(new TestConnection); + + getEventBase()->terminateLoopSoon(); + } +}; + +TEST(AcceptorTest, Basic) { + + EventBase base; + auto socket = TAsyncServerSocket::newSocket(&base); + ServerSocketConfig config; + + TestAcceptor acceptor(config); + socket->addAcceptCallback(&acceptor, &base); + + acceptor.init(socket.get(), &base); + socket->bind(0); + socket->listen(100); + + SocketAddress addy; + socket->getAddress(&addy); + + socket->startAccepting(); + + auto client_socket = TAsyncSocket::newSocket( + &base, addy); + + base.loopForever(); + + CHECK_EQ(acceptor.getNumConnections(), 1); + + CHECK(acceptor.getState() == Acceptor::State::kRunning); + acceptor.forceStop(); + socket->stopAccepting(); + base.loop(); +} diff --git a/proxygen/lib/ssl/ClientHelloExtStats.h b/proxygen/lib/ssl/ClientHelloExtStats.h new file mode 100644 index 0000000000..d6a9f80c06 --- /dev/null +++ b/proxygen/lib/ssl/ClientHelloExtStats.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +class ClientHelloExtStats { + public: + virtual ~ClientHelloExtStats() noexcept {} + + // client hello + virtual void recordAbsentHostname() noexcept = 0; + virtual void recordMatch() noexcept = 0; + virtual void recordNotMatch() noexcept = 0; +}; + +} diff --git a/proxygen/lib/ssl/DHParam.h b/proxygen/lib/ssl/DHParam.h new file mode 100644 index 0000000000..561d56915e --- /dev/null +++ b/proxygen/lib/ssl/DHParam.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +// The following was auto-generated by +// openssl dhparam -C 2048 +DH *get_dh2048() + { + static unsigned char dh2048_p[]={ + 0xF8,0x87,0xA5,0x15,0x98,0x35,0x20,0x1E,0xF5,0x81,0xE5,0x95, + 0x1B,0xE4,0x54,0xEA,0x53,0xF5,0xE7,0x26,0x30,0x03,0x06,0x79, + 0x3C,0xC1,0x0B,0xAD,0x3B,0x59,0x3C,0x61,0x13,0x03,0x7B,0x02, + 0x70,0xDE,0xC1,0x20,0x11,0x9E,0x94,0x13,0x50,0xF7,0x62,0xFC, + 0x99,0x0D,0xC1,0x12,0x6E,0x03,0x95,0xA3,0x57,0xC7,0x3C,0xB8, + 0x6B,0x40,0x56,0x65,0x70,0xFB,0x7A,0xE9,0x02,0xEC,0xD2,0xB6, + 0x54,0xD7,0x34,0xAD,0x3D,0x9E,0x11,0x61,0x53,0xBE,0xEA,0xB8, + 0x17,0x48,0xA8,0xDC,0x70,0xAE,0x65,0x99,0x3F,0x82,0x4C,0xFF, + 0x6A,0xC9,0xFA,0xB1,0xFA,0xE4,0x4F,0x5D,0xA4,0x05,0xC2,0x8E, + 0x55,0xC0,0xB1,0x1D,0xCC,0x17,0xF3,0xFA,0x65,0xD8,0x6B,0x09, + 0x13,0x01,0x2A,0x39,0xF1,0x86,0x73,0xE3,0x7A,0xC8,0xDB,0x7D, + 0xDA,0x1C,0xA1,0x2D,0xBA,0x2C,0x00,0x6B,0x2C,0x55,0x28,0x2B, + 0xD5,0xF5,0x3C,0x9F,0x50,0xA7,0xB7,0x28,0x9F,0x22,0xD5,0x3A, + 0xC4,0x53,0x01,0xC9,0xF3,0x69,0xB1,0x8D,0x01,0x36,0xF8,0xA8, + 0x89,0xCA,0x2E,0x72,0xBC,0x36,0x3A,0x42,0xC1,0x06,0xD6,0x0E, + 0xCB,0x4D,0x5C,0x1F,0xE4,0xA1,0x17,0xBF,0x55,0x64,0x1B,0xB4, + 0x52,0xEC,0x15,0xED,0x32,0xB1,0x81,0x07,0xC9,0x71,0x25,0xF9, + 0x4D,0x48,0x3D,0x18,0xF4,0x12,0x09,0x32,0xC4,0x0B,0x7A,0x4E, + 0x83,0xC3,0x10,0x90,0x51,0x2E,0xBE,0x87,0xF9,0xDE,0xB4,0xE6, + 0x3C,0x29,0xB5,0x32,0x01,0x9D,0x95,0x04,0xBD,0x42,0x89,0xFD, + 0x21,0xEB,0xE9,0x88,0x5A,0x27,0xBB,0x31,0xC4,0x26,0x99,0xAB, + 0x8C,0xA1,0x76,0xDB, + }; + static unsigned char dh2048_g[]={ + 0x02, + }; + DH *dh; + + if ((dh=DH_new()) == nullptr) return(nullptr); + dh->p=BN_bin2bn(dh2048_p,(int)sizeof(dh2048_p),nullptr); + dh->g=BN_bin2bn(dh2048_g,(int)sizeof(dh2048_g),nullptr); + if ((dh->p == nullptr) || (dh->g == nullptr)) + { DH_free(dh); return(nullptr); } + return(dh); + } diff --git a/proxygen/lib/ssl/Makefile.am b/proxygen/lib/ssl/Makefile.am new file mode 100644 index 0000000000..a1326945e9 --- /dev/null +++ b/proxygen/lib/ssl/Makefile.am @@ -0,0 +1,28 @@ +SUBDIRS = . test + +noinst_LTLIBRARIES = libproxygenssl.la + +libproxygenssldir = $(includedir)/proxygen/lib/ssl +nobase_libproxygenssl_HEADERS = \ + ClientHelloExtStats.h \ + DHParam.h \ + PasswordInFile.h \ + SSLCacheOptions.h \ + SSLCacheProvider.h \ + SSLContextConfig.h \ + SSLContextManager.h \ + SSLSessionCacheManager.h \ + SSLStats.h \ + SSLUtil.h \ + TLSTicketKeyManager.h \ + TLSTicketKeySeeds.h + +libproxygenssl_la_SOURCES = \ + PasswordInFile.cpp \ + SSLContextManager.cpp \ + SSLSessionCacheManager.cpp \ + SSLUtil.cpp \ + TLSTicketKeyManager.cpp + +libproxygenssl_la_LIBADD = \ + ../utils/libutils.la diff --git a/proxygen/lib/ssl/PasswordInFile.cpp b/proxygen/lib/ssl/PasswordInFile.cpp new file mode 100644 index 0000000000..c906d641a2 --- /dev/null +++ b/proxygen/lib/ssl/PasswordInFile.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/ssl/PasswordInFile.h" + +#include + +using namespace std; + +namespace proxygen { + +PasswordInFile::PasswordInFile(const string& file) + : fileName_(file) { + folly::readFile(file.c_str(), password_); + auto p = password_.find('\0'); + if (p != std::string::npos) { + password_.erase(p); + } +} + +PasswordInFile::~PasswordInFile() { + OPENSSL_cleanse((char *)password_.data(), password_.length()); +} + +} diff --git a/proxygen/lib/ssl/PasswordInFile.h b/proxygen/lib/ssl/PasswordInFile.h new file mode 100644 index 0000000000..1052b6ddaa --- /dev/null +++ b/proxygen/lib/ssl/PasswordInFile.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include // PasswordCollector + +namespace proxygen { + +class PasswordInFile: public apache::thrift::transport::PasswordCollector { + public: + explicit PasswordInFile(const std::string& file); + ~PasswordInFile(); + + void getPassword(std::string& password, int size) override { + password = password_; + } + + const char* getPasswordStr() const { + return password_.c_str(); + } + + std::string describe() const override { + return fileName_; + } + + protected: + std::string fileName_; + std::string password_; +}; + +} diff --git a/proxygen/lib/ssl/SSLCacheOptions.h b/proxygen/lib/ssl/SSLCacheOptions.h new file mode 100644 index 0000000000..065d2e83c7 --- /dev/null +++ b/proxygen/lib/ssl/SSLCacheOptions.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +struct SSLCacheOptions { + std::chrono::seconds sslCacheTimeout; + uint64_t maxSSLCacheSize; + uint64_t sslCacheFlushSize; +}; + +} diff --git a/proxygen/lib/ssl/SSLCacheProvider.h b/proxygen/lib/ssl/SSLCacheProvider.h new file mode 100644 index 0000000000..8e2d7139a3 --- /dev/null +++ b/proxygen/lib/ssl/SSLCacheProvider.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +class SSLSessionCacheManager; + +/** + * Interface to be implemented by providers of external session caches + */ +class SSLCacheProvider { +public: + /** + * Context saved during an external cache request that is used to + * resume the waiting client. + */ + typedef struct { + std::string sessionId; + SSL_SESSION* session; + SSLSessionCacheManager* manager; + apache::thrift::async::TAsyncSSLSocket* sslSocket; + std::unique_ptr< + apache::thrift::async::TDelayedDestruction::DestructorGuard> guard; + } CacheContext; + + virtual ~SSLCacheProvider() {} + + /** + * Store a session in the external cache. + * @param sessionId Identifier that can be used later to fetch the + * session with getAsync() + * @param value Serialized session to store + * @param expiration Relative expiration time: seconds from now + * @return true if the storing of the session is initiated successfully + * (though not necessarily completed; the completion may + * happen either before or after this method returns), or + * false if the storing cannot be initiated due to an error. + */ + virtual bool setAsync(const std::string& sessionId, + const std::string& value, + std::chrono::seconds expiration) = 0; + + /** + * Retrieve a session from the external cache. When done, call + * the cache manager's onGetSuccess() or onGetFailure() callback. + * @param sessionId Session ID to fetch + * @param context Data to pass back to the SSLSessionCacheManager + * in the completion callback + * @return true if the lookup of the session is initiated successfully + * (though not necessarily completed; the completion may + * happen either before or after this method returns), or + * false if the lookup cannot be initiated due to an error. + */ + virtual bool getAsync(const std::string& sessionId, + CacheContext* context) = 0; + +}; + +} diff --git a/proxygen/lib/ssl/SSLContextConfig.h b/proxygen/lib/ssl/SSLContextConfig.h new file mode 100644 index 0000000000..fa411dd427 --- /dev/null +++ b/proxygen/lib/ssl/SSLContextConfig.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +/** + * SSLContextConfig helps to describe the configs/options for + * a SSL_CTX. For example: + * + * 1. Filename of X509, private key and its password. + * 2. ciphers list + * 3. NPN list + * 4. Is session cache enabled? + * 5. Is it the default X509 in SNI operation? + * 6. .... and a few more + */ +namespace proxygen { + +struct SSLContextConfig { + SSLContextConfig() {} + ~SSLContextConfig() {} + + struct CertificateInfo { + std::string certPath; + std::string keyPath; + std::string passwordPath; + }; + + /** + * Helpers to set/add a certificate + */ + void setCertificate(const std::string& certPath, + const std::string& keyPath, + const std::string& passwordPath) { + certificates.clear(); + addCertificate(certPath, keyPath, passwordPath); + } + + void addCertificate(const std::string& certPath, + const std::string& keyPath, + const std::string& passwordPath) { + certificates.emplace_back(CertificateInfo{certPath, keyPath, passwordPath}); + } + + /** + * Set the optional list of protocols to advertise via TLS + * Next Protocol Negotiation. An empty list means NPN is not enabled. + */ + void setNextProtocols(const std::list& inNextProtocols) { + nextProtocols.clear(); + nextProtocols.push_back({1, inNextProtocols}); + } + + typedef std::function SNINoMatchFn; + + std::vector certificates; + apache::thrift::transport::SSLContext::SSLVersion sslVersion{ + apache::thrift::transport::SSLContext::TLSv1}; + bool sessionCacheEnabled{true}; + bool sessionTicketEnabled{true}; + bool clientHelloParsingEnabled{false}; + std::string sslCiphers{ + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES128-GCM-SHA256:" + "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:" + "AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA:" + "ECDHE-ECDSA-RC4-SHA:ECDHE-RSA-RC4-SHA:RC4-SHA:RC4-MD5:" + "ECDHE-RSA-DES-CBC3-SHA:DES-CBC3-SHA"}; + std::string eccCurveName; + // Ciphers to negotiate if TLS version >= 1.1 + std::string tls11Ciphers{""}; + // Weighted lists of NPN strings to advertise + std::list + nextProtocols; + bool isLocalPrivateKey{true}; + // Should this SSLContextConfig be the default for SNI purposes + bool isDefault{false}; + // Callback function to invoke when there are no matching certificates + // (will only be invoked once) + SNINoMatchFn sniNoMatchFn; + // File containing trusted CA's to validate client certificates + std::string clientCAFile; +}; + +} diff --git a/proxygen/lib/ssl/SSLContextManager.cpp b/proxygen/lib/ssl/SSLContextManager.cpp new file mode 100644 index 0000000000..64ecaba0ae --- /dev/null +++ b/proxygen/lib/ssl/SSLContextManager.cpp @@ -0,0 +1,654 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/ssl/SSLContextManager.h" + +#include "proxygen/lib/ssl/ClientHelloExtStats.h" +#include "proxygen/lib/ssl/DHParam.h" +#include "proxygen/lib/ssl/PasswordInFile.h" +#include "proxygen/lib/ssl/SSLCacheOptions.h" +#include "proxygen/lib/ssl/SSLSessionCacheManager.h" +#include "proxygen/lib/ssl/SSLUtil.h" +#include "proxygen/lib/ssl/TLSTicketKeyManager.h" +#include "proxygen/lib/ssl/TLSTicketKeySeeds.h" +#include "proxygen/lib/utils/Exception.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define OPENSSL_MISSING_FEATURE(name) \ +do { \ + throw proxygen::Exception("missing " #name " support in openssl"); \ +} while(0) + + +using std::string; +using std::shared_ptr; +using apache::thrift::transport::SSLContext; +using apache::thrift::async::TAsyncSSLSocket; + +/** + * SSLContextManager helps to create and manage all SSL_CTX, + * SSLSessionCacheManager and TLSTicketManager for a listening + * VIP:PORT. (Note, in SNI, a listening VIP:PORT can have >1 SSL_CTX(s)). + * + * Other responsibilities: + * 1. It also handles the SSL_CTX selection after getting the tlsext_hostname + * in the client hello message. + * + * Usage: + * 1. Each listening VIP:PORT serving SSL should have one SSLContextManager. + * It maps to Accetpor in the proxygen vocabulary. + * + * 2. Create a SSLContextConfig object (e.g. by parsing the JSON config). + * + * 3. Call SSLContextManager::addSSLContextConfig() which will + * then create and configure the SSL_CTX + * + * Note: Each Acceptor, with SSL support, should have one SSLContextManager to + * manage all SSL_CTX for the VIP:PORT. + */ + +namespace proxygen { + +namespace { + +X509* getX509(SSL_CTX* ctx) { + SSL* ssl = SSL_new(ctx); + SSL_set_connect_state(ssl); + X509* x509 = SSL_get_certificate(ssl); + CRYPTO_add(&x509->references, 1, CRYPTO_LOCK_X509); + SSL_free(ssl); + return x509; +} + +void set_key_from_curve(SSL_CTX* ctx, const std::string& curveName) { +#if OPENSSL_VERSION_NUMBER >= 0x0090800fL +#ifndef OPENSSL_NO_ECDH + EC_KEY* ecdh = nullptr; + int nid; + + /* + * Elliptic-Curve Diffie-Hellman parameters are either "named curves" + * from RFC 4492 section 5.1.1, or explicitly described curves over + * binary fields. OpenSSL only supports the "named curves", which provide + * maximum interoperability. + */ + + nid = OBJ_sn2nid(curveName.c_str()); + if (nid == 0) { + LOG(FATAL) << "Unknown curve name:" << curveName.c_str(); + return; + } + ecdh = EC_KEY_new_by_curve_name(nid); + if (ecdh == nullptr) { + LOG(FATAL) << "Unable to create curve:" << curveName.c_str(); + return; + } + + SSL_CTX_set_tmp_ecdh(ctx, ecdh); + EC_KEY_free(ecdh); +#endif +#endif +} + +// Helper to create TLSTicketKeyManger and aware of the needed openssl +// version/feature. +std::unique_ptr createTicketManagerHelper( + std::shared_ptr ctx, + const TLSTicketKeySeeds* ticketSeeds, + const SSLContextConfig& ctxConfig, + SSLStats* stats) { + + std::unique_ptr ticketManager; +#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB + if (ticketSeeds && ctxConfig.sessionTicketEnabled) { + ticketManager = folly::make_unique(ctx.get(), stats); + ticketManager->setTLSTicketKeySeeds( + ticketSeeds->oldSeeds, + ticketSeeds->currentSeeds, + ticketSeeds->newSeeds); + } else { + ctx->setOptions(SSL_OP_NO_TICKET); + } +#else + if (ticketSeeds && ctxConfig.sessionTicketEnabled) { + OPENSSL_MISSING_FEATURE(TLSTicket); + } +#endif + return ticketManager; +} + +std::string flattenList(const std::list& list) { + std::string s; + bool first = true; + for (auto& item : list) { + if (first) { + first = false; + } else { + s.append(", "); + } + s.append(item); + } + return s; +} + +} + +SSLContextManager::~SSLContextManager() {} + +SSLContextManager::SSLContextManager( + folly::EventBase* eventBase, + const std::string& vipName, + bool strict, + SSLStats* stats) : + stats_(stats), + eventBase_(eventBase), + strict_(strict) { +} + +void SSLContextManager::addSSLContextConfig( + const SSLContextConfig& ctxConfig, + const SSLCacheOptions& cacheOptions, + const TLSTicketKeySeeds* ticketSeeds, + const folly::SocketAddress& vipAddress, + const std::shared_ptr& externalCache) { + + unsigned numCerts = 0; + std::string commonName; + std::string lastCertPath; + std::unique_ptr> subjectAltName; + auto sslCtx = std::make_shared(ctxConfig.sslVersion); + for (const auto& cert : ctxConfig.certificates) { + try { + sslCtx->loadCertificate(cert.certPath.c_str()); + } catch (const std::exception& ex) { + // The exception isn't very useful without the certificate path name, + // so throw a new exception that includes the path to the certificate. + string msg = folly::to("error loading SSL certificate ", + cert.certPath, ": ", + folly::exceptionStr(ex)); + LOG(ERROR) << msg; + throw Exception(msg); + } + + // Verify that the Common Name and (if present) Subject Alternative Names + // are the same for all the certs specified for the SSL context. + numCerts++; + X509* x509 = getX509(sslCtx->getSSLCtx()); + auto guard = folly::makeGuard([x509] { X509_free(x509); }); + auto cn = SSLUtil::getCommonName(x509); + if (!cn) { + throw Exception(folly::to("Cannot get CN for X509 ", + cert.certPath)); + } + auto altName = SSLUtil::getSubjectAltName(x509); + VLOG(2) << "cert " << cert.certPath << " CN: " << *cn; + if (altName) { + altName->sort(); + VLOG(2) << "cert " << cert.certPath << " SAN: " << flattenList(*altName); + } else { + VLOG(2) << "cert " << cert.certPath << " SAN: " << "{none}"; + } + if (numCerts == 1) { + commonName = *cn; + subjectAltName = std::move(altName); + } else { + if (commonName != *cn) { + throw Exception(folly::to("X509 ", cert.certPath, + " does not have same CN as ", + lastCertPath)); + } + if (altName == nullptr) { + if (subjectAltName != nullptr) { + throw Exception(folly::to("X509 ", cert.certPath, + " does not have same SAN as ", + lastCertPath)); + } + } else { + if ((subjectAltName == nullptr) || (*altName != *subjectAltName)) { + throw Exception(folly::to("X509 ", cert.certPath, + " does not have same SAN as ", + lastCertPath)); + } + } + } + lastCertPath = cert.certPath; + + // TODO t4438250 - Add ECDSA support to the crypto_ssl offload server + // so we can avoid storing the ECDSA private key in the + // address space of the Internet-facing process. For + // now, if cert name includes "-EC" to denote elliptic + // curve, we load its private key even if the server as + // a whole has been configured for async crypto. + if (ctxConfig.isLocalPrivateKey || + (cert.certPath.find("-EC") != std::string::npos)) { + // The private key lives in the same process + + // This needs to be called before loadPrivateKey(). + if (!cert.passwordPath.empty()) { + auto sslPassword = std::make_shared(cert.passwordPath); + sslCtx->passwordCollector(sslPassword); + } + + try { + sslCtx->loadPrivateKey(cert.keyPath.c_str()); + } catch (const std::exception& ex) { + // Throw an error that includes the key path, so the user can tell + // which key had a problem. + string msg = folly::to("error loading private SSL key ", + cert.keyPath, ": ", + folly::exceptionStr(ex)); + LOG(ERROR) << msg; + throw Exception(msg); + } + } + } + if (!ctxConfig.isLocalPrivateKey) { + enableAsyncCrypto(sslCtx); + } + + // Let proxygen pick the highest performing cipher from among the client's + // choices. + // + // Let's use a unique private key for all DH key exchanges. + // + // Because some old implementations choke on empty fragments, most SSL + // applications disable them (it's part of SSL_OP_ALL). For proxygen this + // will improve performance and decrease write buffer fragmentation. + sslCtx->setOptions(SSL_OP_CIPHER_SERVER_PREFERENCE | + SSL_OP_SINGLE_DH_USE | + SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); + + // Configure SSL ciphers list + if (!ctxConfig.tls11Ciphers.empty()) { + // FIXME: create a dummy SSL_CTX for cipher testing purpose? It can + // remove the ordering dependency + + // Test to see if the specified TLS1.1 ciphers are valid. Note that + // these will be overwritten by the ciphers() call below. + sslCtx->setCiphersOrThrow(ctxConfig.tls11Ciphers); + } + + // Important that we do this *after* checking the TLS1.1 ciphers above, + // since we test their validity by actually setting them. + sslCtx->ciphers(ctxConfig.sslCiphers); + + // Use a fix DH param + DH* dh = get_dh2048(); + SSL_CTX_set_tmp_dh(sslCtx->getSSLCtx(), dh); + DH_free(dh); + + const string& curve = ctxConfig.eccCurveName; + if (!curve.empty()) { + set_key_from_curve(sslCtx->getSSLCtx(), curve); + } + + if (!ctxConfig.clientCAFile.empty()) { + try { + sslCtx->setVerificationOption(SSLContext::VERIFY_REQ_CLIENT_CERT); + sslCtx->loadTrustedCertificates(ctxConfig.clientCAFile.c_str()); + sslCtx->loadClientCAList(ctxConfig.clientCAFile.c_str()); + } catch (const std::exception& ex) { + string msg = folly::to("error loading client CA", + ctxConfig.clientCAFile, ": ", + folly::exceptionStr(ex)); + LOG(ERROR) << msg; + throw Exception(msg); + } + } + + // - start - SSL session cache config + // the internal cache never does what we want (per-thread-per-vip). + // Disable it. SSLSessionCacheManager will set it appropriately. + SSL_CTX_set_session_cache_mode(sslCtx->getSSLCtx(), SSL_SESS_CACHE_OFF); + SSL_CTX_set_timeout(sslCtx->getSSLCtx(), + cacheOptions.sslCacheTimeout.count()); + std::unique_ptr sessionCacheManager; + if (ctxConfig.sessionCacheEnabled && + cacheOptions.maxSSLCacheSize > 0 && + cacheOptions.sslCacheFlushSize > 0) { + sessionCacheManager = + folly::make_unique( + cacheOptions.maxSSLCacheSize, + cacheOptions.sslCacheFlushSize, + sslCtx.get(), + vipAddress, + commonName, + eventBase_, + stats_, + externalCache); + } + // - end - SSL session cache config + + std::unique_ptr ticketManager = + createTicketManagerHelper(sslCtx, ticketSeeds, ctxConfig, stats_); + + // finalize sslCtx setup by the individual features supported by openssl + ctxSetupByOpensslFeature(sslCtx, ctxConfig); + + try { + insert(sslCtx, + std::move(sessionCacheManager), + std::move(ticketManager), + ctxConfig.isDefault); + } catch (const std::exception& ex) { + string msg = folly::to("Error adding certificate : ", + folly::exceptionStr(ex)); + LOG(ERROR) << msg; + throw Exception(msg); + } + +} + +#ifdef SSL_CTRL_SET_TLSEXT_SERVERNAME_CB +SSLContext::ServerNameCallbackResult +SSLContextManager::serverNameCallback(SSL* ssl) { + shared_ptr ctx; + + const char* sn = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + if (!sn) { + VLOG(6) << "Server Name (tlsext_hostname) is missing"; + if (clientHelloTLSExtStats_) { + clientHelloTLSExtStats_->recordAbsentHostname(); + } + return SSLContext::SERVER_NAME_NOT_FOUND; + } + size_t snLen = strlen(sn); + VLOG(6) << "Server Name (SNI TLS extension): '" << sn << "' "; + + // FIXME: This code breaks the abstraction. Suggestion? + TAsyncSSLSocket* sslSocket = TAsyncSSLSocket::getFromSSL(ssl); + CHECK(sslSocket); + + DNString dnstr(sn, snLen); + + uint32_t count = 0; + do { + // Try exact match first + ctx = getSSLCtx(dnstr); + if (ctx) { + sslSocket->switchServerSSLContext(ctx); + if (clientHelloTLSExtStats_) { + clientHelloTLSExtStats_->recordMatch(); + } + return SSLContext::SERVER_NAME_FOUND; + } + + ctx = getSSLCtxBySuffix(dnstr); + if (ctx) { + sslSocket->switchServerSSLContext(ctx); + if (clientHelloTLSExtStats_) { + clientHelloTLSExtStats_->recordMatch(); + } + return SSLContext::SERVER_NAME_FOUND; + } + + // Give the noMatchFn one chance to add the correct cert + } + while (count++ == 0 && noMatchFn_ && noMatchFn_(sn)); + + VLOG(6) << folly::stringPrintf("Cannot find a SSL_CTX for \"%s\"", sn); + + if (clientHelloTLSExtStats_) { + clientHelloTLSExtStats_->recordNotMatch(); + } + return SSLContext::SERVER_NAME_NOT_FOUND; +} +#endif + +// Consolidate all SSL_CTX setup which depends on openssl version/feature +void +SSLContextManager::ctxSetupByOpensslFeature( + shared_ptr sslCtx, + const SSLContextConfig& ctxConfig) { + // Disable compression - profiling shows this to be very expensive in + // terms of CPU and memory consumption. + // +#ifdef SSL_OP_NO_COMPRESSION + sslCtx->setOptions(SSL_OP_NO_COMPRESSION); +#endif + + // Enable early release of SSL buffers to reduce the memory footprint +#ifdef SSL_MODE_RELEASE_BUFFERS + sslCtx->getSSLCtx()->mode |= SSL_MODE_RELEASE_BUFFERS; +#endif +#ifdef SSL_MODE_EARLY_RELEASE_BBIO + sslCtx->getSSLCtx()->mode |= SSL_MODE_EARLY_RELEASE_BBIO; +#endif + + // This number should (probably) correspond to HTTPSession::kMaxReadSize + // For now, this number must also be large enough to accommodate our + // largest certificate, because some older clients (IE6/7) require the + // cert to be in a single fragment. +#ifdef SSL_CTRL_SET_MAX_SEND_FRAGMENT + SSL_CTX_set_max_send_fragment(sslCtx->getSSLCtx(), 8000); +#endif + + // Specify cipher(s) to be used for TLS1.1 client + if (!ctxConfig.tls11Ciphers.empty()) { +#ifdef SSL_CTRL_SET_TLSEXT_SERVERNAME_CB + // Specified TLS1.1 ciphers are valid + sslCtx->addClientHelloCallback( + std::bind( + &SSLContext::switchCiphersIfTLS11, + sslCtx.get(), + std::placeholders::_1, + ctxConfig.tls11Ciphers + ) + ); +#else + OPENSSL_MISSING_FEATURE(SNI); +#endif + } + + // NPN (Next Protocol Negotiation) + if (!ctxConfig.nextProtocols.empty()) { +#ifdef OPENSSL_NPN_NEGOTIATED + sslCtx->setRandomizedAdvertisedNextProtocols(ctxConfig.nextProtocols); +#else + OPENSSL_MISSING_FEATURE(NPN); +#endif + } + + // SNI +#ifdef SSL_CTRL_SET_TLSEXT_SERVERNAME_CB + noMatchFn_ = ctxConfig.sniNoMatchFn; + if (ctxConfig.isDefault) { + if (defaultCtx_) { + throw Exception(">1 X509 is set as default"); + } + + defaultCtx_ = sslCtx; + defaultCtx_->setServerNameCallback( + std::bind(&SSLContextManager::serverNameCallback, this, + std::placeholders::_1)); + } +#else + if (ctxs_.size() > 1) { + OPENSSL_MISSING_FEATURE(SNI); + } +#endif +} + +void +SSLContextManager::insert(shared_ptr sslCtx, + std::unique_ptr smanager, + std::unique_ptr tmanager, + bool defaultFallback) { + X509* x509 = getX509(sslCtx->getSSLCtx()); + auto guard = folly::makeGuard([x509] { X509_free(x509); }); + auto cn = SSLUtil::getCommonName(x509); + if (!cn) { + throw Exception("Cannot get CN"); + } + + /** + * Some notes from RFC 2818. Only for future quick references in case of bugs + * + * RFC 2818 section 3.1: + * "...... + * If a subjectAltName extension of type dNSName is present, that MUST + * be used as the identity. Otherwise, the (most specific) Common Name + * field in the Subject field of the certificate MUST be used. Although + * the use of the Common Name is existing practice, it is deprecated and + * Certification Authorities are encouraged to use the dNSName instead. + * ...... + * In some cases, the URI is specified as an IP address rather than a + * hostname. In this case, the iPAddress subjectAltName must be present + * in the certificate and must exactly match the IP in the URI. + * ......" + */ + + // Not sure if we ever get this kind of X509... + // If we do, assume '*' is always in the CN and ignore all subject alternative + // names. + if (cn->length() == 1 && (*cn)[0] == '*') { + if (!defaultFallback) { + throw Exception("STAR X509 is not the default"); + } + ctxs_.emplace_back(sslCtx); + sessionCacheManagers_.emplace_back(std::move(smanager)); + ticketManagers_.emplace_back(std::move(tmanager)); + return; + } + + // Insert by CN + insertSSLCtxByDomainName(cn->c_str(), cn->length(), sslCtx); + + // Insert by subject alternative name(s) + auto altNames = SSLUtil::getSubjectAltName(x509); + if (altNames) { + for (auto& name : *altNames) { + insertSSLCtxByDomainName(name.c_str(), name.length(), sslCtx); + } + } + + ctxs_.emplace_back(sslCtx); + sessionCacheManagers_.emplace_back(std::move(smanager)); + ticketManagers_.emplace_back(std::move(tmanager)); +} + +void +SSLContextManager::insertSSLCtxByDomainName(const char* dn, size_t len, + shared_ptr sslCtx) { + try { + insertSSLCtxByDomainNameImpl(dn, len, sslCtx); + } catch (const Exception& ex) { + if (strict_) { + throw ex; + } else { + LOG(ERROR) << ex.what() << " DN=" << dn; + } + } +} +void +SSLContextManager::insertSSLCtxByDomainNameImpl(const char* dn, size_t len, + shared_ptr sslCtx) +{ + VLOG(4) << + folly::stringPrintf("Adding CN/Subject-alternative-name \"%s\" for " + "SNI search", dn); + + // Only support wildcard domains which are prefixed exactly by "*." . + // "*" appearing at other locations is not accepted. + + if (len > 2 && dn[0] == '*') { + if (dn[1] == '.') { + // skip the first '*' + dn++; + len--; + } else { + throw Exception("Invalid wildcard CN/subject-alternative-name \"%s\" " + "(only allow character \".\" after \"*\"", + dn); + } + } + + if (len == 1 && *dn == '.') { + throw Exception("X509 has only '.' in the CN or subject alternative name " + "(after removing any preceding '*')"); + } + + if (strchr(dn, '*')) { + throw Exception("X509 has '*' in the the CN or subject alternative name " + "(after removing any preceding '*')"); + } + + DNString dnstr(dn, len); + const auto v = dnMap_.find(dnstr); + if (v == dnMap_.end()) { + dnMap_.emplace(dnstr, sslCtx); + } else if (v->second == sslCtx) { + VLOG(6)<< "Duplicate CN or subject alternative name found in the same X509." + " Ignore the later name."; + } else { + throw Exception("Duplicate CN or subject alternative name found: \"", + dnstr.c_str(), "\""); + } +} + +shared_ptr +SSLContextManager::getSSLCtxBySuffix(const DNString& dnstr) const +{ + size_t dot; + + if ((dot = dnstr.find_first_of(".")) != DNString::npos) { + DNString suffixDNStr(dnstr, dot); + const auto v = dnMap_.find(suffixDNStr); + if (v != dnMap_.end()) { + VLOG(6) << folly::stringPrintf("\"%s\" is a willcard match to \"%s\"", + dnstr.c_str(), suffixDNStr.c_str()); + return v->second; + } + } + + VLOG(6) << folly::stringPrintf("\"%s\" is not a wildcard match", + dnstr.c_str()); + return shared_ptr(); +} + +shared_ptr +SSLContextManager::getSSLCtx(const DNString& dnstr) const +{ + const auto v = dnMap_.find(dnstr); + if (v == dnMap_.end()) { + VLOG(6) << folly::stringPrintf("\"%s\" is not an exact match", + dnstr.c_str()); + return shared_ptr(); + } else { + VLOG(6) << folly::stringPrintf("\"%s\" is an exact match", dnstr.c_str()); + return v->second; + } +} + +shared_ptr +SSLContextManager::getDefaultSSLCtx() const { + return defaultCtx_; +} + +void +SSLContextManager::reloadTLSTicketKeys( + const std::vector& oldSeeds, + const std::vector& currentSeeds, + const std::vector& newSeeds) { +#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB + for (auto& tmgr: ticketManagers_) { + tmgr->setTLSTicketKeySeeds(oldSeeds, currentSeeds, newSeeds); + } +#endif +} + +} // namespace proxygen diff --git a/proxygen/lib/ssl/SSLContextManager.h b/proxygen/lib/ssl/SSLContextManager.h new file mode 100644 index 0000000000..ca9bc32265 --- /dev/null +++ b/proxygen/lib/ssl/SSLContextManager.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/ssl/SSLContextConfig.h" +#include "proxygen/lib/ssl/SSLSessionCacheManager.h" +#include "proxygen/lib/ssl/TLSTicketKeySeeds.h" +#include "proxygen/lib/utils/DomainNameMisc.h" + +#include +#include +#include +#include +#include +#include + +namespace folly { +class SocketAddress; +class SSLContext; +} + +namespace proxygen { + +class ClientHelloExtStats; +class SSLCacheOptions; +class SSLStats; +class TLSTicketKeyManager; +class TLSTicketKeySeeds; + +class SSLContextManager { + public: + + explicit SSLContextManager(folly::EventBase* eventBase, + const std::string& vipName, bool strict, + SSLStats* stats); + virtual ~SSLContextManager(); + + /** + * Add a new X509 to SSLContextManager. The details of a X509 + * is passed as a SSLContextConfig object. + * + * @param ctxConfig Details of a X509, its private key, password, etc. + * @param cacheOptions Options for how to do session caching. + * @param ticketSeeds If non-null, the initial ticket key seeds to use. + * @param vipAddress Which VIP are the X509(s) used for? It is only for + * for user friendly log message + * @param externalCache Optional external provider for the session cache; + * may be null + */ + void addSSLContextConfig( + const SSLContextConfig& ctxConfig, + const SSLCacheOptions& cacheOptions, + const TLSTicketKeySeeds* ticketSeeds, + const folly::SocketAddress& vipAddress, + const std::shared_ptr &externalCache); + + /** + * Get the default SSL_CTX for a VIP + */ + std::shared_ptr + getDefaultSSLCtx() const; + + /** + * Search by the _one_ level up subdomain + */ + std::shared_ptr + getSSLCtxBySuffix(const DNString& dnstr) const; + + /** + * Search by the full-string domain name + */ + std::shared_ptr + getSSLCtx(const DNString& dnstr) const; + + /** + * Insert a SSLContext by domain name. + */ + void insertSSLCtxByDomainName( + const char* dn, + size_t len, + std::shared_ptr sslCtx); + + void insertSSLCtxByDomainNameImpl( + const char* dn, + size_t len, + std::shared_ptr sslCtx); + + void reloadTLSTicketKeys(const std::vector& oldSeeds, + const std::vector& currentSeeds, + const std::vector& newSeeds); + + /** + * SSLContextManager only collects SNI stats now + */ + + void setClientHelloExtStats(ClientHelloExtStats* stats) { + clientHelloTLSExtStats_ = stats; + } + + protected: + virtual void enableAsyncCrypto( + const std::shared_ptr& sslCtx) { + LOG(FATAL) << "Unsupported in base SSLContextManager"; + } + SSLStats* stats_{nullptr}; + + private: + SSLContextManager(const SSLContextManager&) = delete; + + void ctxSetupByOpensslFeature( + std::shared_ptr sslCtx, + const SSLContextConfig& ctxConfig); + + /** + * Callback function from openssl to find the right X509 to + * use during SSL handshake + */ +#ifdef SSL_CTRL_SET_TLSEXT_SERVERNAME_CB + apache::thrift::transport::SSLContext::ServerNameCallbackResult + serverNameCallback(SSL* ssl); +#endif + + /** + * The following functions help to maintain the data structure for + * domain name matching in SNI. Some notes: + * + * 1. It is a best match. + * + * 2. It allows wildcard CN and wildcard subject alternative name in a X509. + * The wildcard name must be _prefixed_ by '*.'. It errors out whenever + * it sees '*' in any other locations. + * + * 3. It uses one std::unordered_map object to + * do this. For wildcard name like "*.facebook.com", ".facebook.com" + * is used as the key. + * + * 4. After getting tlsext_hostname from the client hello message, it + * will do a full string search first and then try one level up to + * match any wildcard name (if any) in the X509. + * [Note, browser also only looks one level up when matching the requesting + * domain name with the wildcard name in the server X509]. + */ + + void insert( + std::shared_ptr sslCtx, + std::unique_ptr cmanager, + std::unique_ptr tManager, + bool defaultFallback); + + /** + * Container to own the SSLContext, SSLSessionCacheManager and + * TLSTicketKeyManager. + */ + std::vector> ctxs_; + std::vector> + sessionCacheManagers_; + std::vector> ticketManagers_; + + + std::shared_ptr defaultCtx_; + + /** + * Container to store the (DomainName -> SSL_CTX) mapping + */ + std::unordered_map< + DNString, + std::shared_ptr, + DNStringHash> dnMap_; + + folly::EventBase* eventBase_; + ClientHelloExtStats* clientHelloTLSExtStats_{nullptr}; + SSLContextConfig::SNINoMatchFn noMatchFn_; + bool strict_{true}; +}; + +} // namespace proxygen diff --git a/proxygen/lib/ssl/SSLSessionCacheManager.cpp b/proxygen/lib/ssl/SSLSessionCacheManager.cpp new file mode 100644 index 0000000000..e255ea2c61 --- /dev/null +++ b/proxygen/lib/ssl/SSLSessionCacheManager.cpp @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/ssl/SSLSessionCacheManager.h" + +#include "proxygen/lib/ssl/SSLCacheProvider.h" +#include "proxygen/lib/ssl/SSLStats.h" +#include "proxygen/lib/ssl/SSLUtil.h" + +#include + +using std::string; +using std::shared_ptr; +using apache::thrift::transport::SSLContext; +using folly::EventBase; +using apache::thrift::async::TAsyncSSLSocket; +using apache::thrift::async::TDelayedDestruction; + +namespace { + +const uint32_t NUM_CACHE_BUCKETS = 16; + +// We use the default ID generator which fills the maximum ID length +// for the protocol. 16 bytes for SSLv2 or 32 for SSLv3+ +const int MIN_SESSION_ID_LENGTH = 16; + +} + +DEFINE_bool(dcache_unit_test, false, "All VIPs share one session cache"); + +namespace proxygen { + + +int SSLSessionCacheManager::sExDataIndex_ = -1; +shared_ptr SSLSessionCacheManager::sCache_; +std::mutex SSLSessionCacheManager::sCacheLock_; + +LocalSSLSessionCache::LocalSSLSessionCache(uint32_t maxCacheSize, + uint32_t cacheCullSize) + : sessionCache(maxCacheSize, cacheCullSize) { + sessionCache.setPruneHook(std::bind( + &LocalSSLSessionCache::pruneSessionCallback, + this, std::placeholders::_1, + std::placeholders::_2)); +} + +void LocalSSLSessionCache::pruneSessionCallback(const string& sessionId, + SSL_SESSION* session) { + VLOG(4) << "Free SSL session from local cache; id=" + << SSLUtil::hexlify(sessionId); + SSL_SESSION_free(session); + ++removedSessions_; +} + + +// SSLSessionCacheManager implementation + +SSLSessionCacheManager::SSLSessionCacheManager( + uint32_t maxCacheSize, + uint32_t cacheCullSize, + SSLContext* ctx, + const folly::SocketAddress& sockaddr, + const string& context, + EventBase* eventBase, + SSLStats* stats, + const std::shared_ptr& externalCache): + ctx_(ctx), + stats_(stats), + externalCache_(externalCache) { + + SSL_CTX* sslCtx = ctx->getSSLCtx(); + + SSLUtil::getSSLCtxExIndex(&sExDataIndex_); + + SSL_CTX_set_ex_data(sslCtx, sExDataIndex_, this); + SSL_CTX_sess_set_new_cb(sslCtx, SSLSessionCacheManager::newSessionCallback); + SSL_CTX_sess_set_get_cb(sslCtx, SSLSessionCacheManager::getSessionCallback); + SSL_CTX_sess_set_remove_cb(sslCtx, + SSLSessionCacheManager::removeSessionCallback); + if (!FLAGS_dcache_unit_test && !context.empty()) { + // Use the passed in context + SSL_CTX_set_session_id_context(sslCtx, (const uint8_t *)context.data(), + std::min((int)context.length(), + SSL_MAX_SSL_SESSION_ID_LENGTH)); + } + + SSL_CTX_set_session_cache_mode(sslCtx, SSL_SESS_CACHE_NO_INTERNAL + | SSL_SESS_CACHE_SERVER); + + localCache_ = SSLSessionCacheManager::getLocalCache(maxCacheSize, + cacheCullSize); + + VLOG(2) << "On VipID=" << sockaddr.describe() << " context=" << context; +} + +SSLSessionCacheManager::~SSLSessionCacheManager() { +} + +void SSLSessionCacheManager::shutdown() { + std::lock_guard g(sCacheLock_); + sCache_.reset(); +} + +shared_ptr SSLSessionCacheManager::getLocalCache( + uint32_t maxCacheSize, + uint32_t cacheCullSize) { + + std::lock_guard g(sCacheLock_); + if (!sCache_) { + sCache_.reset(new ShardedLocalSSLSessionCache(NUM_CACHE_BUCKETS, + maxCacheSize, + cacheCullSize)); + } + return sCache_; +} + +int SSLSessionCacheManager::newSessionCallback(SSL* ssl, SSL_SESSION* session) { + SSLSessionCacheManager* manager = nullptr; + SSL_CTX* ctx = SSL_get_SSL_CTX(ssl); + manager = (SSLSessionCacheManager *)SSL_CTX_get_ex_data(ctx, sExDataIndex_); + + if (manager == nullptr) { + LOG(FATAL) << "Null SSLSessionCacheManager in callback"; + return -1; + } + return manager->newSession(ssl, session); +} + + +int SSLSessionCacheManager::newSession(SSL* ssl, SSL_SESSION* session) { + string sessionId((char*)session->session_id, session->session_id_length); + VLOG(4) << "New SSL session; id=" << SSLUtil::hexlify(sessionId); + + if (stats_) { + stats_->recordSSLSession(true /* new session */, false, false); + } + + localCache_->storeSession(sessionId, session, stats_); + + if (externalCache_) { + VLOG(4) << "New SSL session: send session to external cache; id=" << + SSLUtil::hexlify(sessionId); + storeCacheRecord(sessionId, session); + } + + return 1; +} + +void SSLSessionCacheManager::removeSessionCallback(SSL_CTX* ctx, + SSL_SESSION* session) { + SSLSessionCacheManager* manager = nullptr; + manager = (SSLSessionCacheManager *)SSL_CTX_get_ex_data(ctx, sExDataIndex_); + + if (manager == nullptr) { + LOG(FATAL) << "Null SSLSessionCacheManager in callback"; + return; + } + return manager->removeSession(ctx, session); +} + +void SSLSessionCacheManager::removeSession(SSL_CTX* ctx, + SSL_SESSION* session) { + string sessionId((char*)session->session_id, session->session_id_length); + + // This hook is only called from SSL when the internal session cache needs to + // flush sessions. Since we run with the internal cache disabled, this should + // never be called + VLOG(3) << "Remove SSL session; id=" << SSLUtil::hexlify(sessionId); + + localCache_->removeSession(sessionId); + + if (stats_) { + stats_->recordSSLSessionRemove(); + } +} + +SSL_SESSION* SSLSessionCacheManager::getSessionCallback(SSL* ssl, + unsigned char* sess_id, + int id_len, + int* copyflag) { + SSLSessionCacheManager* manager = nullptr; + SSL_CTX* ctx = SSL_get_SSL_CTX(ssl); + manager = (SSLSessionCacheManager *)SSL_CTX_get_ex_data(ctx, sExDataIndex_); + + if (manager == nullptr) { + LOG(FATAL) << "Null SSLSessionCacheManager in callback"; + return nullptr; + } + return manager->getSession(ssl, sess_id, id_len, copyflag); +} + +SSL_SESSION* SSLSessionCacheManager::getSession(SSL* ssl, + unsigned char* session_id, + int id_len, + int* copyflag) { + VLOG(7) << "SSL get session callback"; + SSL_SESSION* session = nullptr; + bool foreign = false; + char const* missReason = nullptr; + + if (id_len < MIN_SESSION_ID_LENGTH) { + // We didn't generate this session so it's going to be a miss. + // This doesn't get logged or counted in the stats. + return nullptr; + } + string sessionId((char*)session_id, id_len); + + TAsyncSSLSocket* sslSocket = TAsyncSSLSocket::getFromSSL(ssl); + + assert(sslSocket != nullptr); + + // look it up in the local cache first + session = localCache_->lookupSession(sessionId); +#ifdef SSL_SESSION_CB_WOULD_BLOCK + if (session == nullptr && externalCache_) { + // external cache might have the session + foreign = true; + if (!SSL_want_sess_cache_lookup(ssl)) { + missReason = "reason: No async cache support;"; + } else { + PendingLookupMap::iterator pit = pendingLookups_.find(sessionId); + if (pit == pendingLookups_.end()) { + auto result = pendingLookups_.emplace(sessionId, PendingLookup()); + // initiate fetch + VLOG(4) << "Get SSL session [Pending]: Initiate Fetch; fd=" << + sslSocket->getFd() << " id=" << SSLUtil::hexlify(sessionId); + if (lookupCacheRecord(sessionId, sslSocket)) { + // response is pending + *copyflag = SSL_SESSION_CB_WOULD_BLOCK; + return nullptr; + } else { + missReason = "reason: failed to send lookup request;"; + pendingLookups_.erase(result.first); + } + } else { + // A lookup was already initiated from this thread + if (pit->second.request_in_progress) { + // Someone else initiated the request, attach + VLOG(4) << "Get SSL session [Pending]: Request in progess: attach; " + "fd=" << sslSocket->getFd() << " id=" << + SSLUtil::hexlify(sessionId); + std::unique_ptr dg( + new TDelayedDestruction::DestructorGuard(sslSocket)); + pit->second.waiters.push_back( + std::make_pair(sslSocket, std::move(dg))); + *copyflag = SSL_SESSION_CB_WOULD_BLOCK; + return nullptr; + } + // request is complete + session = pit->second.session; // nullptr if our friend didn't have it + if (session != nullptr) { + CRYPTO_add(&session->references, 1, CRYPTO_LOCK_SSL_SESSION); + } + } + } + } +#endif + + bool hit = (session != nullptr); + if (stats_) { + stats_->recordSSLSession(false, hit, foreign); + } + if (hit) { + sslSocket->setSessionIDResumed(true); + } + + VLOG(4) << "Get SSL session [" << + ((hit) ? "Hit" : "Miss") << "]: " << + ((foreign) ? "external" : "local") << " cache; " << + ((missReason != nullptr) ? missReason : "") << "fd=" << + sslSocket->getFd() << " id=" << SSLUtil::hexlify(sessionId); + + // We already bumped the refcount + *copyflag = 0; + + return session; +} + +bool SSLSessionCacheManager::storeCacheRecord(const string& sessionId, + SSL_SESSION* session) { + std::string sessionString; + uint32_t sessionLen = i2d_SSL_SESSION(session, nullptr); + sessionString.resize(sessionLen); + uint8_t* cp = (uint8_t *)sessionString.data(); + i2d_SSL_SESSION(session, &cp); + size_t expiration = SSL_CTX_get_timeout(ctx_->getSSLCtx()); + return externalCache_->setAsync(sessionId, sessionString, + std::chrono::seconds(expiration)); +} + +bool SSLSessionCacheManager::lookupCacheRecord(const string& sessionId, + TAsyncSSLSocket* sslSocket) { + auto cacheCtx = new SSLCacheProvider::CacheContext(); + cacheCtx->sessionId = sessionId; + cacheCtx->session = nullptr; + cacheCtx->sslSocket = sslSocket; + cacheCtx->guard.reset( + new TDelayedDestruction::DestructorGuard(cacheCtx->sslSocket)); + cacheCtx->manager = this; + bool res = externalCache_->getAsync(sessionId, cacheCtx); + if (!res) { + delete cacheCtx; + } + return res; +} + +void SSLSessionCacheManager::restartSSLAccept( + const SSLCacheProvider::CacheContext* cacheCtx) { + PendingLookupMap::iterator pit = pendingLookups_.find(cacheCtx->sessionId); + CHECK(pit != pendingLookups_.end()); + pit->second.request_in_progress = false; + pit->second.session = cacheCtx->session; + VLOG(7) << "Restart SSL accept"; + cacheCtx->sslSocket->restartSSLAccept(); + for (const auto& attachedLookup: pit->second.waiters) { + // Wake up anyone else who was waiting for this session + VLOG(4) << "Restart SSL accept (waiters) for fd=" << + attachedLookup.first->getFd(); + attachedLookup.first->restartSSLAccept(); + } + pendingLookups_.erase(pit); +} + +void SSLSessionCacheManager::onGetSuccess( + SSLCacheProvider::CacheContext* cacheCtx, + const std::string& value) { + const uint8_t* cp = (uint8_t*)value.data(); + cacheCtx->session = d2i_SSL_SESSION(nullptr, &cp, value.length()); + restartSSLAccept(cacheCtx); + + /* Insert in the LRU after restarting all clients. The stats logic + * in getSession would treat this as a local hit otherwise. + */ + localCache_->storeSession(cacheCtx->sessionId, cacheCtx->session, stats_); + delete cacheCtx; +} + +void SSLSessionCacheManager::onGetFailure( + SSLCacheProvider::CacheContext* cacheCtx) { + restartSSLAccept(cacheCtx); + delete cacheCtx; +} + +} diff --git a/proxygen/lib/ssl/SSLSessionCacheManager.h b/proxygen/lib/ssl/SSLSessionCacheManager.h new file mode 100644 index 0000000000..70e994e5c6 --- /dev/null +++ b/proxygen/lib/ssl/SSLSessionCacheManager.h @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/ssl/SSLCacheProvider.h" +#include "proxygen/lib/ssl/SSLStats.h" + +#include +#include +#include + +namespace proxygen { + +class SSLStats; + +/** + * Basic SSL session cache map: Maps session id -> session + */ +typedef folly::EvictingCacheMap SSLSessionCacheMap; + +/** + * Holds an SSLSessionCacheMap and associated lock + */ +class LocalSSLSessionCache: private boost::noncopyable { + public: + LocalSSLSessionCache(uint32_t maxCacheSize, uint32_t cacheCullSize); + + ~LocalSSLSessionCache() { + std::lock_guard g(lock); + // EvictingCacheMap dtor doesn't free values + sessionCache.clear(); + } + + SSLSessionCacheMap sessionCache; + std::mutex lock; + uint32_t removedSessions_{0}; + + private: + + void pruneSessionCallback(const std::string& sessionId, + SSL_SESSION* session); +}; + +/** + * A sharded LRU for SSL sessions. The sharding is inteneded to reduce + * contention for the LRU locks. Assuming uniform distribution, two workers + * will contend for the same lock with probability 1 / n_buckets^2. + */ +class ShardedLocalSSLSessionCache : private boost::noncopyable { + public: + ShardedLocalSSLSessionCache(uint32_t n_buckets, uint32_t maxCacheSize, + uint32_t cacheCullSize) { + CHECK(n_buckets > 0); + maxCacheSize = (uint32_t)(((double)maxCacheSize) / n_buckets); + cacheCullSize = (uint32_t)(((double)cacheCullSize) / n_buckets); + if (maxCacheSize == 0) { + maxCacheSize = 1; + } + if (cacheCullSize == 0) { + cacheCullSize = 1; + } + for (uint32_t i = 0; i < n_buckets; i++) { + caches_.push_back( + std::unique_ptr( + new LocalSSLSessionCache(maxCacheSize, cacheCullSize))); + } + } + + SSL_SESSION* lookupSession(const std::string& sessionId) { + size_t bucket = hash(sessionId); + SSL_SESSION* session = nullptr; + std::lock_guard g(caches_[bucket]->lock); + + auto itr = caches_[bucket]->sessionCache.find(sessionId); + if (itr != caches_[bucket]->sessionCache.end()) { + session = itr->second; + } + + if (session) { + CRYPTO_add(&session->references, 1, CRYPTO_LOCK_SSL_SESSION); + } + return session; + } + + void storeSession(const std::string& sessionId, SSL_SESSION* session, + SSLStats* stats) { + size_t bucket = hash(sessionId); + SSL_SESSION* oldSession = nullptr; + std::lock_guard g(caches_[bucket]->lock); + + auto itr = caches_[bucket]->sessionCache.find(sessionId); + if (itr != caches_[bucket]->sessionCache.end()) { + oldSession = itr->second; + } + + if (oldSession) { + // LRUCacheMap doesn't free on overwrite, so 2x the work for us + // This can happen in race conditions + SSL_SESSION_free(oldSession); + } + caches_[bucket]->removedSessions_ = 0; + caches_[bucket]->sessionCache.set(sessionId, session, true); + if (stats) { + stats->recordSSLSessionFree(caches_[bucket]->removedSessions_); + } + } + + void removeSession(const std::string& sessionId) { + size_t bucket = hash(sessionId); + std::lock_guard g(caches_[bucket]->lock); + caches_[bucket]->sessionCache.erase(sessionId); + } + + private: + + /* SSL session IDs are 32 bytes of random data, hash based on first 16 bits */ + size_t hash(const std::string& key) { + CHECK(key.length() >= 2); + return (key[0] << 8 | key[1]) % caches_.size(); + } + + std::vector< std::unique_ptr > caches_; +}; + +/* A socket/DestructorGuard pair */ +typedef std::pair< + apache::thrift::async::TAsyncSSLSocket *, + std::unique_ptr> + AttachedLookup; + +/** + * PendingLookup structure + * + * Keeps track of clients waiting for an SSL session to be retrieved from + * the external cache provider. + */ +struct PendingLookup { + bool request_in_progress; + SSL_SESSION* session; + std::list waiters; + + PendingLookup() { + request_in_progress = true; + session = nullptr; + } +}; + +/* Maps SSL session id to a PendingLookup structure */ +typedef std::map PendingLookupMap; + +/** + * SSLSessionCacheManager handles all stateful session caching. There is an + * instance of this object per SSL VIP per thread, with a 1:1 correlation with + * SSL_CTX. The cache can work locally or in concert with an external cache + * to share sessions across proxygen instances. + * + * There is a single in memory session cache shared by all VIPs. The cache is + * split into N buckets (currently 16) with a separate lock per bucket. The + * VIP ID is hashed and stored as part of the session to handle the + * (very unlikely) case of session ID collision. + * + * When a new SSL session is created, it is added to the LRU cache and + * sent to the external cache to be stored. The external cache + * expiration is equal to the SSL session's expiration. + * + * When a resume request is received, SSLSessionCacheManager first looks in the + * local LRU cache for the VIP. If there is a miss there, an asynchronous + * request for this session is dispatched to the external cache. When the + * external cache query returns, the LRU cache is updated if the session was + * found, and the SSL_accept call is resumed. + * + * If additional resume requests for the same session ID arrive in the same + * thread while the request is pending, the 2nd - Nth callers attach to the + * original external cache requests and are resumed when it comes back. No + * attempt is made to coalesce external cache requests for the same session + * ID in different worker threads. Previous work did this, but the + * complexity was deemed to outweigh the potential savings. + * + */ +class SSLSessionCacheManager : private boost::noncopyable { + public: + /** + * Constructor. SSL session related callbacks will be set on the underlying + * SSL_CTX. vipId is assumed to a unique string identifying the VIP and must + * be the same on all servers that wish to share sessions via the same + * external cache. + */ + SSLSessionCacheManager( + uint32_t maxCacheSize, + uint32_t cacheCullSize, + apache::thrift::transport::SSLContext* ctx, + const folly::SocketAddress& sockaddr, + const std::string& context, + folly::EventBase* eventBase, + SSLStats* stats, + const std::shared_ptr& externalCache); + + virtual ~SSLSessionCacheManager(); + + /** + * Call this on shutdown to release the global instance of the + * ShardedLocalSSLSessionCache. + */ + static void shutdown(); + + /** + * Callback for ExternalCache to call when an async get succeeds + * @param context The context that was passed to the async get request + * @param value Serialized session + */ + void onGetSuccess(SSLCacheProvider::CacheContext* context, + const std::string& value); + + /** + * Callback for ExternalCache to call when an async get fails, either + * because the requested session is not in the external cache or because + * of an error. + * @param context The context that was passed to the async get request + */ + void onGetFailure(SSLCacheProvider::CacheContext* context); + + private: + + apache::thrift::transport::SSLContext* ctx_; + std::shared_ptr localCache_; + PendingLookupMap pendingLookups_; + SSLStats* stats_{nullptr}; + std::shared_ptr externalCache_; + + /** + * Invoked by openssl when a new SSL session is created + */ + int newSession(SSL* ssl, SSL_SESSION* session); + + /** + * Invoked by openssl when an SSL session is ejected from its internal cache. + * This can't be invoked in the current implementation because SSL's internal + * caching is disabled. + */ + void removeSession(SSL_CTX* ctx, SSL_SESSION* session); + + /** + * Invoked by openssl when a client requests a stateful session resumption. + * Triggers a lookup in our local cache and potentially an asynchronous + * request to an external cache. + */ + SSL_SESSION* getSession(SSL* ssl, unsigned char* session_id, + int id_len, int* copyflag); + + /** + * Store a new session record in the external cache + */ + bool storeCacheRecord(const std::string& sessionId, SSL_SESSION* session); + + /** + * Lookup a session in the external cache for the specified SSL socket. + */ + bool lookupCacheRecord(const std::string& sessionId, + apache::thrift::async::TAsyncSSLSocket* sslSock); + + /** + * Restart all clients waiting for the answer to an external cache query + */ + void restartSSLAccept(const SSLCacheProvider::CacheContext* cacheCtx); + + /** + * Get or create the LRU cache for the given VIP ID + */ + static std::shared_ptr getLocalCache( + uint32_t maxCacheSize, uint32_t cacheCullSize); + + /** + * static functions registered as callbacks to openssl via + * SSL_CTX_sess_set_new/get/remove_cb + */ + static int newSessionCallback(SSL* ssl, SSL_SESSION* session); + static void removeSessionCallback(SSL_CTX* ctx, SSL_SESSION* session); + static SSL_SESSION* getSessionCallback(SSL* ssl, unsigned char* session_id, + int id_len, int* copyflag); + + static int32_t sExDataIndex_; + static std::shared_ptr sCache_; + static std::mutex sCacheLock_; +}; + +} diff --git a/proxygen/lib/ssl/SSLStats.h b/proxygen/lib/ssl/SSLStats.h new file mode 100644 index 0000000000..d7afd96eb0 --- /dev/null +++ b/proxygen/lib/ssl/SSLStats.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +class SSLStats { + public: + virtual ~SSLStats() noexcept {} + + // downstream + virtual void recordSSLAcceptLatency(int64_t latency) noexcept = 0; + virtual void recordTLSTicket(bool ticketNew, bool ticketHit) noexcept = 0; + virtual void recordSSLSession(bool sessionNew, bool sessionHit, bool foreign) + noexcept = 0; + virtual void recordSSLSessionRemove() noexcept = 0; + virtual void recordSSLSessionFree(uint32_t freed) noexcept = 0; + virtual void recordSSLSessionSetError(uint32_t err) noexcept = 0; + virtual void recordSSLSessionGetError(uint32_t err) noexcept = 0; + virtual void recordClientRenegotiation() noexcept = 0; + + // upstream + virtual void recordSSLUpstreamConnection(bool handshake) noexcept = 0; + virtual void recordSSLUpstreamConnectionError(bool verifyError) noexcept = 0; + virtual void recordCryptoSSLExternalAttempt() noexcept = 0; + virtual void recordCryptoSSLExternalConnAlreadyClosed() noexcept = 0; + virtual void recordCryptoSSLExternalApplicationException() noexcept = 0; + virtual void recordCryptoSSLExternalSuccess() noexcept = 0; + virtual void recordCryptoSSLExternalDuration(uint64_t duration) noexcept = 0; + virtual void recordCryptoSSLLocalAttempt() noexcept = 0; + virtual void recordCryptoSSLLocalSuccess() noexcept = 0; + +}; + +} diff --git a/proxygen/lib/ssl/SSLUtil.cpp b/proxygen/lib/ssl/SSLUtil.cpp new file mode 100644 index 0000000000..d94d2e9949 --- /dev/null +++ b/proxygen/lib/ssl/SSLUtil.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/ssl/SSLUtil.h" + +#include + +#if OPENSSL_VERSION_NUMBER >= 0x1000105fL +#define OPENSSL_GE_101 1 +#include +#include +#else +#undef OPENSSL_GE_101 +#endif + +namespace proxygen { + +std::mutex SSLUtil::sIndexLock_; + +std::unique_ptr SSLUtil::getCommonName(const X509* cert) { + X509_NAME* subject = X509_get_subject_name((X509*)cert); + if (!subject) { + return nullptr; + } + char cn[ub_common_name + 1]; + int res = X509_NAME_get_text_by_NID(subject, NID_commonName, + cn, ub_common_name); + if (res <= 0) { + return nullptr; + } else { + cn[ub_common_name] = '\0'; + return folly::make_unique(cn); + } +} + +std::unique_ptr> SSLUtil::getSubjectAltName( + const X509* cert) { +#ifdef OPENSSL_GE_101 + auto nameList = folly::make_unique>(); + GENERAL_NAMES* names = (GENERAL_NAMES*)X509_get_ext_d2i( + (X509*)cert, NID_subject_alt_name, nullptr, nullptr); + if (names) { + auto guard = folly::makeGuard([names] { GENERAL_NAMES_free(names); }); + size_t count = sk_GENERAL_NAME_num(names); + CHECK(count < std::numeric_limits::max()); + for (int i = 0; i < (int)count; ++i) { + GENERAL_NAME* generalName = sk_GENERAL_NAME_value(names, i); + if (generalName->type == GEN_DNS) { + ASN1_STRING* s = generalName->d.dNSName; + const char* name = (const char*)ASN1_STRING_data(s); + // I can't find any docs on what a negative return value here + // would mean, so I'm going to ignore it. + auto len = ASN1_STRING_length(s); + DCHECK(len >= 0); + if (size_t(len) != strlen(name)) { + // Null byte(s) in the name; return an error rather than depending on + // the caller to safely handle this case. + return nullptr; + } + nameList->emplace_back(name); + } + } + } + return nameList; +#else + return nullptr; +#endif +} + +} diff --git a/proxygen/lib/ssl/SSLUtil.h b/proxygen/lib/ssl/SSLUtil.h new file mode 100644 index 0000000000..f18c43e44b --- /dev/null +++ b/proxygen/lib/ssl/SSLUtil.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +namespace proxygen { + +/** + * SSL session establish/resume status + * + * changing these values will break logging pipelines + */ +enum class SSLResumeEnum : uint8_t { + HANDSHAKE = 0, + RESUME_SESSION_ID = 1, + RESUME_TICKET = 3, + NA = 2 +}; + +enum class SSLErrorEnum { + NO_ERROR, + TIMEOUT, + DROPPED +}; + +class SSLUtil { + private: + static std::mutex sIndexLock_; + + public: + /** + * Ensures only one caller will allocate an ex_data index for a given static + * or global. + */ + static void getSSLCtxExIndex(int* pindex) { + std::lock_guard g(sIndexLock_); + if (*pindex < 0) { + *pindex = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, nullptr); + } + } + + static void getRSAExIndex(int* pindex) { + std::lock_guard g(sIndexLock_); + if (*pindex < 0) { + *pindex = RSA_get_ex_new_index(0, nullptr, nullptr, nullptr, nullptr); + } + } + + static inline std::string hexlify(const std::string& binary) { + std::string hex; + folly::hexlify(binary, hex); + + return hex; + } + + static inline const std::string& hexlify(const std::string& binary, + std::string& hex) { + folly::hexlify(binary, hex); + + return hex; + } + + /** + * Return the SSL resume type for the given socket. + */ + static inline SSLResumeEnum getResumeState( + apache::thrift::async::TAsyncSSLSocket* sslSocket) { + return sslSocket->getSSLSessionReused() ? + (sslSocket->sessionIDResumed() ? + SSLResumeEnum::RESUME_SESSION_ID : + SSLResumeEnum::RESUME_TICKET) : + SSLResumeEnum::HANDSHAKE; + } + + /** + * Get the Common Name from an X.509 certificate + * @param cert certificate to inspect + * @return common name, or null if an error occurs + */ + static std::unique_ptr getCommonName(const X509* cert); + + /** + * Get the Subject Alternative Name value(s) from an X.509 certificate + * @param cert certificate to inspect + * @return set of zero or more alternative names, or null if + * an error occurs + */ + static std::unique_ptr> getSubjectAltName( + const X509* cert); +}; + +} diff --git a/proxygen/lib/ssl/TLSTicketKeyManager.cpp b/proxygen/lib/ssl/TLSTicketKeyManager.cpp new file mode 100644 index 0000000000..04d0e34610 --- /dev/null +++ b/proxygen/lib/ssl/TLSTicketKeyManager.cpp @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/ssl/TLSTicketKeyManager.h" + +#include "proxygen/lib/ssl/SSLStats.h" +#include "proxygen/lib/ssl/SSLUtil.h" + +#include +#include +#include +#include +#include + +#ifdef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB +using std::string; +using apache::thrift::async::TAsyncTimeout; +using folly::EventBase; +using apache::thrift::transport::SSLContext; + +namespace { + +const int kTLSTicketKeyNameLen = 4; +const int kTLSTicketKeySaltLen = 12; + +} + +namespace proxygen { + + +// TLSTicketKeyManager Implementation +int32_t TLSTicketKeyManager::sExDataIndex_ = -1; + +TLSTicketKeyManager::TLSTicketKeyManager(SSLContext* ctx, SSLStats* stats) + : ctx_(ctx), + randState_(0), + stats_(stats) { + SSLUtil::getSSLCtxExIndex(&sExDataIndex_); + SSL_CTX_set_ex_data(ctx_->getSSLCtx(), sExDataIndex_, this); +} + +TLSTicketKeyManager::~TLSTicketKeyManager() { +} + +int +TLSTicketKeyManager::callback(SSL* ssl, unsigned char* keyName, + unsigned char* iv, + EVP_CIPHER_CTX* cipherCtx, + HMAC_CTX* hmacCtx, int encrypt) { + TLSTicketKeyManager* manager = nullptr; + SSL_CTX* ctx = SSL_get_SSL_CTX(ssl); + manager = (TLSTicketKeyManager *)SSL_CTX_get_ex_data(ctx, sExDataIndex_); + + if (manager == nullptr) { + LOG(FATAL) << "Null TLSTicketKeyManager in callback" ; + return -1; + } + return manager->processTicket(ssl, keyName, iv, cipherCtx, hmacCtx, encrypt); +} + +int +TLSTicketKeyManager::processTicket(SSL* ssl, unsigned char* keyName, + unsigned char* iv, + EVP_CIPHER_CTX* cipherCtx, + HMAC_CTX* hmacCtx, int encrypt) { + uint8_t salt[kTLSTicketKeySaltLen]; + uint8_t* saltptr = nullptr; + uint8_t output[SHA256_DIGEST_LENGTH]; + uint8_t* hmacKey = nullptr; + uint8_t* aesKey = nullptr; + TLSTicketKeySource* key = nullptr; + int result = 0; + + if (encrypt) { + key = findEncryptionKey(); + if (key == nullptr) { + // no keys available to encrypt + VLOG(2) << "No TLS ticket key found"; + return -1; + } + VLOG(4) << "Encrypting new ticket with key name=" << + SSLUtil::hexlify(key->keyName_); + + // Get a random salt and write out key name + RAND_pseudo_bytes(salt, (int)sizeof(salt)); + memcpy(keyName, key->keyName_.data(), kTLSTicketKeyNameLen); + memcpy(keyName + kTLSTicketKeyNameLen, salt, kTLSTicketKeySaltLen); + + // Create the unique keys by hashing with the salt + makeUniqueKeys(key->keySource_, sizeof(key->keySource_), salt, output); + // This relies on the fact that SHA256 has 32 bytes of output + // and that AES-128 keys are 16 bytes + hmacKey = output; + aesKey = output + SHA256_DIGEST_LENGTH / 2; + + // Initialize iv and cipher/mac CTX + RAND_pseudo_bytes(iv, AES_BLOCK_SIZE); + HMAC_Init_ex(hmacCtx, hmacKey, SHA256_DIGEST_LENGTH / 2, + EVP_sha256(), nullptr); + EVP_EncryptInit_ex(cipherCtx, EVP_aes_128_cbc(), nullptr, aesKey, iv); + + result = 1; + } else { + key = findDecryptionKey(keyName); + if (key == nullptr) { + // no ticket found for decryption - will issue a new ticket + if (VLOG_IS_ON(4)) { + string skeyName((char *)keyName, kTLSTicketKeyNameLen); + VLOG(4) << "Can't find ticket key with name=" << + SSLUtil::hexlify(skeyName)<< ", will generate new ticket"; + } + + result = 0; + } else { + VLOG(4) << "Decrypting ticket with key name=" << + SSLUtil::hexlify(key->keyName_); + + // Reconstruct the unique key via the salt + saltptr = keyName + kTLSTicketKeyNameLen; + makeUniqueKeys(key->keySource_, sizeof(key->keySource_), saltptr, output); + hmacKey = output; + aesKey = output + SHA256_DIGEST_LENGTH / 2; + + // Initialize cipher/mac CTX + HMAC_Init_ex(hmacCtx, hmacKey, SHA256_DIGEST_LENGTH / 2, + EVP_sha256(), nullptr); + EVP_DecryptInit_ex(cipherCtx, EVP_aes_128_cbc(), nullptr, aesKey, iv); + + result = 1; + } + } + // result records whether a ticket key was found to decrypt this ticket, + // not wether the session was re-used. + if (stats_) { + stats_->recordTLSTicket(encrypt, result); + } + + return result; +} + +bool +TLSTicketKeyManager::setTLSTicketKeySeeds( + const std::vector& oldSeeds, + const std::vector& currentSeeds, + const std::vector& newSeeds) { + + bool result = true; + + activeKeys_.clear(); + ticketKeys_.clear(); + ticketSeeds_.clear(); + const std::vector *seedList = &oldSeeds; + for (uint32_t i = 0; i < 3; i++) { + TLSTicketSeedType type = (TLSTicketSeedType)i; + if (type == SEED_CURRENT) { + seedList = ¤tSeeds; + } else if (type == SEED_NEW) { + seedList = &newSeeds; + } + + for (const auto& seedInput: *seedList) { + TLSTicketSeed* seed = insertSeed(seedInput, type); + if (seed == nullptr) { + result = false; + continue; + } + insertNewKey(seed, 1, nullptr); + } + } + if (!result) { + VLOG(2) << "One or more seeds failed to decode"; + } + + if (ticketKeys_.size() == 0 || activeKeys_.size() == 0) { + LOG(WARNING) << "No keys configured, falling back to default"; + SSL_CTX_set_tlsext_ticket_key_cb(ctx_->getSSLCtx(), nullptr); + return false; + } + SSL_CTX_set_tlsext_ticket_key_cb(ctx_->getSSLCtx(), + TLSTicketKeyManager::callback); + + return true; +} + +string +TLSTicketKeyManager::makeKeyName(TLSTicketSeed* seed, uint32_t n, + unsigned char* nameBuf) { + SHA256_CTX ctx; + + SHA256_Init(&ctx); + SHA256_Update(&ctx, seed->seedName_, sizeof(seed->seedName_)); + SHA256_Update(&ctx, &n, sizeof(n)); + SHA256_Final(nameBuf, &ctx); + return string((char *)nameBuf, kTLSTicketKeyNameLen); +} + +TLSTicketKeyManager::TLSTicketKeySource* +TLSTicketKeyManager::insertNewKey(TLSTicketSeed* seed, uint32_t hashCount, + TLSTicketKeySource* prevKey) { + unsigned char nameBuf[SHA256_DIGEST_LENGTH]; + std::unique_ptr newKey(new TLSTicketKeySource); + + // This function supports hash chaining but it is not currently used. + + if (prevKey != nullptr) { + hashNth(prevKey->keySource_, sizeof(prevKey->keySource_), + newKey->keySource_, 1); + } else { + // can't go backwards or the current is missing, start from the beginning + hashNth((unsigned char *)seed->seed_.data(), seed->seed_.length(), + newKey->keySource_, hashCount); + } + + newKey->hashCount_ = hashCount; + newKey->keyName_ = makeKeyName(seed, hashCount, nameBuf); + newKey->type_ = seed->type_; + auto it = ticketKeys_.insert(std::make_pair(newKey->keyName_, + std::move(newKey))); + + auto key = it.first->second.get(); + if (key->type_ == SEED_CURRENT) { + activeKeys_.push_back(key); + } + VLOG(4) << "Adding key for " << hashCount << " type=" << + (uint32_t)key->type_ << " Name=" << SSLUtil::hexlify(key->keyName_); + + return key; +} + +void +TLSTicketKeyManager::hashNth(const unsigned char* input, size_t input_len, + unsigned char* output, uint32_t n) { + assert(n > 0); + for (uint32_t i = 0; i < n; i++) { + SHA256(input, input_len, output); + input = output; + input_len = SHA256_DIGEST_LENGTH; + } +} + +TLSTicketKeyManager::TLSTicketSeed * +TLSTicketKeyManager::insertSeed(const string& seedInput, + TLSTicketSeedType type) { + TLSTicketSeed* seed = nullptr; + string seedOutput; + + if (!folly::unhexlify(seedInput, seedOutput)) { + LOG(WARNING) << "Failed to decode seed type=" << (uint32_t)type << + " seed=" << seedInput; + return seed; + } + + seed = new TLSTicketSeed(); + seed->seed_ = seedOutput; + seed->type_ = type; + SHA256((unsigned char *)seedOutput.data(), seedOutput.length(), + seed->seedName_); + ticketSeeds_.push_back(std::unique_ptr(seed)); + + return seed; +} + +TLSTicketKeyManager::TLSTicketKeySource * +TLSTicketKeyManager::findEncryptionKey() { + TLSTicketKeySource* result = nullptr; + // call to rand here is a bit hokey since it's not cryptographically + // random, and is predictably seeded with 0. However, activeKeys_ + // is probably not going to have very many keys in it, and most + // likely only 1. + size_t numKeys = activeKeys_.size(); + if (numKeys > 0) { + result = activeKeys_[rand_r(&randState_) % numKeys]; + } + return result; +} + +TLSTicketKeyManager::TLSTicketKeySource * +TLSTicketKeyManager::findDecryptionKey(unsigned char* keyName) { + string name((char *)keyName, kTLSTicketKeyNameLen); + TLSTicketKeySource* key = nullptr; + TLSTicketKeyMap::iterator mapit = ticketKeys_.find(name); + if (mapit != ticketKeys_.end()) { + key = mapit->second.get(); + } + return key; +} + +void +TLSTicketKeyManager::makeUniqueKeys(unsigned char* parentKey, + size_t keyLen, + unsigned char* salt, + unsigned char* output) { + SHA256_CTX hash_ctx; + + SHA256_Init(&hash_ctx); + SHA256_Update(&hash_ctx, parentKey, keyLen); + SHA256_Update(&hash_ctx, salt, kTLSTicketKeySaltLen); + SHA256_Final(output, &hash_ctx); +} + +} // proxygen +#endif diff --git a/proxygen/lib/ssl/TLSTicketKeyManager.h b/proxygen/lib/ssl/TLSTicketKeyManager.h new file mode 100644 index 0000000000..9e1ada4a4f --- /dev/null +++ b/proxygen/lib/ssl/TLSTicketKeyManager.h @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +#ifndef SSL_CTRL_SET_TLSEXT_TICKET_KEY_CB +class TLSTicketKeyManager {}; +#else +class SSLStats; +/** + * The TLSTicketKeyManager handles TLS ticket key encryption and decryption in + * a way that facilitates sharing the ticket keys across a range of servers. + * Hash chaining is employed to achieve frequent key rotation with minimal + * configuration change. The scheme is as follows: + * + * The manager is supplied with three lists of seeds (old, current and new). + * The config should be updated with new seeds periodically (e.g., daily). + * 3 config changes are recommended to achieve the smoothest seed rotation + * eg: + * 1. Introduce new seed in the push prior to rotation + * 2. Rotation push + * 3. Remove old seeds in the push following rotation + * + * Multiple seeds are supported but only a single seed is required. + * + * Generating encryption keys from the seed works as follows. For a given + * seed, hash forward N times where N is currently the constant 1. + * This is the base key. The name of the base key is the first 4 + * bytes of hash(hash(seed), N). This is copied into the first 4 bytes of the + * TLS ticket key name field. + * + * For each new ticket encryption, the manager generates a random 12 byte salt. + * Hash the salt and the base key together to form the encryption key for + * that ticket. The salt is included in the ticket's 'key name' field so it + * can be used to derive the decryption key. The salt is copied into the second + * 8 bytes of the TLS ticket key name field. + * + * A key is valid for decryption for the lifetime of the proxygen instance. + * Sessions will be valid for less time than that, which results in an extra + * symmetric decryption to discover the session is expired. + * + * A TLSTicketKeyManager should be used in only one thread, and should have + * a 1:1 relationship with the SSLContext provided. + * + */ +class TLSTicketKeyManager : private boost::noncopyable { + public: + + explicit TLSTicketKeyManager(apache::thrift::transport::SSLContext* ctx, + SSLStats* stats); + + virtual ~TLSTicketKeyManager(); + + /** + * SSL callback to set up encryption/decryption context for a TLS Ticket Key. + * + * This will be supplied to the SSL library via + * SSL_CTX_set_tlsext_ticket_key_cb. + */ + static int callback(SSL* ssl, unsigned char* keyName, + unsigned char* iv, + EVP_CIPHER_CTX* cipherCtx, + HMAC_CTX* hmacCtx, int encrypt); + + /** + * Initialize the manager with three sets of seeds. There must be at least + * one current seed, or the manager will revert to the default SSL behavior. + * + * @param oldSeeds Seeds previously used which can still decrypt. + * @param currentSeeds Seeds to use for new ticket encryptions. + * @param newSeeds Seeds which will be used soon, can be used to decrypt + * in case some servers in the cluster have already rotated. + */ + bool setTLSTicketKeySeeds(const std::vector& oldSeeds, + const std::vector& currentSeeds, + const std::vector& newSeeds); + + private: + enum TLSTicketSeedType { + SEED_OLD = 0, + SEED_CURRENT, + SEED_NEW + }; + + /* The seeds supplied by the configuration */ + struct TLSTicketSeed { + std::string seed_; + TLSTicketSeedType type_; + unsigned char seedName_[SHA256_DIGEST_LENGTH]; + }; + + struct TLSTicketKeySource { + int32_t hashCount_; + std::string keyName_; + TLSTicketSeedType type_; + unsigned char keySource_[SHA256_DIGEST_LENGTH]; + }; + + /** + * Method to setup encryption/decryption context for a TLS Ticket Key + * + * OpenSSL documentation is thin on the return value semantics. + * + * For encrypt=1, return < 0 on error, >= 0 for successfully initialized + * For encrypt=0, return < 0 on error, 0 on key not found + * 1 on key found, 2 renew_ticket + * + * renew_ticket means a new ticket will be issued. We could return this value + * when receiving a ticket encrypted with a key derived from an OLD seed. + * However, session_timeout seconds after deploying proxygen with a seed + * rotated from CURRENT -> OLD, there will be no valid tickets outstanding + * encrypted with the old key. This grace period means no unnecessary + * handshakes will be performed. If the seed is believed compromised, it + * should NOT be configured as an OLD seed. + */ + int processTicket(SSL* ssl, unsigned char* keyName, + unsigned char* iv, + EVP_CIPHER_CTX* cipherCtx, + HMAC_CTX* hmacCtx, int encrypt); + + // Creates the name for the nth key generated from seed + std::string makeKeyName(TLSTicketSeed* seed, uint32_t n, + unsigned char* nameBuf); + + /** + * Creates the key hashCount hashes from the given seed and inserts it in + * ticketKeys. A naked pointer to the key is returned for additional + * processing if needed. + */ + TLSTicketKeySource* insertNewKey(TLSTicketSeed* seed, uint32_t hashCount, + TLSTicketKeySource* prevKeySource); + + /** + * hashes input N times placing result in output, which must be at least + * SHA256_DIGEST_LENGTH long. + */ + void hashNth(const unsigned char* input, size_t input_len, + unsigned char* output, uint32_t n); + + /** + * Adds the given seed to the manager + */ + TLSTicketSeed* insertSeed(const std::string& seedInput, + TLSTicketSeedType type); + + /** + * Locate a key for encrypting a new ticket + */ + TLSTicketKeySource* findEncryptionKey(); + + /** + * Locate a key for decrypting a ticket with the given keyName + */ + TLSTicketKeySource* findDecryptionKey(unsigned char* keyName); + + /** + * Derive a unique key from the parent key and the salt via hashing + */ + void makeUniqueKeys(unsigned char* parentKey, size_t keyLen, + unsigned char* salt, unsigned char* output); + + /** + * For standalone decryption utility + */ + friend int decrypt_fb_ticket(proxygen::TLSTicketKeyManager* manager, + const std::string& testTicket, + SSL_SESSION **psess); + + typedef std::vector> TLSTicketSeedList; + typedef std::map > + TLSTicketKeyMap; + typedef std::vector TLSActiveKeyList; + + TLSTicketSeedList ticketSeeds_; + // All key sources that can be used for decryption + TLSTicketKeyMap ticketKeys_; + // Key sources that can be used for encryption + TLSActiveKeyList activeKeys_; + + apache::thrift::transport::SSLContext* ctx_; + uint32_t randState_; + SSLStats* stats_{nullptr}; + + static int32_t sExDataIndex_; +}; +#endif +} diff --git a/proxygen/lib/ssl/TLSTicketKeySeeds.h b/proxygen/lib/ssl/TLSTicketKeySeeds.h new file mode 100644 index 0000000000..2715bd220f --- /dev/null +++ b/proxygen/lib/ssl/TLSTicketKeySeeds.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +struct TLSTicketKeySeeds { + std::vector oldSeeds; + std::vector currentSeeds; + std::vector newSeeds; +}; + +} diff --git a/proxygen/lib/ssl/test/Makefile.am b/proxygen/lib/ssl/test/Makefile.am new file mode 100644 index 0000000000..ad404fe4f7 --- /dev/null +++ b/proxygen/lib/ssl/test/Makefile.am @@ -0,0 +1,16 @@ +SUBDIRS = . + +check_PROGRAMS = SSLTests +SSLTests_SOURCES = \ + SSLContextManagerTest.cpp + +SSLTests_LDADD = \ + ../libproxygenssl.la \ + ../../utils/libutils.la \ + ../../test/libtestmain.la + +check_PROGRAMS += SSLCacheTest +SSLCacheTest_SOURCES = \ + SSLCacheTest.cpp + +TESTS = SSLTests diff --git a/proxygen/lib/ssl/test/SSLCacheTest.cpp b/proxygen/lib/ssl/test/SSLCacheTest.cpp new file mode 100644 index 0000000000..249d49587b --- /dev/null +++ b/proxygen/lib/ssl/test/SSLCacheTest.cpp @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +using apache::thrift::async::TAsyncSSLSocket; +using apache::thrift::async::TAsyncSocket; +using apache::thrift::transport::SSLContext; +using apache::thrift::transport::TTransportException; +using folly::EventBase; +using folly::SocketAddress; + +DEFINE_int32(clients, 1, "Number of simulated SSL clients"); +DEFINE_int32(threads, 1, "Number of threads to spread clients across"); +DEFINE_int32(requests, 2, "Total number of requests per client"); +DEFINE_int32(port, 9423, "Server port"); +DEFINE_bool(sticky, false, "A given client sends all reqs to one " + "(random) server"); +DEFINE_bool(global, false, "All clients in a thread use the same SSL session"); +DEFINE_bool(handshakes, false, "Force 100% handshakes"); + +string f_servers[10]; +int f_num_servers = 0; +int tnum = 0; + +class ClientRunner { + public: + + ClientRunner(): reqs(0), hits(0), miss(0), num(tnum++) {} + void run(); + + int reqs; + int hits; + int miss; + int num; +}; + +class SSLCacheClient : public TAsyncSocket::ConnectCallback, + public TAsyncSSLSocket::HandshakeCallback +{ +private: + EventBase* eventBase_; + int currReq_; + int serverIdx_; + TAsyncSocket* socket_; + TAsyncSSLSocket* sslSocket_; + SSL_SESSION* session_; + SSL_SESSION **pSess_; + std::shared_ptr ctx_; + ClientRunner* cr_; + +public: + SSLCacheClient(EventBase* eventBase, SSL_SESSION **pSess, ClientRunner* cr); + ~SSLCacheClient() { + if (session_ && !FLAGS_global) + SSL_SESSION_free(session_); + if (socket_ != nullptr) { + if (sslSocket_ != nullptr) { + sslSocket_->destroy(); + sslSocket_ = nullptr; + } + socket_->destroy(); + socket_ = nullptr; + } + }; + + void start(); + + virtual void connectSuccess() noexcept; + + virtual void connectError(const TTransportException& ex) + noexcept ; + + virtual void handshakeSuccess(TAsyncSSLSocket* sock) noexcept; + + virtual void handshakeError( + TAsyncSSLSocket* sock, + const apache::thrift::transport::TTransportException& ex) noexcept; + +}; + +int +main(int argc, char* argv[]) +{ + google::SetUsageMessage(std::string("\n\n" +"usage: sslcachetest [options] -c -t servers\n" +)); + google::ParseCommandLineFlags(&argc, &argv, true); + int reqs = 0; + int hits = 0; + int miss = 0; + struct timeval start; + struct timeval end; + struct timeval result; + + srand((unsigned int)time(nullptr)); + + for (int i = 1; i < argc; i++) { + f_servers[f_num_servers++] = argv[i]; + } + if (f_num_servers == 0) { + cout << "require at least one server\n"; + return 1; + } + + gettimeofday(&start, nullptr); + if (FLAGS_threads == 1) { + ClientRunner r; + r.run(); + gettimeofday(&end, nullptr); + reqs = r.reqs; + hits = r.hits; + miss = r.miss; + } + else { + std::vector clients; + std::vector threads; + for (int t = 0; t < FLAGS_threads; t++) { + threads.emplace_back([&] { + clients[t].run(); + }); + } + for (auto& thr: threads) { + thr.join(); + } + gettimeofday(&end, nullptr); + + for (const auto& client: clients) { + reqs += client.reqs; + hits += client.hits; + miss += client.miss; + } + } + + timersub(&end, &start, &result); + + cout << "Requests: " << reqs << endl; + cout << "Handshakes: " << miss << endl; + cout << "Resumes: " << hits << endl; + cout << "Runtime(ms): " << result.tv_sec << "." << result.tv_usec / 1000 << + endl; + + cout << "ops/sec: " << (reqs * 1.0) / + ((double)result.tv_sec * 1.0 + (double)result.tv_usec / 1000000.0) << endl; + + return 0; +} + +void +ClientRunner::run() +{ + EventBase eb; + std::list clients; + SSL_SESSION* session = nullptr; + + for (int i = 0; i < FLAGS_clients; i++) { + SSLCacheClient* c = new SSLCacheClient(&eb, &session, this); + c->start(); + clients.push_back(c); + } + + eb.loop(); + + for (auto it = clients.begin(); it != clients.end(); it++) { + delete* it; + } + + reqs += hits + miss; +} + +SSLCacheClient::SSLCacheClient(EventBase* eb, + SSL_SESSION **pSess, + ClientRunner* cr) + : eventBase_(eb), + currReq_(0), + serverIdx_(0), + socket_(nullptr), + sslSocket_(nullptr), + session_(nullptr), + pSess_(pSess), + cr_(cr) +{ + ctx_.reset(new SSLContext()); + ctx_->setOptions(SSL_OP_NO_TICKET); +} + +void +SSLCacheClient::start() +{ + if (currReq_ >= FLAGS_requests) { + cout << "+"; + return; + } + + if (currReq_ == 0 || !FLAGS_sticky) { + serverIdx_ = rand() % f_num_servers; + } + if (socket_ != nullptr) { + if (sslSocket_ != nullptr) { + sslSocket_->destroy(); + sslSocket_ = nullptr; + } + socket_->destroy(); + socket_ = nullptr; + } + socket_ = new TAsyncSocket(eventBase_); + socket_->connect(this, f_servers[serverIdx_], (uint16_t)FLAGS_port); +} + +void +SSLCacheClient::connectSuccess() noexcept +{ + sslSocket_ = new TAsyncSSLSocket(ctx_, eventBase_, socket_->detachFd(), + false); + + if (!FLAGS_handshakes) { + if (session_ != nullptr) + sslSocket_->setSSLSession(session_); + else if (FLAGS_global && pSess_ != nullptr) + sslSocket_->setSSLSession(*pSess_); + } + sslSocket_->sslConnect(this); +} + +void +SSLCacheClient::connectError(const TTransportException& ex) + noexcept +{ + cout << "connectError: " << ex.what() << endl; +} + +void +SSLCacheClient::handshakeSuccess(TAsyncSSLSocket* socket) noexcept +{ + if (sslSocket_->getSSLSessionReused()) { + cr_->hits++; + } else { + cr_->miss++; + if (session_ != nullptr) { + SSL_SESSION_free(session_); + } + session_ = sslSocket_->getSSLSession(); + if (FLAGS_global && pSess_ != nullptr && *pSess_ == nullptr) { + *pSess_ = session_; + } + } + if ( ((cr_->hits + cr_->miss) % 100) == ((100 / FLAGS_threads) * cr_->num)) { + cout << "."; + cout.flush(); + } + sslSocket_->closeNow(); + currReq_++; + this->start(); +} + +void +SSLCacheClient::handshakeError( + TAsyncSSLSocket* sock, + const apache::thrift::transport::TTransportException& ex) + noexcept +{ + cout << "handshakeError: " << ex.what() << endl; +} diff --git a/proxygen/lib/ssl/test/SSLContextManagerTest.cpp b/proxygen/lib/ssl/test/SSLContextManagerTest.cpp new file mode 100644 index 0000000000..eec34f34fe --- /dev/null +++ b/proxygen/lib/ssl/test/SSLContextManagerTest.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/ssl/SSLContextManager.h" +#include "proxygen/lib/utils/DomainNameMisc.h" + +#include +#include +#include +#include + +using apache::thrift::transport::SSLContext; +using folly::EventBase; +using proxygen::DNString; +using proxygen::SSLContextManager; +using std::shared_ptr; + +TEST(SSLContextManagerTest, Test1) +{ + EventBase eventBase; + SSLContextManager sslCtxMgr(&eventBase, "vip_ssl_context_manager_test_", + true, nullptr); + auto www_facebook_com_ctx = std::make_shared(); + auto start_facebook_com_ctx = std::make_shared(); + auto start_abc_facebook_com_ctx = std::make_shared(); + + sslCtxMgr.insertSSLCtxByDomainName( + "www.facebook.com", + strlen("www.facebook.com"), + www_facebook_com_ctx); + sslCtxMgr.insertSSLCtxByDomainName( + "www.facebook.com", + strlen("www.facebook.com"), + www_facebook_com_ctx); + try { + sslCtxMgr.insertSSLCtxByDomainName( + "www.facebook.com", + strlen("www.facebook.com"), + std::make_shared()); + } catch (const std::exception& ex) { + } + sslCtxMgr.insertSSLCtxByDomainName( + "*.facebook.com", + strlen("*.facebook.com"), + start_facebook_com_ctx); + sslCtxMgr.insertSSLCtxByDomainName( + "*.abc.facebook.com", + strlen("*.abc.facebook.com"), + start_abc_facebook_com_ctx); + try { + sslCtxMgr.insertSSLCtxByDomainName( + "*.abc.facebook.com", + strlen("*.abc.facebook.com"), + std::make_shared()); + FAIL(); + } catch (const std::exception& ex) { + } + + shared_ptr retCtx; + retCtx = sslCtxMgr.getSSLCtx(DNString("www.facebook.com")); + EXPECT_EQ(retCtx, www_facebook_com_ctx); + retCtx = sslCtxMgr.getSSLCtx(DNString("WWW.facebook.com")); + EXPECT_EQ(retCtx, www_facebook_com_ctx); + EXPECT_FALSE(sslCtxMgr.getSSLCtx(DNString("xyz.facebook.com"))); + + retCtx = sslCtxMgr.getSSLCtxBySuffix(DNString("xyz.facebook.com")); + EXPECT_EQ(retCtx, start_facebook_com_ctx); + retCtx = sslCtxMgr.getSSLCtxBySuffix(DNString("XYZ.facebook.com")); + EXPECT_EQ(retCtx, start_facebook_com_ctx); + + retCtx = sslCtxMgr.getSSLCtxBySuffix(DNString("www.abc.facebook.com")); + EXPECT_EQ(retCtx, start_abc_facebook_com_ctx); + + // ensure "facebook.com" does not match "*.facebook.com" + EXPECT_FALSE(sslCtxMgr.getSSLCtxBySuffix(DNString("facebook.com"))); + // ensure "Xfacebook.com" does not match "*.facebook.com" + EXPECT_FALSE(sslCtxMgr.getSSLCtxBySuffix(DNString("Xfacebook.com"))); + // ensure wildcard name only matches one domain up + EXPECT_FALSE(sslCtxMgr.getSSLCtxBySuffix(DNString("abc.xyz.facebook.com"))); + + eventBase.loop(); // Clean up events before SSLContextManager is destructed +} diff --git a/proxygen/lib/test/Makefile.am b/proxygen/lib/test/Makefile.am new file mode 100644 index 0000000000..947850a35c --- /dev/null +++ b/proxygen/lib/test/Makefile.am @@ -0,0 +1,27 @@ +SUBDIRS = . + +BUILT_SOURCES = gmock-1.6.0/gtest/src/gtest-all.cc + +gmock-1.6.0/gtest/src/gtest-all.cc: + wget https://googlemock.googlecode.com/files/gmock-1.6.0.zip + unzip gmock-1.6.0.zip + + +check_LTLIBRARIES = libtesttransport.la +libtesttransport_la_SOURCES = TestAsyncTransport.cpp + +libtesttransportdir = $(includedir)/proxygen/lib/test +nobase_libtesttransport_HEADERS = TestAsyncTransport.h + +# libgmockgtest.la is gmock + gtest + +check_LTLIBRARIES += libgmockgtest.la +libgmockgtest_la_CPPFLAGS = -Igmock-1.6.0/gtest -Igmock-1.6.0 -Igmock-1.6.0/gtest/include -Igmock-1.6.0/include -lglog + +libgmockgtest_la_SOURCES = \ + gmock-1.6.0/gtest/src/gtest-all.cc \ + gmock-1.6.0/src/gmock-all.cc + +check_LTLIBRARIES += libtestmain.la +libtestmain_la_SOURCES = TestMain.cpp +libtestmain_la_LIBADD = libgmockgtest.la diff --git a/proxygen/lib/test/TestAsyncTransport.cpp b/proxygen/lib/test/TestAsyncTransport.cpp new file mode 100644 index 0000000000..7eb56a781d --- /dev/null +++ b/proxygen/lib/test/TestAsyncTransport.cpp @@ -0,0 +1,635 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/test/TestAsyncTransport.h" + +#include +#include +#include +#include +#include + +using apache::thrift::async::TAsyncTimeout; +using apache::thrift::async::WriteFlags; +using apache::thrift::concurrency::Util; +using apache::thrift::transport::TTransportException; +using folly::EventBase; +using folly::IOBuf; +using folly::SocketAddress; +using std::shared_ptr; +using std::unique_ptr; + +/* + * TestAsyncTransport::ReadEvent + */ + +class TestAsyncTransport::ReadEvent { + public: + ReadEvent(const void* buf, size_t buflen, int64_t delay) + : buffer_(nullptr), + readStart_(nullptr), + dataEnd_(nullptr), + isError_(false), + exception_(), + delay_(delay) { + if (buflen == 0) { + // This means EOF + return; + } + CHECK_NOTNULL(buf); + + buffer_ = static_cast(malloc(buflen)); + if (buffer_ == nullptr) { + throw std::bad_alloc(); + } + memcpy(buffer_, buf, buflen); + readStart_ = buffer_; + dataEnd_ = buffer_ + buflen; + } + + ReadEvent(const TTransportException& ex, int64_t delay) + : buffer_(nullptr), + readStart_(nullptr), + dataEnd_(nullptr), + isError_(true), + exception_(ex), + delay_(delay) {} + + ~ReadEvent() { + free(buffer_); + } + + int64_t getDelay() const { + return delay_; + } + + bool isFinalEvent() const { + return buffer_ == nullptr; + } + + const char* getBuffer() const { + return readStart_; + } + + size_t getLength() const { + return (dataEnd_ - readStart_); + } + + void consumeData(size_t length) { + CHECK_LE(readStart_ + length, dataEnd_); + readStart_ += length; + } + + bool isError() const { + return isError_; + } + + const TTransportException& getException() const { + return exception_; + } + + private: + char* buffer_; + char* readStart_; + char* dataEnd_; + + bool isError_; + TTransportException exception_; + + int64_t delay_; // milliseconds +}; + +/* + * TestAsyncTransport::WriteEvent methods + */ + +TestAsyncTransport::WriteEvent::WriteEvent(int64_t time, size_t count) + : time_(time), + count_(count) { + // Initialize all of the iov_base pointers to nullptr. This way we won't free + // an uninitialized pointer in the destructor if we fail to allocate any of + // the buffers. + for (size_t n = 0; n < count_; ++n) { + vec_[n].iov_base = nullptr; + } +} + +TestAsyncTransport::WriteEvent::~WriteEvent() { + for (size_t n = 0; n < count_; ++n) { + free(vec_[n].iov_base); + } +} + +shared_ptr +TestAsyncTransport::WriteEvent::newEvent(const struct iovec* vec, + size_t count) { + size_t len = sizeof(WriteEvent) + (count * sizeof(struct iovec)); + void* buf = malloc(len); + if (buf == nullptr) { + throw std::bad_alloc(); + } + + int64_t now = Util::currentTime(); + shared_ptr event(new(buf) WriteEvent(now, count), + destroyEvent); + for (size_t n = 0; n < count; ++n) { + size_t len = vec[n].iov_len; + event->vec_[n].iov_len = len; + if (len == 0) { + event->vec_[n].iov_base = nullptr; + continue; + } + + event->vec_[n].iov_base = malloc(len); + if (event->vec_[n].iov_base == nullptr) { + throw std::bad_alloc(); + } + memcpy(event->vec_[n].iov_base, vec[n].iov_base, len); + } + + return event; +} + +void +TestAsyncTransport::WriteEvent::destroyEvent(WriteEvent* event) { + event->~WriteEvent(); + free(event); +} + +/* + * TestAsyncTransport methods + */ + +TestAsyncTransport::TestAsyncTransport(EventBase* eventBase) + : TAsyncTimeout(eventBase), + eventBase_(eventBase), + readCallback_(nullptr), + sendTimeout_(0), + prevReadEventTime_(0), + nextReadEventTime_(0), + readState_(kStateOpen), + writeState_(kStateOpen), + readEvents_() +{ +} + +void +TestAsyncTransport::setReadCallback(ReadCallback* callback) { + if (readCallback_ == callback) { + return; + } + + if (callback == nullptr) { + cancelTimeout(); + readCallback_ = nullptr; + return; + } + + bool wasNull = (readCallback_ == nullptr); + + if (readState_ == kStateClosed) { + callback->readEOF(); + return; + } else if (readState_ == kStateError) { + TTransportException ex(TTransportException::NOT_OPEN, + "setReadCallback() called with socket in " + "invalid state"); + callback->readError(ex); + return; + } + + CHECK_EQ(readState_, kStateOpen); + readCallback_ = callback; + + // If the callback was previously nullptr, read events were paused, so we need + // to reschedule them now. + // + // If it was set before, read events are still scheduled, so we are done now + // and can return. + if (!wasNull) { + return; + } + + if (nextReadEventTime_ == 0) { + // Either readEvents_ is empty, or startReadEvents() hasn't been called yet + return; + } + CHECK(!readEvents_.empty()); + scheduleNextReadEvent(Util::currentTime()); +} + +TestAsyncTransport::ReadCallback* +TestAsyncTransport::getReadCallback() const { + return readCallback_; +} + +void +TestAsyncTransport::write(WriteCallback* callback, + const void* buf, size_t bytes, + WriteFlags flags) { + iovec op; + op.iov_base = const_cast(buf); + op.iov_len = bytes; + this->writev(callback, &op, 1, flags); +} + +void +TestAsyncTransport::writev(WriteCallback* callback, + const iovec* vec, size_t count, + WriteFlags flags) { + if (isSet(flags, WriteFlags::CORK)) { + corkCount_++; + } else if (isSet(flags, WriteFlags::EOR)) { + eorCount_++; + } + if (!writesAllowed()) { + TTransportException ex(TTransportException::NOT_OPEN, + "write() called on non-open TestAsyncTransport"); + callback->writeError(0, ex); + return; + } + + shared_ptr event = WriteEvent::newEvent(vec, count); + if (writeState_ == kStatePaused || pendingWriteEvents_.size() > 0) { + pendingWriteEvents_.push_back(std::make_pair(event, callback)); + } else { + CHECK(writeState_ == kStateOpen); + writeEvents_.push_back(event); + callback->writeSuccess(); + } +} + +void +TestAsyncTransport::writeChain(WriteCallback* callback, + std::unique_ptr&& iob, + WriteFlags flags) { + size_t count = iob->countChainElements(); + iovec vec[count]; + const IOBuf* head = iob.get(); + const IOBuf* next = head; + unsigned i = 0; + do { + vec[i].iov_base = const_cast(next->buffer()); + vec[i++].iov_len = next->length(); + next = next->next(); + } while (next != head); + this->writev(callback, vec, count, flags); +} + +void +TestAsyncTransport::close() { + closeNow(); +} + +void +TestAsyncTransport::closeNow() { + if (readState_ == kStateOpen) { + readState_ = kStateClosed; + + if (readCallback_ != nullptr) { + ReadCallback* callback = readCallback_; + readCallback_ = nullptr; + callback->readEOF(); + } + } + shutdownWriteNow(); +} + +void +TestAsyncTransport::shutdownWrite() { + shutdownWriteNow(); +} + +void +TestAsyncTransport::shutdownWriteNow() { + DestructorGuard g(this); + failPendingWrites(); + if (writeState_ == kStateOpen || + writeState_ == kStatePaused) { + writeState_ = kStateClosed; + } +} + +void +TestAsyncTransport::getPeerAddress(SocketAddress* addr) const { + // This isn't really accurate, but close enough for testing. + addr->setFromIpPort("127.0.0.1", 0); +} + +void +TestAsyncTransport::getLocalAddress(SocketAddress* addr) const { + // This isn't really accurate, but close enough for testing. + addr->setFromIpPort("127.0.0.1", 0); +} + +bool +TestAsyncTransport::good() const { + return (readState_ == kStateOpen && writesAllowed()); +} + +bool +TestAsyncTransport::readable() const { + return false; +} + +bool +TestAsyncTransport::connecting() const { + return false; +} + +bool +TestAsyncTransport::error() const { + return (readState_ == kStateError || writeState_ == kStateError); +} + +void +TestAsyncTransport::attachEventBase(EventBase* eventBase) { + CHECK(nullptr == eventBase_); + CHECK(nullptr == readCallback_); + eventBase_ = eventBase; +} + +void +TestAsyncTransport::detachEventBase() { + CHECK_NOTNULL(eventBase_); + CHECK(nullptr == readCallback_); + eventBase_ = nullptr; +} + +bool +TestAsyncTransport::isDetachable() const { + return true; +} + +EventBase* +TestAsyncTransport::getEventBase() const { + return eventBase_; +} + +void +TestAsyncTransport::setSendTimeout(uint32_t milliseconds) { + sendTimeout_ = milliseconds; +} + +uint32_t +TestAsyncTransport::getSendTimeout() const { + return sendTimeout_; +} + +void +TestAsyncTransport::pauseWrites() { + if (writeState_ != kStateOpen) { + LOG(FATAL) << "cannot pause writes on non-open transport; state=" << + writeState_; + } + writeState_ = kStatePaused; +} + +void +TestAsyncTransport::resumeWrites() { + if (writeState_ != kStatePaused) { + LOG(FATAL) << "cannot resume writes on non-paused transport; state=" << + writeState_; + } + writeState_ = kStateOpen; + for (auto event = pendingWriteEvents_.begin(); + event != pendingWriteEvents_.end() && writeState_ == kStateOpen; + event = pendingWriteEvents_.begin()) { + writeEvents_.push_back(event->first); + pendingWriteEvents_.pop_front(); + event->second->writeSuccess(); + } +} + +void +TestAsyncTransport::failPendingWrites() { + // writeError() callback might try to delete this object + DestructorGuard g(this); + while (!pendingWriteEvents_.empty()) { + auto event = pendingWriteEvents_.front(); + pendingWriteEvents_.pop_front(); + TTransportException ex(TTransportException::NOT_OPEN, + "Transport closed locally"); + event.second->writeError(0, ex); + } +} + +void +TestAsyncTransport::addReadEvent(folly::IOBufQueue& chain, + int64_t delayFromPrevious) { + while (true) { + unique_ptr cur = chain.pop_front(); + if (!cur) { + break; + } + addReadEvent(cur->data(), cur->length(), delayFromPrevious); + } +} + +void +TestAsyncTransport::addReadEvent(const void* buf, size_t buflen, + int64_t delayFromPrevious) { + if (!readEvents_.empty() && readEvents_.back()->isFinalEvent()) { + LOG(FATAL) << "cannot add more read events after an error or EOF"; + } + + auto event = std::make_shared(buf, buflen, delayFromPrevious); + addReadEvent(event); +} + +void +TestAsyncTransport::addReadEvent(const char* buf, int64_t delayFromPrevious) { + addReadEvent(buf, strlen(buf), delayFromPrevious); +} + +void +TestAsyncTransport::addReadEOF(int64_t delayFromPrevious) { + addReadEvent(nullptr, 0, delayFromPrevious); +} + +void +TestAsyncTransport::addReadError(const TTransportException& ex, + int64_t delayFromPrevious) { + if (!readEvents_.empty() && readEvents_.back()->isFinalEvent()) { + LOG(FATAL) << "cannot add a read error after an error or EOF"; + } + + auto event = std::make_shared(ex, delayFromPrevious); + addReadEvent(event); +} + +void +TestAsyncTransport::addReadEvent(const shared_ptr& event) { + bool firstEvent = readEvents_.empty(); + readEvents_.push_back(event); + + if (!firstEvent) { + return; + } + if (prevReadEventTime_ == 0) { + return; + } + + nextReadEventTime_ = prevReadEventTime_ + event->getDelay(); + if (readCallback_ == nullptr) { + return; + } + + scheduleNextReadEvent(Util::currentTime()); +} + +void +TestAsyncTransport::startReadEvents() { + int64_t now = Util::currentTime(); + prevReadEventTime_ = now; + + if (readEvents_.empty()) { + return; + } + nextReadEventTime_ = prevReadEventTime_ + readEvents_.front()->getDelay(); + + if (readCallback_ == nullptr) { + return; + } + + scheduleNextReadEvent(now); +} + +void +TestAsyncTransport::scheduleNextReadEvent(int64_t now) { + if (nextReadEventTime_ <= now) { + fireNextReadEvent(); + } else { + scheduleTimeout(nextReadEventTime_ - now); + } +} + +void +TestAsyncTransport::fireNextReadEvent() { + DestructorGuard dg(this); + CHECK(!readEvents_.empty()); + CHECK(readCallback_ != nullptr); + + // maxReadAtOnce prevents us from starving other users of this EventBase + unsigned int const maxReadAtOnce = 30; + for (unsigned int n = 0; n < maxReadAtOnce; ++n) { + fireOneReadEvent(); + + if (readCallback_ == nullptr || eventBase_ == nullptr || + nextReadEventTime_ == 0) { + return; + } + int64_t now = Util::currentTime(); + if (nextReadEventTime_ > now) { + scheduleTimeout(nextReadEventTime_ - now); + return; + } + } + + // Trigger fireNextReadEvent() to be called the next time around the event + // loop. + eventBase_->runInLoop(std::bind(&TestAsyncTransport::fireNextReadEvent, + this)); +} + +void +TestAsyncTransport::fireOneReadEvent() { + CHECK(!readEvents_.empty()); + CHECK(readCallback_ != nullptr); + + const shared_ptr& event = readEvents_.front(); + + // Note that we call getReadBuffer() here even if we know the next event may + // be an EOF or an error. This matches the behavior of TAsyncSocket. + // (Because TAsyncSocket merely gets notification that the socket is readable, + // and has to call getReadBuffer() before it can make the actual read call to + // get an error or EOF.) + void* buf; + size_t buflen; + try { + readCallback_->getReadBuffer(&buf, &buflen); + } catch (...) { + // TODO: we should convert the error to a TTransportException and call + // readError() here. + LOG(FATAL) << "readCallback_->getReadBuffer() threw an error"; + } + if (buf == nullptr || buflen == 0) { + // TODO: we should just call readError() here. + LOG(FATAL) << "readCallback_->getReadBuffer() returned a nullptr or " + "empty buffer"; + } + + // Handle errors + if (event->isError()) { + // Errors immediately move both read and write to an error state + readState_ = kStateError; + writeState_ = kStateError; + + // event is just a reference to the shared_ptr, so make a real copy of the + // pointer before popping it off the readEvents_ list. + shared_ptr eventPointerCopy = readEvents_.front(); + readEvents_.pop_front(); + CHECK(readEvents_.empty()); + nextReadEventTime_ = 0; + + ReadCallback* callback = readCallback_; + readCallback_ = nullptr; + callback->readError(eventPointerCopy->getException()); + return; + } + + // Handle EOF + size_t available = event->getLength(); + if (available == 0) { + readState_ = kStateClosed; + + readEvents_.pop_front(); + CHECK(readEvents_.empty()); + nextReadEventTime_ = 0; + + ReadCallback* callback = readCallback_; + readCallback_ = nullptr; + callback->readEOF(); + return; + } + + // Handle a normal read event + size_t readlen; + bool more; + if (available <= buflen) { + readlen = available; + more = false; + } else { + readlen = buflen; + more = true; + } + memcpy(buf, event->getBuffer(), readlen); + if (more) { + event->consumeData(readlen); + } else { + prevReadEventTime_ = nextReadEventTime_; + // Note: since event is just a reference to the shared_ptr in readEvents_, + // we shouldn't access the event any more after popping it off here. + readEvents_.pop_front(); + + if (readEvents_.empty()) { + nextReadEventTime_ = 0; + } else { + nextReadEventTime_ = prevReadEventTime_ + readEvents_.front()->getDelay(); + } + } + readCallback_->readDataAvailable(readlen); +} + +void +TestAsyncTransport::timeoutExpired() noexcept { + CHECK_NOTNULL(readCallback_); + CHECK(!readEvents_.empty()); + fireNextReadEvent(); +} diff --git a/proxygen/lib/test/TestAsyncTransport.h b/proxygen/lib/test/TestAsyncTransport.h new file mode 100644 index 0000000000..619ca25417 --- /dev/null +++ b/proxygen/lib/test/TestAsyncTransport.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include +#include +#include + +class TestAsyncTransport : public apache::thrift::async::TAsyncTransport, + private apache::thrift::async::TAsyncTimeout { + public: + class WriteEvent { + public: + static std::shared_ptr newEvent(const struct iovec* vec, + size_t count); + + int64_t getTime() const { + return time_; + } + const struct iovec* getIoVec() const { + return vec_; + } + size_t getCount() const { + return count_; + } + + private: + static void destroyEvent(WriteEvent* event); + + WriteEvent(int64_t time, size_t count); + ~WriteEvent(); + + int64_t time_; + size_t count_; + struct iovec vec_[]; + }; + + explicit TestAsyncTransport(folly::EventBase* eventBase); + + // TAsyncTransport methods + void setReadCallback(ReadCallback* callback) override; + ReadCallback* getReadCallback() const override; + void write(WriteCallback* callback, + const void* buf, size_t bytes, + apache::thrift::async::WriteFlags flags = + apache::thrift::async::WriteFlags::NONE) override; + void writev(WriteCallback* callback, + const struct iovec* vec, size_t count, + apache::thrift::async::WriteFlags flags = + apache::thrift::async::WriteFlags::NONE) override; + void writeChain(WriteCallback* callback, + std::unique_ptr&& iob, + apache::thrift::async::WriteFlags flags = + apache::thrift::async::WriteFlags::NONE) override; + void close() override; + void closeNow() override; + void shutdownWrite() override; + void shutdownWriteNow() override; + void getPeerAddress(folly::SocketAddress* addr) + const override; + void getLocalAddress(folly::SocketAddress* addr) + const override; + bool good() const override; + bool readable() const override; + bool connecting() const override; + bool error() const override; + void attachEventBase(folly::EventBase* eventBase) override; + void detachEventBase() override; + bool isDetachable() const override; + folly::EventBase* getEventBase() const override; + void setSendTimeout(uint32_t milliseconds) override; + uint32_t getSendTimeout() const override; + + // Methods to control read events + void addReadEvent(const void* buf, size_t buflen, + int64_t delayFromPrevious); + void addReadEvent(folly::IOBufQueue& chain, + int64_t delayFromPrevious); + void addReadEvent(const char* buf, int64_t delayFromPrevious); + void addReadEOF(int64_t delayFromPrevious); + void addReadError(const apache::thrift::transport::TTransportException& ex, + int64_t delayFromPrevious); + void startReadEvents(); + + void pauseWrites(); + void resumeWrites(); + + // Methods to get the data written to this transport + std::deque< std::shared_ptr >* getWriteEvents() { + return &writeEvents_; + } + + uint32_t getEORCount() { + return eorCount_; + } + + uint32_t getCorkCount() { + return corkCount_; + } + + size_t getAppBytesWritten() const { return 0; } + size_t getRawBytesWritten() const { return 0; } + size_t getAppBytesReceived() const { return 0; } + size_t getRawBytesReceived() const { return 0; } + bool isEorTrackingEnabled() const { return false; } + void setEorTracking(bool) { return; } + + private: + enum StateEnum { + kStateOpen, + kStatePaused, + kStateClosed, + kStateError, + }; + + class ReadEvent; + + bool writesAllowed() const { return writeState_ == kStateOpen || + writeState_ == kStatePaused; } + + // Forbidden copy constructor and assignment opererator + TestAsyncTransport(TestAsyncTransport const&); + TestAsyncTransport& operator=(TestAsyncTransport const&); + + void addReadEvent(const std::shared_ptr& event); + void scheduleNextReadEvent(int64_t now); + void fireNextReadEvent(); + void fireOneReadEvent(); + void failPendingWrites(); + + // TAsyncTimeout methods + virtual void timeoutExpired() noexcept; + + folly::EventBase* eventBase_; + ReadCallback* readCallback_; + uint32_t sendTimeout_; + + int64_t prevReadEventTime_; + int64_t nextReadEventTime_; + StateEnum readState_; + StateEnum writeState_; + std::deque< std::shared_ptr > readEvents_; + std::deque< std::shared_ptr > writeEvents_; + std::deque< std::pair, WriteCallback*>> + pendingWriteEvents_; + + uint32_t eorCount_{0}; + uint32_t corkCount_{0}; +}; diff --git a/proxygen/lib/test/TestMain.cpp b/proxygen/lib/test/TestMain.cpp new file mode 100644 index 0000000000..8c83377702 --- /dev/null +++ b/proxygen/lib/test/TestMain.cpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +// Use this main function in gtest unit tests to enable glog +#include +#include +#include + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + google::InstallFailureSignalHandler(); + LOG(INFO) << "Running tests from TestMain.cpp"; + return RUN_ALL_TESTS(); +} diff --git a/proxygen/lib/utils/CobHelper.h b/proxygen/lib/utils/CobHelper.h new file mode 100644 index 0000000000..507df25092 --- /dev/null +++ b/proxygen/lib/utils/CobHelper.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +/** + * A helper class to schedule N async jobs in (possibly) different threads + * and invoke a final callback in a given thread once all are done. + */ +class CobHelper { + public: + CobHelper(size_t itemsLeft, + const std::function& cob, + const std::function& ecob) + : itemsLeft_(itemsLeft) +, cob_(cob) +, ecob_(ecob) {} + + void setError(const std::string& emsg) { + CHECK(!emsg.empty()); + emsg_ = emsg; + } + + void workerDone() { + uint32_t oldValue = itemsLeft_.fetch_sub(1); + if (oldValue != 1) { + return; + } + + allDone(); + } + private: + void allDone() { + if (!emsg_.empty()) { + ecob_(std::runtime_error(emsg_)); + } else { + cob_(); + } + + delete this; + } + + std::atomic itemsLeft_; + std::string emsg_; + std::function cob_; + std::function ecob_; +}; + +} diff --git a/proxygen/lib/utils/CryptUtil.cpp b/proxygen/lib/utils/CryptUtil.cpp new file mode 100644 index 0000000000..a71b6fa63c --- /dev/null +++ b/proxygen/lib/utils/CryptUtil.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/CryptUtil.h" + +#include +#include +#include +#include +#include +#include + +namespace proxygen { + +// Base64 encode using openssl +std::string base64Encode(folly::ByteRange text) { + std::string result; + BIO *b64 = BIO_new(BIO_f_base64()); + if (b64 == nullptr) { + return result; + } + BIO *bmem = BIO_new(BIO_s_mem()); + if (bmem == nullptr) { + BIO_free_all(b64); + return result; + } + BUF_MEM *bptr; + + // chain base64 filter with the memory buffer + // so that text will be encoded by base64 and flushed to buffer + BIO *chain = BIO_push(b64, bmem); + if (chain == nullptr) { + BIO_free_all(b64); + return result; + } + BIO_set_flags(chain, BIO_FLAGS_BASE64_NO_NL); + BIO_write(chain, text.begin(), text.size()); + if (BIO_flush(chain) != 1) { + BIO_free_all(chain); + return result; + } + + BIO_get_mem_ptr(chain, &bptr); + + if (bptr && bptr->length > 0) { + result = std::string((char *)bptr->data, bptr->length); + } + + // free the whole BIO chain (b64 and mem) + BIO_free_all(chain); + return result; +} + +// MD5 encode using openssl +std::string md5Encode(folly::ByteRange text) { + static_assert(MD5_DIGEST_LENGTH == 16, ""); + + unsigned char digest[MD5_DIGEST_LENGTH]; + MD5(text.begin(), text.size(), digest); + + // convert digest to hex string + std::ostringstream ss; + ss << std::hex << std::setfill('0'); + for(int i = 0; i < MD5_DIGEST_LENGTH; i++) { + ss << std::setw(2) << (unsigned int)digest[i]; + } + return ss.str(); +} + +} diff --git a/proxygen/lib/utils/CryptUtil.h b/proxygen/lib/utils/CryptUtil.h new file mode 100644 index 0000000000..e9f1dc49be --- /dev/null +++ b/proxygen/lib/utils/CryptUtil.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +namespace proxygen { + +// Base64 encode using openssl, may return empty string on allocation failure +std::string base64Encode(folly::ByteRange text); + +// MD5 encode using openssl +std::string md5Encode(folly::ByteRange text); +} diff --git a/proxygen/lib/utils/DestructorCheck.h b/proxygen/lib/utils/DestructorCheck.h new file mode 100644 index 0000000000..2e823054f4 --- /dev/null +++ b/proxygen/lib/utils/DestructorCheck.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +namespace proxygen { + +/** + * DestructorCheck marks a boolean when it is destroyed. You can use it with + * the inner Safety class to know if the DestructorCheck was destroyed during + * some asynchronous call. + * + * + * Example: + * + * class AsyncFoo : public DestructorCheck { + * public: + * ~AsyncFoo(); + * // awesome async code with circuitous deletion paths + * void async1(); + * void async2(); + * }; + * + * righteousFunc(AsyncFoo& f) { + * DestructorCheck::Safety safety(f); + * + * f.async1(); // might have deleted f, oh noes + * if (!safety.destroyed()) { + * // phew, still there + * f.async2(); + * } + * } + */ + +class DestructorCheck { + public: + virtual ~DestructorCheck() { + if (pDestroyed_) { + *pDestroyed_ = true; + } + } + + void setDestroyedPtr(bool* pDestroyed) { + pDestroyed_ = pDestroyed; + } + + void clearDestroyedPtr() { + pDestroyed_ = nullptr; + } + + // See above example for usage + class Safety { + public: + explicit Safety(DestructorCheck& destructorCheck) + : destructorCheck_(destructorCheck) { + destructorCheck_.setDestroyedPtr(&destroyed_); + } + + ~Safety() { + if (!destroyed_) { + destructorCheck_.clearDestroyedPtr(); + } + } + + bool destroyed() const { + return destroyed_; + } + + private: + DestructorCheck& destructorCheck_; + bool destroyed_{false}; + }; + + private: + bool* pDestroyed_{nullptr}; +}; + +} diff --git a/proxygen/lib/utils/DomainNameMisc.h b/proxygen/lib/utils/DomainNameMisc.h new file mode 100644 index 0000000000..d50bf37b15 --- /dev/null +++ b/proxygen/lib/utils/DomainNameMisc.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +struct dn_char_traits : public std::char_traits { + static bool eq(char c1, char c2) { + return ::tolower(c1) == ::tolower(c2); + } + + static bool ne(char c1, char c2) { + return ::tolower(c1) != ::tolower(c2); + } + + static bool lt(char c1, char c2) { + return ::tolower(c1) < ::tolower(c2); + } + + static int compare(const char* s1, const char* s2, size_t n) { + while (n--) { + if(::tolower(*s1) < ::tolower(*s2) ) { + return -1; + } + if(::tolower(*s1) > ::tolower(*s2) ) { + return 1; + } + ++s1; + ++s2; + } + return 0; + } + + static const char* find(const char* s, size_t n, char a) { + char la = ::tolower(a); + while (n--) { + if(::tolower(*s) == la) { + return s; + } else { + ++s; + } + } + return nullptr; + } +}; + +// Case insensitive string +typedef std::basic_string DNString; + +struct DNStringHash : public std::hash { + size_t operator()(const DNString& s) const noexcept { + size_t h = static_cast(0xc70f6907UL); + const char* d = s.data(); + for (size_t i = 0; i < s.length(); ++i) { + char a = ::tolower(*d++); + h = std::_Hash_impl::hash(&a, sizeof(a), h); + } + return h; + } +}; + +} // proxygen diff --git a/proxygen/lib/utils/Exception.cpp b/proxygen/lib/utils/Exception.cpp new file mode 100644 index 0000000000..0268c3e798 --- /dev/null +++ b/proxygen/lib/utils/Exception.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/Exception.h" + +namespace proxygen { + +Exception::Exception(std::string const& msg) : + msg_(msg), + code_(0) { +} + +Exception::Exception(const Exception& other) : + msg_(other.msg_), + code_(other.code_) { +} + +Exception::Exception(Exception&& other) noexcept : + msg_(other.msg_), + code_(other.code_) { +} + +const char * +Exception::what(void) const throw() { + return msg_.c_str(); +} + +} diff --git a/proxygen/lib/utils/Exception.h b/proxygen/lib/utils/Exception.h new file mode 100644 index 0000000000..65ac6da226 --- /dev/null +++ b/proxygen/lib/utils/Exception.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +namespace proxygen { + +/** + * Base class for all exceptions. + */ +class Exception : public std::exception { + public: + explicit Exception(std::string const& msg); + Exception(const Exception&); + Exception(Exception&&) noexcept; + + template + explicit Exception(Args&&... args) + : msg_(folly::to(std::forward(args)...)), + code_(0) {} + + ~Exception(void) noexcept override {} + + // std::exception methods + virtual const char* what(void) const noexcept; + + void setCode(int code) { code_ = code; } + + int getCode() const { return code_; } + + private: + const std::string msg_; + int code_; +}; + +} diff --git a/proxygen/lib/utils/FilterChain.h b/proxygen/lib/utils/FilterChain.h new file mode 100644 index 0000000000..a8022f0fe9 --- /dev/null +++ b/proxygen/lib/utils/FilterChain.h @@ -0,0 +1,358 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include +#include + +namespace proxygen { + +/** + * A generic two-way filter. That is, this filter intercepts calls from an + * object A to some object B on an interface T1. It also intercepts calls from B + * to A on an interface T2. + * + * Subclass GenericFilter templatized on the appropriate interfaces to intercept + * calls between T1 and T2. + * + * T2 must have the ability to tell T1 to call it back using a certain + * object. Many different callback patterns have a different name for this + * function, but this filter needs to be informed when the callback object + * changes. Therefore, when you subclass this object, you must call + * + * void setCallbackInternal(T2*); + * + * from the appropriate virtual function and not forward that function call + * along the chain, or the filter will not work and may have errors! + */ +template > +class GenericFilter: public T1, public T2 { + public: + typedef GenericFilter Filter; + /** + * @param calls You will intercept calls to T1 interface iff you + * pass true for this parameter. + * @param callbacks You will intercept calls to T2 interface iff you + * pass true for this parameter. + */ + GenericFilter(bool calls, bool callbacks): + kWantsCalls_(calls), + kWantsCallbacks_(callbacks) {} + + ~GenericFilter() override { + if (TakeOwnership) { + callbackSource_ = nullptr; + } + // For the last filter in the chain, next_ is null, and call_ points + // to the concrete implementation object. + auto next = next_ ? next_ : call_; + drop(); + if (TakeOwnership && next) { + Dp()(next); + } + } + + /** + * @param nextFilter the new filter to insert after this filter + */ + void append(Filter* nextFilter) { + nextFilter->next_ = next_; + nextFilter->prev_ = this; + nextFilter->call_ = call_; + nextFilter->callback_ = kWantsCallbacks_ ? this : callback_; + nextFilter->callSource_ = kWantsCalls_ ? this : callSource_; + nextFilter->callbackSource_ = callbackSource_; + if (next_) { + next_->prev_ = nextFilter; + } + + if (nextFilter->kWantsCalls_) { + if (kWantsCalls_) { + call_ = nextFilter; + } else { + callSource_->call_ = nextFilter; + } + if (next_) { + next_->callSource_ = nextFilter; + } + } + if (nextFilter->kWantsCallbacks_) { + // Find the first filter before this one that wants callbacks + auto cur = this; + while (cur->prev_ && !cur->kWantsCallbacks_) { + cur = cur->prev_; + } + cur->callbackSource_ = nextFilter; + // Make sure nextFilter gets callbacks + ((nextFilter->callbackSource_)->*(set_callback))(nextFilter); + } + next_ = nextFilter; + } + + const bool kWantsCalls_; + const bool kWantsCallbacks_; + + protected: + /** + * Note: you MUST override the set_callback function and call + * setCallbackInternal() from your derived class. + */ + void setCallbackInternal(T2* cb) { + setCallbackInternalImpl(cb, this); + } + + /** + * Removes this filter from the chain. For owning chains the caller must + * manually delete the filter + */ + void drop() { + if (prev_) { + prev_->next_ = next_; + } + if (next_) { + next_->prev_ = prev_; + } + // TODO: could call fn gated by std::enable_if + if (kWantsCalls_ && callSource_) { + callSource_->call_ = call_; + if (call_) { + auto callFilter = dynamic_cast(call_); + if (callFilter) { + callFilter->callSource_ = callSource_; + } + } + } + // TODO: could call fn gated by std::enable_if + if (kWantsCallbacks_ && callbackSource_) { + ((callbackSource_)->*(set_callback))(callback_); + if (callback_) { + auto callbackFilter = dynamic_cast(callback_); + if (callbackFilter) { + callbackFilter->callbackSource_ = callbackSource_; + } + } + } + call_ = callbackSource_ = nullptr; + callback_ = nullptr; + callSource_ = next_ = prev_ = nullptr; + } + + // Next "call" filter (call_ == next_ iff next_->kWantsCalls_) + T1* call_{nullptr}; + // Next "callback" filter (callback_ == prev_ iff prev_->kWantsCallbacks_) + T2* callback_{nullptr}; + + private: + + void setCallbackInternalImpl(T2* cb, T2* sourceSet) { + if (callback_ != cb) { + callback_ = cb; + ((callbackSource_)->*(set_callback))(cb ? sourceSet : nullptr); + } + } + + // The next filter in the chain (towards T1 implementation) + Filter* next_{nullptr}; + // The previous filter in the chain (towards T2 implementation) + Filter* prev_{nullptr}; + // The first filter before this one in the chain that wants calls. + // Only valid if kWantsCalls_ is true. + Filter* callSource_{nullptr}; + // Pointer to the first filter or object after this one in the chain + // that makes callbacks. Only valid if kWantsCallbacks_ is true. + T1* callbackSource_{nullptr}; + + template + friend class FilterChain; +}; + +/** + * This class can be treated the same as a T1*, however internally it contains a + * chain of filters for T1 and T2. These filters are inserted between the + * original callback and destination. + * + * If a filter does not care about one side of the calls, it will pass through + * the calls, saving a virtual function call. In the example below Filter1 only + * wants to intercept callbacks, and Filter2 only wants to intercept calls. + * + * + * FilterChain Filter1 Filter2 destination + * _____________ _____________ _____________ _____________ + * | | | | | | | | + * | call_------------------------>| call_------->|"call" | T1 + * | | | | | | | | + * T2 | callback_ |<-----callback_ |<--------------------|"callback" | + * |___________| |___________| |___________| |___________| + * + * + * This class is templatized on the two interfaces, T1, T2, as well as on the + * pass through filter implementation, FilterType, the special set callback + * function, and a boolean indicating whether the chain owns the filters (if + * false, the filters must delete themselves at the correct time). FilterType + * must have GenericFilter as an ancestor. + */ +template +class FilterChain: private FilterType { + public: + explicit FilterChain(std::unique_ptr destination): + FilterType(false, false) { + static_assert(TakeOwnership, "unique_ptr constructor only available " + "if the chain owns the filters."); + this->call_ = CHECK_NOTNULL(destination.release()); + this->callback_ = nullptr; //must call setCallback() explicitly + this->callSource_ = this; + this->callbackSource_ = this->call_; + this->chainEnd_ = this->call_; + } + + explicit FilterChain(T1* destination): + FilterType(false, false) { + static_assert(!TakeOwnership, "raw pointer constructor only available " + "if the chain doesn't own the filters."); + this->call_ = CHECK_NOTNULL(destination); + this->callback_ = nullptr; //must call setCallback() explicitly + this->callSource_ = this; + this->callbackSource_ = this->call_; + this->chainEnd_ = this->call_; + } + + /** + * Set the callback for this entire filter chain. Setting this to null will + * uninstall the callback from the concrete object at the end of the chain. + */ + void setCallback(T2* cb) { + this->setCallbackInternalImpl(cb, cb); + } + + /** + * Returns the head of the call chain. Do* not* call T1::set_callback() on + * this member. To change the callback of this entire filter chain, use the + * separate setCallback() method. + */ + T1* call() { + return this->call_; + } + const T1* call() const { + return this->call_; + } + + /** + * Returns the concrete implementation at the end of the filter chain. + */ + const T1& getChainEnd() const { + return *chainEnd_; + } + + typedef GenericFilter FilterChainType; + std::unique_ptr setDestination(std::unique_ptr destination) { + static_assert(TakeOwnership, "unique_ptr setDestination only available " + "if the chain owns the filters."); + // find the last filter in the chain, and the last filter that wants calls, + // callbacks + FilterChainType* lastFilter = this; + FilterChainType* lastCall = this; + FilterChainType* lastCallback = this; + while (lastFilter->next_) { + if (lastFilter->kWantsCalls_) { + lastCall = lastFilter; + } + if (lastFilter->kWantsCallbacks_) { + lastCallback = lastFilter; + } + if (lastFilter->call_ == this->chainEnd_) { + // Search and replace, the last N non-call filters all point to dest + lastFilter->call_ = destination.get(); + } + lastFilter = lastFilter->next_; + } + if (lastFilter->kWantsCalls_) { + lastCall = lastFilter; + } + if (lastFilter->kWantsCallbacks_) { + lastCallback = lastFilter; + } + lastFilter->call_ = CHECK_NOTNULL(destination.release()); + lastCall->call_ = lastFilter->call_; + lastCallback->callbackSource_ = lastFilter->call_; + auto oldChainEnd = this->chainEnd_; + this->chainEnd_ = lastFilter->call_; + + this->chainEnd_->setCallback(lastCallback); + return std::unique_ptr(oldChainEnd); + } + + /** + * Adds filters with the given types to the front of the chain. + */ + template + std::enable_if::value> addFilters() { + // Callback <-> F1 <-> F2 ... <-> F_new <-> Destination + this->append(new C()); + addFilters(); + } + + /** + * Base case of above function where we add a single filter. + */ + template + std::enable_if::value> addFilters() { + this->append(new C()); + } + + /** + * Adds already constructed filters (inside unique_ptr) to the front of the + * chain. + */ + template + void addFilters(std::unique_ptr cur, Types&&... remaining) { + static_assert(TakeOwnership, "addFilters() can only take " + "unique_ptr if the chain owns the filters"); + this->append(cur.release()); + addFilters(std::forward(remaining)...); + } + + template + void addFilters(C* cur, Types&&... remaining) { + static_assert(!TakeOwnership, "addFilters() can only take " + "pointers if the chain doesn't own the filters"); + this->append(cur); + addFilters(std::forward(remaining)...); + } + + /** + * Another way to add filters. This way is similar to 'emplace_front' and + * returns a reference to itself so you can chain add() calls if you like. + */ + template + FilterChain& + add(Args&&... args) { + this->append(new C(std::forward(args)...)); + return *this; + } + + const T1* operator->() const { return call(); } + T1* operator->() { return call(); } + + private: + /** + * Base case for addFilters() called with no arguments. It doesn't need to be + * public since trying to add zero filters doesn't make sense from a public + * API pov. + */ + void addFilters() {} + + T1* chainEnd_; +}; + +} diff --git a/proxygen/lib/utils/HTTPTime.cpp b/proxygen/lib/utils/HTTPTime.cpp new file mode 100644 index 0000000000..f489a0bd64 --- /dev/null +++ b/proxygen/lib/utils/HTTPTime.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/HTTPTime.h" + +#include +#include + +namespace proxygen { + +folly::Optional parseHTTPDateTime(const std::string& s) { + struct tm tm = {0}; + + if (s.empty()) { + return folly::Optional(); + } + + // Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123 + if (strptime(s.c_str(), "%a, %d %b %Y %H:%M:%S %Z", &tm) != nullptr) { + return folly::Optional(mktime(&tm)); + } + + // Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036 + if (strptime(s.c_str(), "%a, %d-%b-%y %H:%M:%S %Z", &tm) != nullptr) { + return folly::Optional(mktime(&tm)); + } + + // Sun Nov 6 08:49:37 1994 ; ANSI C's asctime() format + if(strptime(s.c_str(), "%a %b %d %H:%M:%S %Y", &tm) != nullptr) { + return folly::Optional(mktime(&tm)); + } + + LOG(INFO) << "Invalid http time: " << s; + return folly::Optional(); +} + +} // proxygen diff --git a/proxygen/lib/utils/HTTPTime.h b/proxygen/lib/utils/HTTPTime.h new file mode 100644 index 0000000000..cf7e27f610 --- /dev/null +++ b/proxygen/lib/utils/HTTPTime.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +namespace proxygen { + +folly::Optional parseHTTPDateTime(const std::string& s); + +} // proxygen diff --git a/proxygen/lib/utils/Makefile.am b/proxygen/lib/utils/Makefile.am new file mode 100644 index 0000000000..a29a1a137b --- /dev/null +++ b/proxygen/lib/utils/Makefile.am @@ -0,0 +1,62 @@ +SUBDIRS = . test + +BUILT_SOURCES = TraceEventType.h TraceEventType.cpp TraceFieldType.h TraceFieldType.cpp + +TraceGen: TraceEventType.txt TraceFieldType.txt + python gen_trace_event_constants.py \ + --output_type=cpp \ + --input_files=TraceEventType.txt,TraceFieldType.txt \ + --output_scope=proxygen \ + --header_path=proxygen/lib/utils \ + --install_dir=$(srcdir) \ + --fbcode_dir=$(srcdir) + touch TraceGen + +TraceEventType.h: TraceGen + +TraceEventType.cpp: TraceGen + +TraceFieldType.h: TraceGen + +TraceFieldType.cpp: TraceGen + +noinst_LTLIBRARIES = libutils.la + +# We put the generated files first so that we create them first +libutilsdir = $(includedir)/proxygen/lib/utils +nobase_libutils_HEADERS = \ + CobHelper.h \ + CryptUtil.h \ + DestructorCheck.h \ + DomainNameMisc.h \ + Exception.h \ + FilterChain.h \ + HTTPTime.h \ + NullTraceEventObserver.h \ + ParseURL.h \ + Result.h \ + SocketOptions.h \ + StateMachine.h \ + TestUtils.h \ + Time.h \ + TraceEvent.h \ + TraceEventContext.h \ + TraceEventObserver.h \ + TraceEventType.h \ + TraceFieldType.h \ + UtilInl.h + +# We put the generated files first so that we create them first +libutils_la_SOURCES = \ + ../../external/http_parser/http_parser_cpp.cpp \ + Exception.cpp \ + HTTPTime.cpp \ + NullTraceEventObserver.cpp \ + ParseURL.cpp \ + SocketOptions.cpp \ + TraceEvent.cpp \ + TraceEventType.cpp \ + TraceEventType.cpp \ + TraceFieldType.cpp \ + TraceFieldType.cpp \ + CryptUtil.cpp diff --git a/proxygen/lib/utils/NullTraceEventObserver.cpp b/proxygen/lib/utils/NullTraceEventObserver.cpp new file mode 100644 index 0000000000..e46db456b5 --- /dev/null +++ b/proxygen/lib/utils/NullTraceEventObserver.cpp @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/NullTraceEventObserver.h" + +namespace proxygen { + +NullTraceEventObserver NullTraceEventObserver::nullObserver; + +} diff --git a/proxygen/lib/utils/NullTraceEventObserver.h b/proxygen/lib/utils/NullTraceEventObserver.h new file mode 100644 index 0000000000..65107b9e5d --- /dev/null +++ b/proxygen/lib/utils/NullTraceEventObserver.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/utils/TraceEventObserver.h" + +namespace proxygen { + +struct NullTraceEventObserver : public TraceEventObserver { + // Do nothing. + void traceEventAvailable(TraceEvent event) noexcept override {} + + static NullTraceEventObserver nullObserver; +}; + +} diff --git a/proxygen/lib/utils/ParseURL.cpp b/proxygen/lib/utils/ParseURL.cpp new file mode 100644 index 0000000000..c155b1b331 --- /dev/null +++ b/proxygen/lib/utils/ParseURL.cpp @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/ParseURL.h" + +#include "proxygen/external/http_parser/http_parser.h" +#include "proxygen/lib/utils/UtilInl.h" + +#include +#include + +using folly::fbstring; +using std::string; + +namespace proxygen { + +void ParseURL::parse() noexcept { + if (caseInsensitiveEqual(url_.subpiece(0, 4), "http")) { + struct http_parser_url u; + memset(&u, 0, sizeof(struct http_parser_url)); // init before used + valid_ = !(http_parser_parse_url(url_.data(), url_.size(), 0, &u)); + + if(valid_) { + // Since we init the http_parser_url with all fields to 0, if the field + // not present in url, it would be [0, 0], means that this field starts at + // 0 and len = 0, we will get "" from this. So no need to check field_set + // before get field. + + scheme_ = url_.subpiece(u.field_data[UF_SCHEMA].off, + u.field_data[UF_SCHEMA].len); + + if(u.field_data[UF_HOST].off != 0 && + url_[u.field_data[UF_HOST].off - 1] == '[') { + // special case: host: [::1] + host_ = url_.subpiece(u.field_data[UF_HOST].off - 1, + u.field_data[UF_HOST].len + 2); + } else { + host_ = url_.subpiece(u.field_data[UF_HOST].off, + u.field_data[UF_HOST].len); + } + + port_ = u.port; + + path_ = url_.subpiece(u.field_data[UF_PATH].off, + u.field_data[UF_PATH].len); + query_ = url_.subpiece(u.field_data[UF_QUERY].off, + u.field_data[UF_QUERY].len); + fragment_ = url_.subpiece(u.field_data[UF_FRAGMENT].off, + u.field_data[UF_FRAGMENT].len); + + authority_ = (port_) ? folly::to(host_, ":", port_) + : host_.str(); + } + } else { + parseNonFully(); + } +} + +void ParseURL::parseNonFully() noexcept { + if (url_.empty()) { + valid_ = false; + return; + } + + auto pathStart = url_.find('/'); + auto queryStart = url_.find('?'); + auto hashStart = url_.find('#'); + + auto queryEnd = std::min(hashStart, std::string::npos); + auto pathEnd = std::min(queryStart, hashStart); + auto authorityEnd = std::min(pathStart, pathEnd); + + authority_ = url_.subpiece(0, authorityEnd).str(); + + if (pathStart < pathEnd) { + path_ = url_.subpiece(pathStart, pathEnd - pathStart); + } else { + // missing the '/', e.g. '?query=3' + path_ = ""; + } + + if (queryStart < queryEnd) { + query_ = url_.subpiece(queryStart + 1, queryEnd - queryStart - 1); + } else if (queryStart != std::string::npos && hashStart < queryStart) { + valid_ = false; + return; + } + + if (hashStart != std::string::npos) { + fragment_ = url_.subpiece(hashStart + 1, std::string::npos); + } + + if (!parseAuthority()) { + valid_ = false; + return; + } + + valid_ = true; +} + +bool ParseURL::parseAuthority() noexcept { + auto left = authority_.find("["); + auto right = authority_.find("]"); + + auto pos = authority_.find(":", right != std::string::npos ? right : 0); + if (pos != std::string::npos) { + try { + port_ = folly::to( + folly::StringPiece(authority_, pos+1, std::string::npos)); + } catch (...) { + return false; + } + } + + if (left == std::string::npos && right == std::string::npos) { + // not a ipv6 literal + host_ = folly::StringPiece(authority_, 0, pos); + return true; + } else if (left < right && right != std::string::npos) { + // a ipv6 literal + host_ = folly::StringPiece(authority_, left, right - left + 1); + return true; + } else { + return false; + } +} + +bool ParseURL::hostIsIPAddress() { + if (!valid_) { + return false; + } + + stripBrackets(); + int af = hostNoBrackets_.find(':') == std::string::npos ? AF_INET : AF_INET6; + char buf4[sizeof(in_addr)]; + char buf6[sizeof(in6_addr)]; + // we have to make a copy of hostNoBrackets_ since the string piece is not + // null-terminated + return inet_pton(af, hostNoBrackets_.str().c_str(), + af == AF_INET ? buf4 : buf6) == 1; +} + +void ParseURL::stripBrackets() noexcept { + if (hostNoBrackets_.empty()) { + if (!host_.empty() && host_.front() == '[' && host_.back() == ']') { + hostNoBrackets_ = host_.subpiece(1, host_.size() - 2); + } else { + hostNoBrackets_ = host_; + } + } +} + +} diff --git a/proxygen/lib/utils/ParseURL.h b/proxygen/lib/utils/ParseURL.h new file mode 100644 index 0000000000..1105346517 --- /dev/null +++ b/proxygen/lib/utils/ParseURL.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +namespace proxygen { + +// ParseURL can handle non-fully-formed URLs. This class must not persist beyond +// the lifetime of the buffer underlying the input StringPiece + +class ParseURL { + public: + ParseURL() {} + explicit ParseURL(folly::StringPiece urlVal) noexcept { + init(urlVal); + } + + void init(folly::StringPiece urlVal) { + CHECK(!initialized_); + url_ = urlVal; + parse(); + initialized_ = true; + } + + folly::StringPiece url() const { + return url_; + } + + folly::StringPiece scheme() const { + return scheme_; + } + + std::string authority() const { + return authority_; + } + + bool hasHost() const { + return valid() && !host_.empty(); + } + + folly::StringPiece host() const { + return host_; + } + + uint16_t port() const { + return port_; + } + + std::string hostAndPort() const { + std::string rc = host_.str(); + if (port_ != 0) { + folly::toAppend(":", port_, &rc); + } + return rc; + } + + folly::StringPiece path() const { + return path_; + } + + folly::StringPiece query() const { + return query_; + } + + folly::StringPiece fragment() const { + return fragment_; + } + + bool valid() const { + return valid_; + } + + folly::StringPiece hostNoBrackets() { + stripBrackets(); + return hostNoBrackets_; + } + + bool hostIsIPAddress(); + + void stripBrackets() noexcept; + + private: + void parse() noexcept; + + void parseNonFully() noexcept; + + bool parseAuthority() noexcept; + + folly::StringPiece url_; + folly::StringPiece scheme_; + std::string authority_; + folly::StringPiece host_; + folly::StringPiece hostNoBrackets_; + folly::StringPiece path_; + folly::StringPiece query_; + folly::StringPiece fragment_; + uint16_t port_{0}; + bool valid_{false}; + bool initialized_{false}; +}; + +} diff --git a/proxygen/lib/utils/Result.h b/proxygen/lib/utils/Result.h new file mode 100644 index 0000000000..bc150b1fed --- /dev/null +++ b/proxygen/lib/utils/Result.h @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include +#include + +namespace proxygen { + +/** + * The Result type. It is similar to Haskell's Either type and Rust's + * Result type. It is essentially a tagged union. The type T represents + * the expected type and E represents the error type. + * + * This implementation is also inspired by the C++17 proposal n4109: + * http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4109.pdf + * + * The interface is patterned after + * http://doc.rust-lang.org/std/result/enum.Result.html + * + * In the future, we can add an interface that takes two callbacks and + * executes the appropriate one based on whether the Result is an error or + * if it is ok. + * + * We also provide two factory functions to construct Results using + * emplacement. Ideally we would support this in the constructor of Result + * since we already forbid the ok type to equal the error type. + */ +template +class Result { + public: + static_assert(!std::is_same::value, + "The result and error types must be different"); + + // Emplace constructor TODO XXX + // We need to prefer the variadic template form over the non-template + // implicit conversion constructors for this to work correctly. + // template ::value || + // std::is_constructible::value + // > > + // Result(Args&&... args) { + // } + + Result(Result&& src) noexcept( + std::is_nothrow_move_constructible::value && + std::is_nothrow_move_constructible::value): + state_(src.state_) { + + if (src.isOk()) { + constructOk(std::move(src.ok())); + } else { + constructError(std::move(src.error())); + } + } + + Result(const Result& src) noexcept( + std::is_nothrow_copy_constructible::value && + std::is_nothrow_copy_constructible::value): + state_(src.state_) { + + if (src.isOk()) { + constructOk(src.ok()); + } else { + constructError(src.error()); + } + } + + /* implicit */ Result(T&& okVal) noexcept( + std::is_nothrow_move_constructible::value): + state_(State::IS_OK) { + constructOk(std::move(okVal)); + } + + /* implicit */ Result(const T& okVal) noexcept( + std::is_nothrow_copy_constructible::value): + state_(State::IS_OK) { + constructOk(okVal); + } + + /* implicit */ Result(E&& errorVal) noexcept( + std::is_nothrow_move_constructible::value): + state_(State::IS_ERROR) { + constructError(std::move(errorVal)); + } + + /* implicit */ Result(const E& errorVal) noexcept( + std::is_nothrow_copy_constructible::value): + state_(State::IS_ERROR) { + constructError(errorVal); + } + + ~Result() noexcept { + clear(); + } + + bool isOk() const noexcept { + return state_ == State::IS_OK; + } + + bool isError() const noexcept { + return state_ == State::IS_ERROR; + } + + T& ok() { + assert(isOk()); + return storage_.ok; + } + + const T& ok() const { + assert(isOk()); + return storage_.ok; + } + + E& error() { + assert(isError()); + return storage_.error; + } + + const E& error() const { + assert(isError()); + return storage_.error; + } + + Result& operator=(T&& arg) { + clear(); + constructOk(std::forward(arg)); + return *this; + } + + Result& operator=(E&& arg) { + clear(); + constructError(std::forward(arg)); + return *this; + } + + Result& operator=(Result&& src) + noexcept (std::is_nothrow_move_constructible::value && + std::is_nothrow_move_constructible::value) { + + clear(); + if (src.isOk()) { + constructOk(std::move(src.ok())); + } else { + constructError(std::move(src.error())); + } + return *this; + } + + Result& operator=(const Result& src) + noexcept (std::is_nothrow_copy_constructible::value && + std::is_nothrow_copy_constructible::value) { + + clear(); + if (src.isOk()) { + constructOk(src.ok()); + } else { + constructError(src.error()); + } + return *this; + } + + template + friend Result make_ok(Args&&...); + + template + friend Result make_error(Args&&...); + + private: + + // For use by friend factory functions only + Result() noexcept { + } + + void clear() noexcept { + if (isOk()) { + storage_.ok.~T(); + } else { + storage_.error.~E(); + } + } + + template + void constructOk(Args&&... args) { + new(&storage_) T(std::forward(args)...); + state_ = State::IS_OK; + } + + template + void constructError(Args&&... args) { + new(&storage_) E(std::forward(args)...); + state_ = State::IS_ERROR; + } + + union Storage { + Storage() {} + ~Storage() {} + T ok; + E error; + }; + Storage storage_; + + enum class State: uint8_t { + IS_OK, + IS_ERROR, + }; + State state_; +}; + +/** + * Efficiently constructs a Result while avoiding any implicit + * conversions. This function is preferred when emplacement is required. + */ +template +Result make_ok(Args&&... args) { + Result result; // temporarily empty + result.constructOk(std::forward(args)...); + return result; +} + +/** + * Efficiently constructs a Result while avoiding any implicit + * conversions. This function is preferred when emplacement is required. + */ +template +Result make_error(Args&&... args) { + Result result; // temporarily empty + result.constructError(std::forward(args)...); + return result; +} + +} diff --git a/proxygen/lib/utils/SocketOptions.cpp b/proxygen/lib/utils/SocketOptions.cpp new file mode 100644 index 0000000000..521588c895 --- /dev/null +++ b/proxygen/lib/utils/SocketOptions.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/SocketOptions.h" + +#include +#include + +namespace proxygen { + +apache::thrift::async::TAsyncSocket::OptionMap filterIPSocketOptions( + const apache::thrift::async::TAsyncSocket::OptionMap& allOptions, + const int addrFamily) { + apache::thrift::async::TAsyncSocket::OptionMap opts; + int exclude; + if (addrFamily == AF_INET) { + exclude = IPPROTO_IPV6; + } else if (addrFamily == AF_INET6) { + exclude = IPPROTO_IP; + } else { + LOG(FATAL) << "Address family " << addrFamily << " was not IPv4 or IPv6"; + return opts; + } + for (const auto& opt: allOptions) { + if (opt.first.level != exclude) { + opts[opt.first] = opt.second; + } + } + return opts; +} + +} diff --git a/proxygen/lib/utils/SocketOptions.h b/proxygen/lib/utils/SocketOptions.h new file mode 100644 index 0000000000..6c212b179c --- /dev/null +++ b/proxygen/lib/utils/SocketOptions.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +/** + * Returns a copy of the socket options excluding options with the given + * level. + */ +apache::thrift::async::TAsyncSocket::OptionMap filterIPSocketOptions( + const apache::thrift::async::TAsyncSocket::OptionMap& allOptions, + const int addrFamily); + +} diff --git a/proxygen/lib/utils/StateMachine.h b/proxygen/lib/utils/StateMachine.h new file mode 100644 index 0000000000..27956a6a74 --- /dev/null +++ b/proxygen/lib/utils/StateMachine.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +template +class StateMachine { + public: + typedef typename T::State State; + typedef typename T::Event Event; + + static State getNewInstance() { + return T::getInitialState(); + } + + static bool transit(State& state, Event event) { + bool ok; + State newState; + + std::tie(newState, ok) = T::find(state, event); + if (!ok) { + LOG(ERROR) << "Invalid transition tried: " << state << " " << event; + return false; + } + VLOG(6) << "Transitioning from " << state << " to " << newState; + state = newState; + return true; + } + + static bool canTransit(const State state, Event event) { + bool ok; + + std::tie(std::ignore, ok) = T::find(state, event); + return ok; + } +}; diff --git a/proxygen/lib/utils/TestUtils.h b/proxygen/lib/utils/TestUtils.h new file mode 100644 index 0000000000..4ac662fc94 --- /dev/null +++ b/proxygen/lib/utils/TestUtils.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include + +#ifndef NDEBUG +#define EXPECT_DEATH_NO_CORE(token, regex) { \ + rlimit oldLim; \ + getrlimit(RLIMIT_CORE, &oldLim); \ + rlimit newLim{0, oldLim.rlim_max}; \ + setrlimit(RLIMIT_CORE, &newLim); \ + EXPECT_DEATH(token, regex); \ + setrlimit(RLIMIT_CORE, &oldLim); \ + } +#else +#define EXPECT_DEATH_NO_CORE(tken, regex) {} +#endif + +inline folly::StringPiece +getContainingDirectory(folly::StringPiece input) { + auto pos = folly::rfind(input, '/'); + if (pos == std::string::npos) { + pos = 0; + } else { + pos += 1; + } + return input.subpiece(0, pos); +} diff --git a/proxygen/lib/utils/Time.h b/proxygen/lib/utils/Time.h new file mode 100644 index 0000000000..38b8a3b04a --- /dev/null +++ b/proxygen/lib/utils/Time.h @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include +#include +#include + +namespace proxygen { + +typedef std::chrono::steady_clock ClockType; +typedef ClockType::time_point TimePoint; + +template +bool durationInitialized(const T& duration) { + static T zero(0); + return duration != T::max() && duration >= zero; +} + +template +bool timePointInitialized(const T& time) { + static T epoch; + return time > epoch; +} + +inline TimePoint getCurrentTime() { + static_assert(ClockType::is_steady, ""); + return ClockType::now(); +} + +inline std::chrono::system_clock::time_point toSystemTimePoint(TimePoint t) { + return std::chrono::system_clock::now() + + std::chrono::duration_cast( + t - std::chrono::steady_clock::now()); +} + +inline time_t toTimeT(TimePoint t) { + return std::chrono::system_clock::to_time_t(toSystemTimePoint(t)); +} + +inline std::chrono::milliseconds millisecondsSinceEpoch() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()); +} + +inline std::chrono::seconds secondsSinceEpoch() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()); +} + +inline std::chrono::milliseconds millisecondsSinceEpoch(TimePoint t) { + return std::chrono::duration_cast( + toSystemTimePoint(t).time_since_epoch()); +} + +inline std::chrono::seconds secondsSinceEpoch(TimePoint t) { + return std::chrono::duration_cast( + toSystemTimePoint(t).time_since_epoch()); +} + +inline std::chrono::milliseconds +millisecondsBetween(TimePoint finish, TimePoint start) { + return std::chrono::duration_cast( + finish - start); +} + +inline std::chrono::seconds +secondsBetween(TimePoint finish, TimePoint start) { + return std::chrono::duration_cast( + finish - start); +} + +inline std::chrono::milliseconds millisecondsSince(TimePoint t) { + return millisecondsBetween(getCurrentTime(), t); +} + +inline std::chrono::seconds secondsSince(TimePoint t) { + return secondsBetween(getCurrentTime(), t); +} + +/** + * Get the current date and time in string formats: %Y-%m-%d and %H:%M:%S. + */ +inline void getDateTimeStr(char datebuf[32], char timebuf[32]) { + time_t now = toTimeT(getCurrentTime()); + struct tm now_tm; + localtime_r(&now, &now_tm); + if (datebuf) { + strftime(datebuf, sizeof(char) * 32, "%Y-%m-%d", &now_tm); + } + if (timebuf) { + strftime(timebuf, sizeof(char) * 32, "%H:%M:%S", &now_tm); + } +} + +/** + * Class used to get steady time. We use a separate class to mock it easier. + */ +class TimeUtil { + public: + virtual ~TimeUtil() {} + + virtual TimePoint now() const { + return getCurrentTime(); + } + + static const TimePoint& getZeroTimePoint() { + const static TimePoint kZeroTimePoint{}; + return kZeroTimePoint; + } + + /** + * Please use strongly typed time_point. This is for avoiding the copy and + * garbage collection of time_point in Lua. + */ + virtual uint64_t msSinceEpoch() { + return millisecondsSinceEpoch().count(); + } +}; + +} diff --git a/proxygen/lib/utils/TraceEvent.cpp b/proxygen/lib/utils/TraceEvent.cpp new file mode 100644 index 0000000000..ec86155fc6 --- /dev/null +++ b/proxygen/lib/utils/TraceEvent.cpp @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/TraceEvent.h" + +#include +#include +#include +#include +#include + +namespace { + +class TraceEventIDGenerator { + public: + TraceEventIDGenerator() { + static std::mutex mtx; + std::lock_guard lock(mtx); + static std::mt19937 generator; + std::uniform_int_distribution distribution; + nextID_ = distribution(generator); + } + + uint32_t nextID() { + return ++nextID_; + } + + private: + uint32_t nextID_; +}; + +} + +namespace proxygen { +TraceEvent::TraceEvent(TraceEventType type, uint32_t parentID): + type_(type), + parentID_(parentID) { + static folly::ThreadLocal idGenerator; + id_ = idGenerator->nextID(); +} + +void TraceEvent::start(const TimeUtil& tm) { + stateFlags_ |= State::STARTED; + start_ = tm.now(); +} + +void TraceEvent::start(TimePoint startTime) { + stateFlags_ |= State::STARTED; + start_ = startTime; +} + +void TraceEvent::end(const TimeUtil& tm) { + stateFlags_ |= State::ENDED; + end_ = tm.now(); +} + +void TraceEvent::end(TimePoint endTime) { + stateFlags_ |= State::ENDED; + end_ = endTime; +} + +bool TraceEvent::hasStarted() const { + return stateFlags_ & State::STARTED; +} + +bool TraceEvent::hasEnded() const { + return stateFlags_ & State::ENDED; +} + +bool TraceEvent::addMeta(TraceFieldType key, folly::dynamic&& value) { + auto rc = metaData_.emplace(key, value); + + // replace if key already exist + if (!rc.second) { + rc.first->second = value; + } + + return rc.second; +} + +bool TraceEvent::readBoolMeta(TraceFieldType key, bool& dest) const { + if (metaData_.count(key)) { + DCHECK(metaData_.at(key).isBool()); + dest = metaData_.at(key).asBool(); + return true; + } + return false; +} + +bool TraceEvent::readStrMeta(TraceFieldType key, std::string& dest) const { + if (metaData_.count(key)) { + // no need to check if value is string type + dest = metaData_.at(key).asString().toStdString(); + return true; + } + return false; +} + +std::string TraceEvent::toString() const { + std::ostringstream out; + int startSinceEpoch = std::chrono::duration_cast( + start_.time_since_epoch()).count(); + int endSinceEpoch = std::chrono::duration_cast( + end_.time_since_epoch()).count(); + out << "TraceEvent("; + out << "type='" << getTraceEventTypeString(type_) << "', "; + out << "id='" << id_ << "', "; + out << "parentID='" << parentID_ << "', "; + out << "start='" << startSinceEpoch << "', "; + out << "end='" << endSinceEpoch << "', "; + out << "metaData='{"; + for (auto data : metaData_) { + out << getTraceFieldTypeString(data.first) << ": " + << folly::convertTo(data.second) << ", "; + } + out << "}')"; + return out.str(); +} + +std::ostream& operator << (std::ostream& out, const TraceEvent& event) { + out << event.toString(); + return out; +} +} diff --git a/proxygen/lib/utils/TraceEvent.h b/proxygen/lib/utils/TraceEvent.h new file mode 100644 index 0000000000..f213591297 --- /dev/null +++ b/proxygen/lib/utils/TraceEvent.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/utils/Time.h" +#include "proxygen/lib/utils/TraceEventType.h" +#include "proxygen/lib/utils/TraceFieldType.h" + +#include +#include + +namespace proxygen { + +/** + * Simple structure to track timming of event in request flow then we can + * report back to the application. + */ +class TraceEvent { + public: + typedef std::map MetaDataMap; + + explicit TraceEvent(TraceEventType type, uint32_t parentID = 0); + + /** + * Sets the end time to the current time according to the TimeUtil. + */ + void start(const TimeUtil& tm); + + /** + * Sets the start time to the given TimePoint. + */ + void start(TimePoint startTime); + + /** + * Sets the end time to the current time according to the TimeUtil. + */ + void end(const TimeUtil& tm); + + /** + * Sets the end time to the given TimePoint. + */ + void end(TimePoint endTime); + + /** + * @Returns true iff start() has been called on this TraceEvent. + */ + bool hasStarted() const; + + /** + * @Returns true iff end() has been called on this TraceEvent. + */ + bool hasEnded() const; + + TimePoint getStartTime() const { + return start_; + } + + TimePoint getEndTime() const { + return end_; + } + + TraceEventType getType() const { + return type_; + } + + uint32_t getID() const { + return id_; + } + + void setParentID(uint32_t parent) { + parentID_ = parent; + } + + uint32_t getParentID() const { + return parentID_; + } + + void setMetaData(MetaDataMap&& input) { + metaData_ = input; + } + + const MetaDataMap& getMetaData() const { + return metaData_; + } + + bool addMeta(TraceFieldType key, folly::dynamic&& value); + + template + bool increaseIntMeta(TraceFieldType key, const T delta) { + T value = 0; + readIntMeta(key, value); + value += delta; + return addMeta(key, value); + }; + + template + bool readIntMeta(TraceFieldType key, T& dest) const { + if (getMetaData().count(key)) { + DCHECK(getMetaData().at(key).isInt()); + dest = getMetaData().at(key).asInt(); + return true; + } + return false; + }; + + bool readBoolMeta(TraceFieldType key, bool& dest) const; + + bool readStrMeta(TraceFieldType key, std::string& dest) const; + + std::string toString() const; + + friend std::ostream& operator << (std::ostream& out, + const TraceEvent& event); + + private: + + enum State { + NOT_STARTED = 0, + STARTED = 1, + ENDED = 2, + }; + + uint8_t stateFlags_{0}; + TraceEventType type_; + uint32_t id_; + uint32_t parentID_; + TimePoint start_; + TimePoint end_; + MetaDataMap metaData_; +}; + +} diff --git a/proxygen/lib/utils/TraceEventContext.h b/proxygen/lib/utils/TraceEventContext.h new file mode 100644 index 0000000000..0ab5fb6fb2 --- /dev/null +++ b/proxygen/lib/utils/TraceEventContext.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +struct TraceEventObserver; + +struct TraceEventContext { + // Optional parent id for all sub trace events to add. + uint32_t parentID; + + // Optional observer to observe all trace events about to occur + TraceEventObserver* observer; + + explicit TraceEventContext(uint32_t pID = 0, + TraceEventObserver* ob = nullptr) + : parentID(pID) +, observer(ob) { + } +}; + +} diff --git a/proxygen/lib/utils/TraceEventObserver.h b/proxygen/lib/utils/TraceEventObserver.h new file mode 100644 index 0000000000..39375156ed --- /dev/null +++ b/proxygen/lib/utils/TraceEventObserver.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/utils/TraceEvent.h" + +namespace proxygen { + +/* + * Obersver interface to record trace events. + */ +struct TraceEventObserver { + virtual ~TraceEventObserver() {} + virtual void traceEventAvailable(TraceEvent event) noexcept {} +}; + +} diff --git a/proxygen/lib/utils/TraceEventType.txt b/proxygen/lib/utils/TraceEventType.txt new file mode 100644 index 0000000000..70c1a0c38d --- /dev/null +++ b/proxygen/lib/utils/TraceEventType.txt @@ -0,0 +1,18 @@ +/* + * Trace Event Type, please follow the pattern when you add new one. + */ + +RequestExchange, "HTTPRequestExchange" +ResponseBodyRead, "HTTPResponseBodyRead" +PreConnect, "PreConnect" +PostConnect, "PostConnect" +DnsResolution, "DNSResolution" +TcpConnect, "TCPConnect" +TlsSetup, "TLSSetup" +Decompression, "decompression_filter" +CertVerification, "cert_verification" +ProxyConnect, "proxy_connect" +ReadSocket, "read_socket" +WriteSocket, "write_socket" +Scheduling, "scheduling" +NetworkChange, "network_change" diff --git a/proxygen/lib/utils/TraceFieldType.txt b/proxygen/lib/utils/TraceFieldType.txt new file mode 100644 index 0000000000..ec9a8f8e1b --- /dev/null +++ b/proxygen/lib/utils/TraceFieldType.txt @@ -0,0 +1,98 @@ +/* + * Trace Event Meta Data Fields, please follow the pattern when you add new one. + */ + +/* ---- Error Fields ---- */ +ErrorStage, "error_stage" +Error, "error_description" +ProxygenError, "proxygen_error" +HTTPStatus, "http_status" +DirectionError, "error_direction" +CodecError, "codec_error" + +/* ---- Used in RequestExchange ---- */ +Uri, "uri" +IsSecure, "is_secure" +UsingProxy, "using_proxy" +StatusCode, "status_code" +ServerAddr, "server_address" +ServerPort, "server_port" +Protocol, "protocol" +ContentType, "content_type" +UsingSpdy, "using_spdy" +ReqHeaderSize, "request_header_size" +ReqBodySize, "request_body_size" +RspHeaderSize, "response_header_size" +RedirectLocation, "redirect_location" +NumRedirects, "num_redirects" +RedirectResponseCode, "redirect_response_code" +NumRetries, "num_retries" + +/* ---- Used in ResponseBodyRead ---- */ +RspBodySize, "response_body_size" +RspIntvlAvg, "response_interval_average" +RspIntvlStdDev, "response_interval_stddev" +RspNumOnBody, "response_number_on_body" +ServerQuality, "response_server_quality" + +/* ---- Used in PreConnect ---- */ +NewConnection, "new_connection" +SessionManagerType, "sess_mgr_type" +InFlightConns, "in_flight_conns" + +/* ---- Used in PostConnect ---- */ +NewSession, "new_session" +OriginalParentID, "original_parent_id" +NumWaiting, "num_waiting" + +/* ---- Used in DnsResolution ---- */ +HostName, "host_name" +IpAddr, "ip_address" +Port, "port" +CNameRedirects, "cname_redirects" +RedirectTime, "redirect_time" +NumberResolvers, "number_resolvers" +RequestFamily, "request_family" +NumberAnswers, "number_answers" + +/* ---- Used in TlsSetup ---- */ +TLSReused, "tls_reused" +TLSCacheHit, "tls_cache_hit" +CipherName, "cipher_name" +TLSVersion, "ssl_version" + +/* ---- Used in Decompression ---- */ +CompressedSizeKey, "compressed_size" +UncompressedSizeKey, "uncompressed_size" +CompressionType, "compression_type" + +/* ---- Used in CertVerification ---- */ +VerifiedSuccess, "verified_success" +VerifiedChain, "verified_chain" +VerifiedParams, "verified_params" +VerifiedTime, "verified_time" +VerifiedServerAddress, "verified_server_address" +VerifiedProxyAddress, "verified_proxy_address" +VerifiedError, "verified_error" +VerifiedReason, "verified_reason" +VerifiedHostname, "verified_hostname" + +/* ---- Used in ProxyConnect ---- */ +ProxyHost, "proxy_host" +ProxyPort, "proxy_port" + +/* ---- Used in ReadSocket ---- */ +ReadBytes, "read_bytes" + +/* ---- Used in WriteSocket ---- */ +WriteBytes, "write_bytes" + +/* ---- Used in Scheduling ---- */ +SchedulerType, "scheduler_type" +InitialPriority, "initial_priority" +SizeOfQueue, "size_of_queue" + +/* ---- Used in NetworkChange ---- */ +PreviousState, "previous_state" +CurrentState, "current_state" +NetworkID, "network_id" diff --git a/proxygen/lib/utils/UtilInl.h b/proxygen/lib/utils/UtilInl.h new file mode 100644 index 0000000000..ad2bada342 --- /dev/null +++ b/proxygen/lib/utils/UtilInl.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace proxygen { + +// Case-insensitive string comparison +inline bool caseInsensitiveEqual(folly::StringPiece s, folly::StringPiece t) { + if (s.size() != t.size()) { + return false; + } + return std::equal(s.begin(), s.end(), t.begin(), folly::asciiCaseInsensitive); +} + +} diff --git a/proxygen/lib/utils/gen_trace_event_constants.py b/proxygen/lib/utils/gen_trace_event_constants.py new file mode 100755 index 0000000000..f0174f2b60 --- /dev/null +++ b/proxygen/lib/utils/gen_trace_event_constants.py @@ -0,0 +1,163 @@ +#!/bin/env python +# @lint-avoid-python-3-compatibility-imports + +import os +import re +import sys +import optparse + +def main(argv): + # args parser + parser = optparse.OptionParser() + parser.add_option("--install_dir", dest="install_dir", type="string", + default=None, help="Absolute path to generate files") + parser.add_option("--fbcode_dir", dest="fbcode_dir", type="string", + default=None, help="Absolute path to fbcode directory") + parser.add_option("--input_files", dest="input_files", type="string", + default=None, help="Relative path of input file") + parser.add_option("--output_scope", dest="output_scope", type="string", + default=None, help="namespace / package of output file") + parser.add_option("--output_type", dest="output_type", type="choice", + choices=["java", "cpp"], default=None, + help="File type to generate") + parser.add_option("--header_path", dest="header_path", type="string", + default=None, help="Relative path to cpp header") + options, _ = parser.parse_args() + + assert options.install_dir is not None, "Missing arg: --install_dir" + assert options.fbcode_dir is not None, "Missing arg: --fbcode_dir" + assert options.input_files is not None, "Missing arg: --input_files" + assert options.output_scope is not None, "Missing arg: --output_scope" + assert options.output_type is not None, "Missing arg: --output_type" + + file_names = options.input_files.split(",") + for file_name in file_names: + # strip the file extension and use the file name for class name + class_name = file_name.split(".")[0] + + # parse items from source + items = [] + with open(file_name, 'r') as inf: + for line in inf: + sp = re.match(r'(.*), \"(.*)\"', line, re.I) + if sp: + items.append((sp.group(1), sp.group(2))) + + if options.output_type == "java": + gen_java(items, + class_name, + options.install_dir, + options.output_scope) + + elif options.output_type == "cpp": + assert options.header_path is not None, "Missing arg: --header_path" + + gen_cpp_header(items, + class_name, + options.install_dir, + options.output_scope) + gen_cpp_source(items, + class_name, + options.install_dir, + options.header_path, + options.output_scope) + +""" +Generate java interface class +""" +def gen_java(items, class_name, install_dir, output_scope): + packages = output_scope.split(".") + file_name = "%s.java" % class_name + file_path = os.path.join(*([install_dir, "src"] + packages)) + output_file = os.path.join(file_path, file_name) + + if not os.path.exists(file_path): + os.makedirs(file_path) + + with open(output_file, 'w+') as outf: + outf.write('// Copyright 2004-present Facebook. All Rights Reserved.\n') + outf.write('// ** AUTOGENERATED FILE. DO NOT HAND-EDIT **\n\n') + outf.write('package %s;\n\n' % ".".join(packages)) + outf.write('public interface %s {\n' % class_name) + + for item in items: + outf.write(' public static final String %s = "%s";\n' % + (item[0], item[1])) + + outf.write('}\n') + +""" +Generate cpp enum class and provide convert function from / to string +""" +def gen_cpp_header(items, class_name, install_dir, output_scope): + namespaces = output_scope.split("::") + file_name = "%s.h" % class_name + output_file = os.path.join(install_dir, file_name) + + with open(output_file, 'w+') as outf: + outf.write('// Copyright 2004-present Facebook. All Rights Reserved.\n') + outf.write('// ** AUTOGENERATED FILE. DO NOT HAND-EDIT **\n\n') + outf.write('#pragma once\n\n') + outf.write('#include \n\n') + for ns in namespaces: + outf.write('namespace %s { ' % ns) + outf.write('\n\n') + + # generate enum class + outf.write('enum class %s {\n' % class_name) + for item in items: + outf.write(' %s,\n' % item[0]) + outf.write('};\n\n') + + # enum to string convert function + outf.write('extern const std::string& get%sString(%s);\n' + % (class_name, class_name)) + + for ns in namespaces: + outf.write('}') + outf.write('\n\n') + +""" +Generate cpp const string and implement convert function +""" +def gen_cpp_source(items, class_name, install_dir, header_path, output_scope): + namespaces = output_scope.split("::") + file_name = "%s.cpp" % class_name + output_file = os.path.join(install_dir, file_name) + + with open(output_file, 'w+') as outf: + outf.write('// Copyright 2004-present Facebook. All Rights Reserved.\n') + outf.write('// ** AUTOGENERATED FILE. DO NOT HAND-EDIT **\n\n') + outf.write('#include "%s/%s.h"\n\n' % (header_path, class_name)) + outf.write('#include \n\n') + + for ns in namespaces: + outf.write('namespace %s { ' % ns) + outf.write('\n\n') + + # generate enum to string convert function + outf.write('const std::string& get%sString(%s type) {\n' % + (class_name, class_name)) + + outf.write(' static const std::string k%sInvalidType = "";\n' % + class_name) + + # const string names + for item in items: + outf.write(' static const std::string k%s%s = "%s";\n' % + (class_name, item[0], item[1])) + + outf.write('\n switch (type) {\n') + for item in items: + outf.write(' case %s::%s : return k%s%s;\n' % + (class_name, item[0], class_name, item[0])) + outf.write(' }\n') + outf.write(' return k%sInvalidType;\n' % class_name) + outf.write('};\n\n') + + for ns in namespaces: + outf.write('}') + outf.write('\n\n') + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/proxygen/lib/utils/test/CryptUtilTest.cpp b/proxygen/lib/utils/test/CryptUtilTest.cpp new file mode 100644 index 0000000000..2fbd401934 --- /dev/null +++ b/proxygen/lib/utils/test/CryptUtilTest.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/CryptUtil.h" + +#include +#include + +using namespace proxygen; + +using folly::ByteRange; + +TEST(CryptUtilTest, Base64EncodeTest) { + ASSERT_EQ("", + base64Encode( + ByteRange( + reinterpret_cast(""), (size_t) 0))); + ASSERT_EQ("YQ==", + base64Encode( + ByteRange( + reinterpret_cast("a"), 1))); + ASSERT_EQ("YWE=", + base64Encode( + ByteRange( + reinterpret_cast("aa"), 2))); + ASSERT_EQ("QWxhZGRpbjpvcGVuIHNlc2FtZQ==", + base64Encode( + ByteRange( + reinterpret_cast("Aladdin:open sesame"), 19))); +} + +TEST(CryptUtilTest, MD5EncodeTest) { + ASSERT_EQ("d41d8cd98f00b204e9800998ecf8427e", + md5Encode( + ByteRange( + reinterpret_cast(""), + (size_t) 0))); + ASSERT_EQ("a7a93b8ac14a48faa68e4afb57b00fc7", + md5Encode( + ByteRange( + reinterpret_cast("Aladdin:open sesame"), + 19))); +} diff --git a/proxygen/lib/utils/test/GenericFilterTest.cpp b/proxygen/lib/utils/test/GenericFilterTest.cpp new file mode 100644 index 0000000000..a662bda9c2 --- /dev/null +++ b/proxygen/lib/utils/test/GenericFilterTest.cpp @@ -0,0 +1,538 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/FilterChain.h" + +#include +#include +#include +#include +#include +#include + +using namespace folly; +using namespace proxygen; +using namespace testing; + +using std::unique_ptr; + +namespace detail { + +// This is defined in boost 1.53, but we only have 1.51 so far + +template +T* get_pointer(const unique_ptr& ptr) { + return ptr.get(); +} + +template +T* get_pointer(T* ptr) { + return ptr; +} + +} + +class TesterInterface { + public: + class Callback { + public: + virtual ~Callback() {} + virtual void onA() = 0; + }; + virtual ~TesterInterface() {} + virtual void setCallback(Callback* cb) = 0; + virtual void doA() = 0; +}; + +class MockTester: public TesterInterface { + public: + Callback* cb_{nullptr}; + void setCallback(Callback* cb) override { + cb_ = cb; + } + MOCK_METHOD0(doA, void()); +}; + +class MockTesterCallback: public TesterInterface::Callback { + public: + MOCK_METHOD0(onA, void()); +}; + +template +class TestFilter: public GenericFilter { + public: + TestFilter(): + GenericFilter(true, true) {} + + TestFilter(bool calls, bool callbacks): + GenericFilter(calls, callbacks) {} + + void setCallback(TesterInterface::Callback* cb) override { + this->setCallbackInternal(cb); + } + void doA() override { + do_++; + this->call_->doA(); + } + void onA() override { + on_++; + this->callback_->onA(); + } + uint32_t do_{0}; + uint32_t on_{0}; + uint32_t id_{idCounter_++}; + static uint32_t idCounter_; +}; +template +uint32_t TestFilter::idCounter_ = 0; + +template +class TestFilterNoCallback: public TestFilter { + public: + TestFilterNoCallback(): TestFilter(true, false) {} +}; + +template +class TestFilterNoCall: public TestFilter { + public: + TestFilterNoCall(): TestFilter(false, true) {} +}; + +template +class TestFilterNoCallbackNoCall: public TestFilter { + public: + TestFilterNoCallbackNoCall(): TestFilter(false, false) {} +}; + +template +typename std::enable_if>::type getTester() { + return folly::make_unique(); +} + +template +typename std::enable_if::type getTester() { + return new MockTester(); +} + +template +class GenericFilterTest: public testing::Test { + public: + void basicTest(); + + void testFilters(const std::deque*>& filters, + MockTesterCallback* expectedCb); + + void SetUp() override { + chain_ = folly::make_unique< + FilterChain, + &TesterInterface::setCallback, + Owned>>(getTester()); + chain().setCallback(&callback_); + actor_ = CHECK_NOTNULL(static_cast(chain_->call())); + } + FilterChain, + &TesterInterface::setCallback, Owned>& chain() { + return *chain_; + } + + template + typename std::enable_if::type getFilter() { + return new FilterT(); + } + + template + typename std::enable_if>::type getFilter() { + return folly::make_unique(); + } + + template + void addFilterToChain(std::deque*>& refs) { + auto f = getFilter(); + refs.push_front(::detail::get_pointer(f)); + chain().addFilters(std::move(f)); + } + + std::deque*> getRandomFilters(unsigned num) { + std::deque*> filters; + srand(0); + for (unsigned i = 0; i < num; ++i) { + auto r = rand() % 4; + if (r == 0) { + addFilterToChain>(filters); + } else if (r == 1) { + addFilterToChain>(filters); + } else if (r == 2) { + addFilterToChain>(filters); + } else if (r == 3) { + addFilterToChain>(filters); + } + basicTest(); + } + return filters; + } + + MockTesterCallback callback_; + unique_ptr, + &TesterInterface::setCallback, Owned>> chain_; + MockTester* actor_{nullptr}; +}; + +template +void GenericFilterTest::basicTest() { + InSequence enforceOrder; + + // Test call side + EXPECT_CALL(*actor_, doA()); + chain()->doA(); + + // Now poke the callback side + EXPECT_CALL(callback_, onA()); + CHECK_NOTNULL(actor_->cb_); + actor_->cb_->onA(); +} + +template void GenericFilterTest::testFilters( + const std::deque*>& filters, + MockTesterCallback* expectedCb) { + for (auto f: filters) { + f->do_ = 0; + f->on_ = 0; + } + // Call + EXPECT_CALL(*actor_, doA()); + chain()->doA(); + // Callback + if (expectedCb) { + EXPECT_CALL(*expectedCb, onA()); + CHECK_NOTNULL(actor_->cb_); + actor_->cb_->onA(); + } + for (auto f: filters) { + if (f->kWantsCalls_) { + EXPECT_EQ(f->do_, 1); + } else { + EXPECT_EQ(f->do_, 0); + } + if (f->kWantsCallbacks_) { + EXPECT_EQ(f->on_, expectedCb ? 1 : 0); + } else { + EXPECT_EQ(f->on_, 0); + } + } +} + +typedef GenericFilterTest OwnedGenericFilterTest; +typedef GenericFilterTest UnownedGenericFilterTest; + +TEST_F(OwnedGenericFilterTest, empty_chain) { + basicTest(); +} + +TEST_F(OwnedGenericFilterTest, single_elem_chain) { + auto filterUnique = folly::make_unique>(); + auto filter = filterUnique.get(); + chain().addFilters(std::move(filterUnique)); + EXPECT_EQ(filter->do_, 0); + EXPECT_EQ(filter->on_, 0); + basicTest(); + EXPECT_EQ(filter->do_, 1); + EXPECT_EQ(filter->on_, 1); +} + +TEST_F(OwnedGenericFilterTest, multi_elem_chain) { + auto f1 = folly::make_unique>(); + auto f2 = folly::make_unique>(); + auto f3 = folly::make_unique>(); + TestFilter* fp1 = f1.get(); + TestFilter* fp2 = f2.get(); + TestFilter* fp3 = f3.get(); + chain().addFilters(std::move(f1), std::move(f2), std::move(f3)); + basicTest(); + EXPECT_EQ(fp1->do_, 1); + EXPECT_EQ(fp1->on_, 1); + EXPECT_EQ(fp2->do_, 1); + EXPECT_EQ(fp2->on_, 1); + EXPECT_EQ(fp3->do_, 1); + EXPECT_EQ(fp3->on_, 1); +} + +TEST_F(OwnedGenericFilterTest, multi_elem_multi_add) { + std::deque*> filters; + for (unsigned i = 0; i < 10; ++i) { + auto filter = folly::make_unique>(); + filters.push_back(filter.get()); + chain().addFilters(std::move(filter)); + } + basicTest(); + for (auto filter: filters) { + EXPECT_EQ(filter->do_, 1); + EXPECT_EQ(filter->on_, 1); + } +} + +TEST_F(OwnedGenericFilterTest, wants) { + auto f1 = folly::make_unique>(); + auto f2 = folly::make_unique>(); + auto f3 = folly::make_unique>(); + auto f4 = folly::make_unique>(); + TestFilter* fp1 = f1.get(); + TestFilter* fp2 = f2.get(); + TestFilter* fp3 = f3.get(); + TestFilter* fp4 = f4.get(); + chain().addFilters(std::move(f1), std::move(f2), + std::move(f3), std::move(f4)); + basicTest(); + EXPECT_EQ(fp1->do_, 1); + EXPECT_EQ(fp1->on_, 1); + // Only calls + EXPECT_EQ(fp2->do_, 1); + EXPECT_EQ(fp2->on_, 0); + // Only callbacks + EXPECT_EQ(fp3->do_, 0); + EXPECT_EQ(fp3->on_, 1); + // No callbacks or calls + EXPECT_EQ(fp4->do_, 0); + EXPECT_EQ(fp4->on_, 0); +} + +TEST_F(OwnedGenericFilterTest, wants_multi_add) { + auto f1 = folly::make_unique>(); + auto f2 = folly::make_unique>(); + TestFilter* fp1 = f1.get(); + TestFilter* fp2 = f2.get(); + chain().addFilters(std::move(f1)); + basicTest(); + + EXPECT_EQ(fp1->do_, 1); + EXPECT_EQ(fp1->on_, 0); + EXPECT_EQ(fp2->do_, 0); + EXPECT_EQ(fp2->on_, 0); + + chain().addFilters(std::move(f2)); + basicTest(); + + EXPECT_EQ(fp1->do_, 2); + EXPECT_EQ(fp1->on_, 0); + EXPECT_EQ(fp2->do_, 0); + EXPECT_EQ(fp2->on_, 1); +} + +TEST_F(OwnedGenericFilterTest, wants_multi_add_hard) { + const unsigned NUM_FILTERS = 5000; + auto filters = getRandomFilters(NUM_FILTERS); + // Now check the counts on each filter. Filters are pushed to the front + // of the chain, so filters towards the front have low call/callback counts + for (unsigned i = 0; i < NUM_FILTERS; ++i) { + auto f = filters[i]; + if (f->kWantsCalls_) { + EXPECT_EQ(f->do_, i + 1); + } else { + EXPECT_EQ(f->do_, 0); + } + if (f->kWantsCallbacks_) { + EXPECT_EQ(f->on_, i + 1); + } else { + EXPECT_EQ(f->on_, 0); + } + } +} + +TEST_F(OwnedGenericFilterTest, change_callback) { + // The call-only filter in the chain doesn't want callbacks, so doing + // chain()->setCallback() is an error! Instead, must use chain().setCallback() + auto f = folly::make_unique>(); + MockTesterCallback callback2; + + TestFilter* fp = f.get(); + chain().addFilters(std::move(f)); + basicTest(); + + EXPECT_EQ(fp->do_, 1); + EXPECT_EQ(fp->on_, 0); + + chain().setCallback(&callback2); + EXPECT_EQ(actor_->cb_, &callback2); + EXPECT_CALL(callback2, onA()); + actor_->cb_->onA(); + + EXPECT_EQ(fp->on_, 0); +} + +TEST_F(UnownedGenericFilterTest, all) { + const unsigned NUM_FILTERS = 5000; + auto filters = getRandomFilters(NUM_FILTERS); + // Now check the counts on each filter + unsigned i = 0; + for (auto f: filters) { + if (f->kWantsCalls_) { + EXPECT_EQ(f->do_, i + 1); + } else { + EXPECT_EQ(f->do_, 0); + } + if (f->kWantsCallbacks_) { + EXPECT_EQ(f->on_, i + 1); + } else { + EXPECT_EQ(f->on_, 0); + } + delete f; + ++i; + } + delete actor_; +} + +TEST_F(OwnedGenericFilterTest, set_null_cb) { + // Some objects have a special behavior when the callback is set to + // nullptr. So in this case, we need to make sure it propagates + auto filters = getRandomFilters(100); + chain().setCallback(nullptr); + CHECK(nullptr == actor_->cb_); + + testFilters(filters, nullptr); + + MockTesterCallback head; + chain().setCallback(&head); + + testFilters(filters, &head); + + TesterInterface::Callback* cb = &head; + for (auto f: filters) { + if (f->kWantsCallbacks_) { + cb = f; + } + } + // The actor's callback should be the last filter in the chain that + // wants callbacks + ASSERT_EQ(actor_->cb_, cb); +} + +// This class owns itself +class TestFilterOddDeleteDo: public TestFilter { + public: + explicit TestFilterOddDeleteDo(int* deletions): + TestFilter(true, true), + deletions_(CHECK_NOTNULL(deletions)) {} + ~TestFilterOddDeleteDo() { + ++*deletions_; + } + + void doA() override { + auto call = call_; + if (id_ % 2) { + delete this; + } else if (times_++) { + delete this; + } + call->doA(); + }; + unsigned times_{0}; + int* const deletions_; +}; + +TEST_F(UnownedGenericFilterTest, delete_do) { + // Test where a filter in the middle of the chain deletes itself early + int deletions = 0; + + for (unsigned i = 0; i < 4; ++i) { + chain().addFilters(new TestFilterOddDeleteDo(&deletions)); + } + + for (unsigned i = 0; i < 2; ++i) { + // First time around, the odd id's get deleted + // Second time should just forward the calls normally + EXPECT_CALL(*actor_, doA()); + chain()->doA(); + EXPECT_EQ(deletions, (i + 1) * 2); + } + basicTest(); + delete actor_; +} + +template +class TestFilterOddDeleteOn: public TestFilter { + public: + explicit TestFilterOddDeleteOn(int* deletions): + deletions_(CHECK_NOTNULL(deletions)) {} + ~TestFilterOddDeleteOn() { + ++*deletions_; + } + + void onA() override { + auto callback = this->callback_; + if (this->id_ % 2) { + delete this; + } else if (times_++) { + delete this; + } + callback->onA(); + }; + unsigned times_{0}; + int* const deletions_; +}; + +TEST_F(UnownedGenericFilterTest, delete_on) { + // Test where a filter in the middle of the chain deletes itself early + int deletions = 0; + + for (unsigned i = 0; i < 4; ++i) { + chain().addFilters(new TestFilterOddDeleteOn<>(&deletions)); + } + + for (unsigned i = 0; i < 2; ++i) { + // First time around, the odd id's get deleted + // Second time should just forward the calls normally + EXPECT_CALL(callback_, onA()); + actor_->cb_->onA(); + EXPECT_EQ(deletions, (i + 1) * 2); + } + basicTest(); + delete actor_; +} + +TEST_F(OwnedGenericFilterTest, delete_chain) { + // Add some filters to the chain and reset the chain. Make sure all the + // filters are deleted. + const unsigned NUM_FILTERS = 1000; + int deletions = 0; + for (unsigned i = 0; i < NUM_FILTERS; ++i) { + chain().addFilters( + folly::make_unique>(&deletions)); + } + chain_.reset(); + EXPECT_EQ(deletions, NUM_FILTERS); +} + +TEST_F(OwnedGenericFilterTest, get_chain_end) { + for (unsigned i = 1; i < 100; ++i) { + auto filters = getRandomFilters(i); + EXPECT_EQ(actor_, &chain().getChainEnd()); + } +} + +TEST_F(OwnedGenericFilterTest, set_destination) { + auto filters = getRandomFilters(20); + EXPECT_CALL(*actor_, doA()); + chain()->doA(); + auto tester2 = getTester(); + actor_ = tester2.get(); + auto oldTester = chain().setDestination(std::move(tester2)); + EXPECT_CALL(*actor_, doA()); + chain()->doA(); +} diff --git a/proxygen/lib/utils/test/HTTPTimeTest.cpp b/proxygen/lib/utils/test/HTTPTimeTest.cpp new file mode 100644 index 0000000000..401bb29810 --- /dev/null +++ b/proxygen/lib/utils/test/HTTPTimeTest.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/HTTPTime.h" + +#include + +using proxygen::parseHTTPDateTime; + +TEST(HTTPTimeTests, InvalidTimeTest) { + uint64_t a; + EXPECT_FALSE(parseHTTPDateTime("Hello, World").hasValue()); + EXPECT_FALSE(parseHTTPDateTime("Sun, 33 Nov 1994 08:49:37 GMT").hasValue()); + EXPECT_FALSE(parseHTTPDateTime("Sun, 06 Nov 1800").hasValue()); +} + +TEST(HTTPTimeTests, ValidTimeTest) { + // From http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3 + EXPECT_TRUE(parseHTTPDateTime("Sun, 06 Nov 1994 08:49:37 GMT").hasValue()); + EXPECT_TRUE(parseHTTPDateTime("Sunday, 06-Nov-94 08:49:37 GMT").hasValue()); + EXPECT_TRUE(parseHTTPDateTime("Sun Nov 6 08:49:37 1994").hasValue()); +} + +TEST(HTTPTimeTests, EqualTimeTest) { + auto a = parseHTTPDateTime("Thu, 07 Mar 2013 08:49:37 GMT"); + EXPECT_TRUE(a.hasValue()); + auto b = parseHTTPDateTime("Thursday, 07-Mar-13 08:49:37 GMT"); + EXPECT_TRUE(b.hasValue()); + auto c = parseHTTPDateTime("Thu Mar 7 08:49:37 2013"); + EXPECT_TRUE(c.hasValue()); + + EXPECT_EQ(a.value(), b.value()); + EXPECT_EQ(a.value(), c.value()); + EXPECT_EQ(b.value(), c.value()); +} + +TEST(HTTPTimeTests, ReallyOldTimeTest) { + auto a = parseHTTPDateTime("Thu, 07 Mar 1770 08:49:37 GMT"); + EXPECT_TRUE(a.hasValue()); + auto b = parseHTTPDateTime("Thu, 07 Mar 1771 08:49:37 GMT"); + EXPECT_TRUE(b.hasValue()); + auto c = parseHTTPDateTime("Thu, 07 Mar 1980 08:49:37 GMT"); + EXPECT_TRUE(c.hasValue()); + + EXPECT_LT(a, b); + EXPECT_LT(a, c); + EXPECT_LT(b, c); +} diff --git a/proxygen/lib/utils/test/Makefile.am b/proxygen/lib/utils/test/Makefile.am new file mode 100644 index 0000000000..7a566c72c5 --- /dev/null +++ b/proxygen/lib/utils/test/Makefile.am @@ -0,0 +1,13 @@ +SUBDIRS = . + +check_PROGRAMS = UtilTests +UtilTests_SOURCES = \ + GenericFilterTest.cpp \ + HTTPTimeTest.cpp \ + ParseURLTest.cpp \ + ResultTest.cpp \ + UtilTest.cpp + +UtilTests_LDADD = ../libutils.la ../../test/libtestmain.la + +TESTS = UtilTests diff --git a/proxygen/lib/utils/test/MockTime.h b/proxygen/lib/utils/test/MockTime.h new file mode 100644 index 0000000000..142fafbf4c --- /dev/null +++ b/proxygen/lib/utils/test/MockTime.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include "proxygen/lib/utils/Time.h" + +#include + +namespace proxygen { + +class MockTimeUtil : public TimeUtil { + public: + + void advance(std::chrono::milliseconds ms) { + t_ += ms; + } + + void setCurrentTime(TimePoint t) { + CHECK(t.time_since_epoch() > t_.time_since_epoch()) + << "Time can not move backwards"; + t_ = t; + } + + void verifyAndClear() { + } + + TimePoint now() const override { + return t_; + } + + private: + TimePoint t_; +}; + +} diff --git a/proxygen/lib/utils/test/ParseURLTest.cpp b/proxygen/lib/utils/test/ParseURLTest.cpp new file mode 100644 index 0000000000..73c6f4ca6b --- /dev/null +++ b/proxygen/lib/utils/test/ParseURLTest.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/ParseURL.h" + +#include + +using proxygen::ParseURL; +using std::string; + +void testParseURL(const string& url, + const string& expectedPath, + const string& expectedQuery, + const string& expectedHost, + const uint16_t expectedPort, + const string& expectedAuthority, + const bool expectedValid = true) { + ParseURL u(url); + + if (expectedValid) { + EXPECT_EQ(url, u.url()); + EXPECT_EQ(expectedPath, u.path()); + EXPECT_EQ(expectedQuery, u.query()); + EXPECT_EQ(expectedHost, u.host()); + EXPECT_EQ(expectedPort, u.port()); + EXPECT_EQ(expectedAuthority, u.authority()); + EXPECT_EQ(expectedValid, u.valid()); + } else { + // invalid, do not need to test values + EXPECT_EQ(expectedValid, u.valid()); + } +} + +void testHostIsIpAddress(const string& url, const bool expected) { + ParseURL u(url); + EXPECT_EQ(expected, u.hostIsIPAddress()); +} + +TEST(ParseURL, HostNoBrackets) { + ParseURL p("/bar"); + + EXPECT_EQ("", p.host()); + EXPECT_EQ("", p.hostNoBrackets()); +} + +TEST(ParseURL, FullyFormedURL) { + testParseURL("http://localhost:80/foo?bar#qqq", "/foo", "bar", + "localhost", 80, "localhost:80"); + testParseURL("http://localhost:80/foo?bar", "/foo", "bar", + "localhost", 80, "localhost:80"); + testParseURL("http://localhost:80/foo", "/foo", "", + "localhost", 80, "localhost:80"); + testParseURL("http://localhost:80/", "/", "", + "localhost", 80, "localhost:80"); + testParseURL("http://localhost:80", "", "", + "localhost", 80, "localhost:80"); + testParseURL("http://localhost", "", "", + "localhost", 0, "localhost"); + testParseURL("http://[2401:db00:2110:3051:face:0:3f:0]/", "/", "", + "[2401:db00:2110:3051:face:0:3f:0]", 0, + "[2401:db00:2110:3051:face:0:3f:0]"); +} + +TEST(ParseURL, NoScheme) { + testParseURL("localhost:80/foo?bar#qqq", "/foo", "bar", + "localhost", 80, "localhost:80"); + testParseURL("localhost:80/foo?bar", "/foo", "bar", + "localhost", 80, "localhost:80"); + testParseURL("localhost:80/foo", "/foo", "", + "localhost", 80, "localhost:80"); + testParseURL("localhost:80/", "/", "", + "localhost", 80, "localhost:80"); + testParseURL("localhost:80", "", "", + "localhost", 80, "localhost:80"); + testParseURL("localhost", "", "", + "localhost", 0, "localhost"); +} + +TEST(ParseURL, NoSchemeIP) { + testParseURL("1.2.3.4:54321/foo?bar#qqq", + "/foo", "bar", "1.2.3.4", 54321, "1.2.3.4:54321"); + testParseURL("[::1]:80/foo?bar", "/foo", "bar", "[::1]", 80, "[::1]:80"); + testParseURL("[::1]/foo?bar", "/foo", "bar", "[::1]", 0, "[::1]"); +} + +TEST(ParseURL, PathOnly) { + testParseURL("/f/o/o?bar#qqq", "/f/o/o", "bar", "", 0, ""); + testParseURL("/f/o/o?bar", "/f/o/o", "bar", "", 0, ""); + testParseURL("/f/o/o", "/f/o/o", "", "", 0, ""); + testParseURL("/", "/", "", "", 0, ""); + testParseURL("?foo=bar", "", "foo=bar", "", 0, ""); + testParseURL("?#", "", "", "", 0, ""); + testParseURL("#/foo/bar", "", "", "", 0, ""); +} + +TEST(ParseURL, QueryIsURL) { + testParseURL("/?ids=http://vlc.afreecodec.com/download/", + "/", "ids=http://vlc.afreecodec.com/download/", "", 0, ""); + testParseURL("/plugins/facepile.php?href=http://www.vakan.nl/hotels", + "/plugins/facepile.php", + "href=http://www.vakan.nl/hotels", "", 0, ""); +} + +TEST(ParseURL, InvalidURL) { + testParseURL("http://tel:198433511/", "", "", "", 0, "", false); + testParseURL("localhost:80/foo#bar?qqq", "", "", "", 0, "", false); + testParseURL("#?", "", "", "", 0, "", false); + testParseURL("#?hello", "", "", "", 0, "", false); + testParseURL("[::1/foo?bar", "", "", "", 0, "", false); + testParseURL("", "", "", "", 0, "", false); +} + +TEST(ParseURL, IsHostIPAddress) { + testHostIsIpAddress("http://127.0.0.1:80", true); + testHostIsIpAddress("127.0.0.1", true); + testHostIsIpAddress("http://[::1]:80", true); + testHostIsIpAddress("[::1]", true); + testHostIsIpAddress("[::AB]", true); + + testHostIsIpAddress("http://localhost:80", false); + testHostIsIpAddress("http://localhost", false); + testHostIsIpAddress("localhost", false); + testHostIsIpAddress("1.2.3.-1", false); + testHostIsIpAddress("1.2.3.999", false); + testHostIsIpAddress("::1", false); + testHostIsIpAddress("[::99999999]", false); + + // invalid url + testHostIsIpAddress("", false); + testHostIsIpAddress("127.0.0.1:80/foo#bar?qqq", false); +} diff --git a/proxygen/lib/utils/test/ResultBenchmark.cpp b/proxygen/lib/utils/test/ResultBenchmark.cpp new file mode 100644 index 0000000000..1f192ec44f --- /dev/null +++ b/proxygen/lib/utils/test/ResultBenchmark.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "folly/Benchmark.h" +#include "proxygen/lib/utils/Result.h" + +#include + +using namespace folly; +using namespace proxygen; + +struct MediumStruct { + int32_t field1; + int32_t field2; +}; + +typedef Result MediumResult; + +static_assert(std::is_nothrow_copy_constructible::value, ""); +static_assert(sizeof(MediumStruct) == 8, ""); + +MediumResult __attribute__ ((__noinline__)) +parseMediumResult(bool which) { + if (which) { + return MediumStruct{1, 2}; + } else { + return 0; + } +} + +uint8_t __attribute__ ((__noinline__)) +parseMediumReference(bool which, MediumStruct& outMedium) { + if (which) { + outMedium.field1 = 1; + outMedium.field2 = 2; + return 1; + } else { + return 0; + } +} + +BENCHMARK(result_medium_struct_or_byte, numIters) { + const auto halfIters = numIters / 2; + MediumResult result = 2; + bool succeed = false; + for (unsigned i = 0; i < numIters; ++i) { + result = parseMediumResult(i < halfIters); + succeed = result.isOk(); + } + VLOG(4) << succeed << result.isError(); +} + +BENCHMARK(reference_medium_struct_or_byte, numIters) { + const auto halfIters = numIters / 2; + MediumStruct result; + bool succeed = false; + for (unsigned i = 0; i < numIters; ++i) { + auto res = parseMediumReference(i < halfIters, result); + succeed = res != 0; + } + VLOG(4) << succeed << result.field1; +} + +int main(int argc, char* argv[]) { + folly::runBenchmarks(); + return 0; +} diff --git a/proxygen/lib/utils/test/ResultTest.cpp b/proxygen/lib/utils/test/ResultTest.cpp new file mode 100644 index 0000000000..b14b5609d9 --- /dev/null +++ b/proxygen/lib/utils/test/ResultTest.cpp @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/Result.h" + +#include +#include + +using namespace proxygen; +using namespace std; + +static_assert(sizeof(Result) == 2, ""); + +TEST(Result, Copy) { + // This instance of Result is copyable + Result expected = 'b'; + Result error = 42; + + EXPECT_TRUE(expected.isOk()); + EXPECT_FALSE(expected.isError()); + EXPECT_FALSE(error.isOk()); + EXPECT_TRUE(error.isError()); + + EXPECT_EQ('b', expected.ok()); + EXPECT_EQ(42, error.error()); + + auto expectedCopy = expected; + auto errorCopy = error; + + EXPECT_TRUE(expectedCopy.isOk()); + EXPECT_FALSE(expectedCopy.isError()); + EXPECT_FALSE(errorCopy.isOk()); + EXPECT_TRUE(errorCopy.isError()); + + EXPECT_EQ(expectedCopy.ok(), expected.ok()); + EXPECT_EQ(errorCopy.error(), error.error()); +} + +TEST(Result, Move) { + struct ErrStruct { + ErrStruct(int foo): val(foo) {} + ErrStruct(const ErrStruct& src): val(src.val) {} + ErrStruct(ErrStruct&& src) noexcept: val(src.val) { + src.moved = true; + } + int val; + bool moved{false}; + }; + static_assert(std::is_nothrow_move_constructible::value, + "This struct must be nothrow move constructible"); + + Result original{4}; + + // move + auto error = std::move(original); + + EXPECT_TRUE(original.error().moved); + EXPECT_TRUE(!error.error().moved); +} + +TEST(Result, Assign) { + bool intWasDeleted = false; + auto deleter = [&](int* ptr) { + intWasDeleted = true; + delete ptr; + }; + typedef unique_ptr int_ptr; + + // emplace + Result result = 'a'; + + // copy assign + EXPECT_EQ('a', result.ok()); + result = 'b'; + EXPECT_EQ('b', result.ok()); + + // move assign + result = int_ptr(new int(5), deleter); + EXPECT_EQ(5, *result.error()); + + // check deletion + EXPECT_FALSE(intWasDeleted); + result = 'c'; + EXPECT_TRUE(intWasDeleted); + EXPECT_EQ('c', result.ok()); +} + +TEST(Result, Emplace) { + int constructCount = 0; + struct Expensive { + /* implicit */ Expensive(int* counter) { + ++*counter; + } + Expensive(const Expensive& other) { + EXPECT_TRUE(false); // Should not be invoked + } + Expensive(Expensive&& other) { + EXPECT_TRUE(false); // Should not be invoked + } + }; + + // Use emplacement so we only construct the Expensive object once. + auto tester = make_ok(&constructCount); + EXPECT_EQ(1, constructCount); + EXPECT_TRUE(tester.isOk()); +} diff --git a/proxygen/lib/utils/test/UtilTest.cpp b/proxygen/lib/utils/test/UtilTest.cpp new file mode 100644 index 0000000000..378876e289 --- /dev/null +++ b/proxygen/lib/utils/test/UtilTest.cpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "proxygen/lib/utils/UtilInl.h" + +#include + +using namespace proxygen; + +TEST(UtilTest, CaseInsensitiveEqual) { + ASSERT_TRUE(caseInsensitiveEqual("foo", "FOO")); + ASSERT_TRUE(caseInsensitiveEqual(std::string("foo"), "FOO")); + ASSERT_FALSE(caseInsensitiveEqual(std::string("foo"), "FOO2")); + ASSERT_FALSE(caseInsensitiveEqual("fo", "FOO")); + ASSERT_FALSE(caseInsensitiveEqual("FO", "FOO")); +} diff --git a/proxygen/lib/workers/WorkerThread.h b/proxygen/lib/workers/WorkerThread.h new file mode 100644 index 0000000000..2bd5a87195 --- /dev/null +++ b/proxygen/lib/workers/WorkerThread.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2014, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#ifndef WORKERTHREAD_H +#define WORKERTHREAD_H + +// Temporary stub to make hhvm-third-party compile +#include "proxygen/lib/services/WorkerThread.h" + +#endif diff --git a/proxygen/m4/ac_cxx_compile_stdcxx_0x.m4 b/proxygen/m4/ac_cxx_compile_stdcxx_0x.m4 new file mode 100644 index 0000000000..ee2947304c --- /dev/null +++ b/proxygen/m4/ac_cxx_compile_stdcxx_0x.m4 @@ -0,0 +1,109 @@ +# =========================================================================== +# http://autoconf-archive.cryp.to/ac_cxx_compile_stdcxx_0x.html +# =========================================================================== +# +# SYNOPSIS +# +# AC_CXX_COMPILE_STDCXX_0X +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++0x +# standard. +# +# LAST MODIFICATION +# +# 2008-04-17 +# +# COPYLEFT +# +# Copyright (c) 2008 Benjamin Kosnik +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. + +AC_DEFUN([AC_CXX_COMPILE_STDCXX_0X], [ + AC_CACHE_CHECK(if g++ supports C++0x features without additional flags, + ac_cv_cxx_compile_cxx0x_native, + [AC_LANG_SAVE + AC_LANG_CPLUSPLUS + AC_TRY_COMPILE([ + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + typedef check> right_angle_brackets; + + int a; + decltype(a) b; + + typedef check check_type; + check_type c; + check_type&& cr = static_cast(c);],, + ac_cv_cxx_compile_cxx0x_native=yes, ac_cv_cxx_compile_cxx0x_native=no) + AC_LANG_RESTORE + ]) + + AC_CACHE_CHECK(if g++ supports C++0x features with -std=c++0x, + ac_cv_cxx_compile_cxx0x_cxx, + [AC_LANG_SAVE + AC_LANG_CPLUSPLUS + ac_save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS -std=c++0x" + AC_TRY_COMPILE([ + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + typedef check> right_angle_brackets; + + int a; + decltype(a) b; + + typedef check check_type; + check_type c; + check_type&& cr = static_cast(c);],, + ac_cv_cxx_compile_cxx0x_cxx=yes, ac_cv_cxx_compile_cxx0x_cxx=no) + CXXFLAGS="$ac_save_CXXFLAGS" + AC_LANG_RESTORE + ]) + + AC_CACHE_CHECK(if g++ supports C++0x features with -std=gnu++0x, + ac_cv_cxx_compile_cxx0x_gxx, + [AC_LANG_SAVE + AC_LANG_CPLUSPLUS + ac_save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS -std=gnu++0x" + AC_TRY_COMPILE([ + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + typedef check> right_angle_brackets; + + int a; + decltype(a) b; + + typedef check check_type; + check_type c; + check_type&& cr = static_cast(c);],, + ac_cv_cxx_compile_cxx0x_gxx=yes, ac_cv_cxx_compile_cxx0x_gxx=no) + CXXFLAGS="$ac_save_CXXFLAGS" + AC_LANG_RESTORE + ]) + + if test "$ac_cv_cxx_compile_cxx0x_native" = yes || + test "$ac_cv_cxx_compile_cxx0x_cxx" = yes || + test "$ac_cv_cxx_compile_cxx0x_gxx" = yes; then + AC_DEFINE(HAVE_STDCXX_0X,,[Define if g++ supports C++0x features. ]) + else + AC_MSG_ERROR([Could not find cxx0x support in g++]) + fi +]) diff --git a/proxygen/m4/ax_boost_base.m4 b/proxygen/m4/ax_boost_base.m4 new file mode 100644 index 0000000000..3507788514 --- /dev/null +++ b/proxygen/m4/ax_boost_base.m4 @@ -0,0 +1,258 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_boost_base.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_BASE([MINIMUM-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +# +# DESCRIPTION +# +# Test for the Boost C++ libraries of a particular version (or newer) +# +# If no path to the installed boost library is given the macro searchs +# under /usr, /usr/local, /opt and /opt/local and evaluates the +# $BOOST_ROOT environment variable. Further documentation is available at +# . +# +# This macro calls: +# +# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# +# And sets: +# +# HAVE_BOOST +# +# LICENSE +# +# Copyright (c) 2008 Thomas Porschberg +# Copyright (c) 2009 Peter Adolphs +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 20 + +AC_DEFUN([AX_BOOST_BASE], +[ +AC_ARG_WITH([boost], + [AS_HELP_STRING([--with-boost@<:@=ARG@:>@], + [use Boost library from a standard location (ARG=yes), + from the specified location (ARG=), + or disable it (ARG=no) + @<:@ARG=yes@:>@ ])], + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ac_boost_path="" + else + want_boost="yes" + ac_boost_path="$withval" + fi + ], + [want_boost="yes"]) + + +AC_ARG_WITH([boost-libdir], + AS_HELP_STRING([--with-boost-libdir=LIB_DIR], + [Force given directory for boost libraries. Note that this will override library path detection, so use this parameter only if default library detection fails and you know exactly where your boost libraries are located.]), + [ + if test -d "$withval" + then + ac_boost_lib_path="$withval" + else + AC_MSG_ERROR(--with-boost-libdir expected directory name) + fi + ], + [ac_boost_lib_path=""] +) + +if test "x$want_boost" = "xyes"; then + boost_lib_version_req=ifelse([$1], ,1.20.0,$1) + boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'` + boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'` + boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'` + boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test "x$boost_lib_version_req_sub_minor" = "x" ; then + boost_lib_version_req_sub_minor="0" + fi + WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor` + AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req) + succeeded=no + + dnl On 64-bit systems check for system libraries in both lib64 and lib. + dnl The former is specified by FHS, but e.g. Debian does not adhere to + dnl this (as it rises problems for generic multi-arch support). + dnl The last entry in the list is chosen by default when no libraries + dnl are found, e.g. when only header-only libraries are installed! + libsubdirs="lib" + ax_arch=`uname -m` + if test $ax_arch = x86_64 -o $ax_arch = ppc64 -o $ax_arch = s390x -o $ax_arch = sparc64; then + libsubdirs="lib64 lib lib64" + fi + + dnl first we check the system location for boost libraries + dnl this location ist chosen if boost libraries are installed with the --layout=system option + dnl or if you install boost with RPM + if test "$ac_boost_path" != ""; then + BOOST_CPPFLAGS="-I$ac_boost_path/include" + for ac_boost_path_tmp in $libsubdirs; do + if test -d "$ac_boost_path"/"$ac_boost_path_tmp" ; then + BOOST_LDFLAGS="-L$ac_boost_path/$ac_boost_path_tmp" + break + fi + done + elif test "$cross_compiling" != yes; then + for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then + for libsubdir in $libsubdirs ; do + if ls "$ac_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + BOOST_LDFLAGS="-L$ac_boost_path_tmp/$libsubdir" + BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include" + break; + fi + done + fi + + dnl overwrite ld flags if we have required special directory with + dnl --with-boost-libdir parameter + if test "$ac_boost_lib_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_lib_path" + fi + + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_REQUIRE([AC_PROG_CXX]) + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + + + + dnl if we found no boost with system layout we search for boost libraries + dnl built and installed without the --layout=system option or for a staged(not installed) version + if test "x$succeeded" != "xyes"; then + _version=0 + if test "$ac_boost_path" != ""; then + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + fi + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE" + done + fi + else + if test "$cross_compiling" != yes; then + for ac_boost_path in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + best_path=$ac_boost_path + fi + done + fi + done + + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" + if test "$ac_boost_lib_path" = ""; then + for libsubdir in $libsubdirs ; do + if ls "$best_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + BOOST_LDFLAGS="-L$best_path/$libsubdir" + fi + fi + + if test "x$BOOST_ROOT" != "x"; then + for libsubdir in $libsubdirs ; do + if ls "$BOOST_ROOT/stage/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/$libsubdir" && test -r "$BOOST_ROOT/stage/$libsubdir"; then + version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'` + stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` + stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` + V_CHECK=`expr $stage_version_shorten \>\= $_version` + if test "$V_CHECK" = "1" -a "$ac_boost_lib_path" = "" ; then + AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) + BOOST_CPPFLAGS="-I$BOOST_ROOT" + BOOST_LDFLAGS="-L$BOOST_ROOT/stage/$libsubdir" + fi + fi + fi + fi + + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + fi + + if test "$succeeded" != "yes" ; then + if test "$_version" = "0" ; then + AC_MSG_NOTICE([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in . See http://randspringer.de/boost for more documentation.]]) + else + AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).]) + fi + # execute ACTION-IF-NOT-FOUND (if present): + ifelse([$3], , :, [$3]) + else + AC_SUBST(BOOST_CPPFLAGS) + AC_SUBST(BOOST_LDFLAGS) + AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available]) + # execute ACTION-IF-FOUND (if present): + ifelse([$2], , :, [$2]) + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" +fi + +]) \ No newline at end of file diff --git a/proxygen/m4/ax_boost_regex.m4 b/proxygen/m4/ax_boost_regex.m4 new file mode 100644 index 0000000000..f3e5cc1856 --- /dev/null +++ b/proxygen/m4/ax_boost_regex.m4 @@ -0,0 +1,111 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_boost_regex.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_REGEX +# +# DESCRIPTION +# +# Test for Regex library from the Boost C++ libraries. The macro requires +# a preceding call to AX_BOOST_BASE. Further documentation is available at +# . +# +# This macro calls: +# +# AC_SUBST(BOOST_REGEX_LIB) +# +# And sets: +# +# HAVE_BOOST_REGEX +# +# LICENSE +# +# Copyright (c) 2008 Thomas Porschberg +# Copyright (c) 2008 Michael Tindal +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 20 + +AC_DEFUN([AX_BOOST_REGEX], +[ + AC_ARG_WITH([boost-regex], + AS_HELP_STRING([--with-boost-regex@<:@=special-lib@:>@], + [use the Regex library from boost - it is possible to specify a certain library for the linker + e.g. --with-boost-regex=boost_regex-gcc-mt-d-1_33_1 ]), + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ax_boost_user_regex_lib="" + else + want_boost="yes" + ax_boost_user_regex_lib="$withval" + fi + ], + [want_boost="yes"] + ) + + if test "x$want_boost" = "xyes"; then + AC_REQUIRE([AC_PROG_CC]) + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_CACHE_CHECK(whether the Boost::Regex library is available, + ax_cv_boost_regex, + [AC_LANG_PUSH([C++]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include + ]], + [[boost::regex r(); return 0;]])], + ax_cv_boost_regex=yes, ax_cv_boost_regex=no) + AC_LANG_POP([C++]) + ]) + if test "x$ax_cv_boost_regex" = "xyes"; then + AC_DEFINE(HAVE_BOOST_REGEX,,[define if the Boost::Regex library is available]) + BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` + if test "x$ax_boost_user_regex_lib" = "x"; then + for libextension in `ls $BOOSTLIBDIR/libboost_regex*.so* $BOOSTLIBDIR/libboost_regex*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_regex.*\)\.so.*$;\1;' -e 's;^lib\(boost_regex.*\)\.a*$;\1;'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_REGEX_LIB="-l$ax_lib"; AC_SUBST(BOOST_REGEX_LIB) link_regex="yes"; break], + [link_regex="no"]) + done + if test "x$link_regex" != "xyes"; then + for libextension in `ls $BOOSTLIBDIR/boost_regex*.{dll,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_regex.*\)\.dll.*$;\1;' -e 's;^\(boost_regex.*\)\.a*$;\1;'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_REGEX_LIB="-l$ax_lib"; AC_SUBST(BOOST_REGEX_LIB) link_regex="yes"; break], + [link_regex="no"]) + done + fi + + else + for ax_lib in $ax_boost_user_regex_lib boost_regex-$ax_boost_user_regex_lib; do + AC_CHECK_LIB($ax_lib, main, + [BOOST_REGEX_LIB="-l$ax_lib"; AC_SUBST(BOOST_REGEX_LIB) link_regex="yes"; break], + [link_regex="no"]) + done + fi + if test "x$ax_lib" = "x"; then + AC_MSG_ERROR(Could not find a version of the Boost::Regex library!) + fi + if test "x$link_regex" != "xyes"; then + AC_MSG_ERROR(Could not link against $ax_lib !) + fi + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + fi +]) \ No newline at end of file