ganeti.http

1 # 2 # 3 4 # Copyright (C) 2007, 2008, 2010, 2012 Google Inc. 5 # All rights reserved. 6 # 7 # Redistribution and use in source and binary forms, with or without 8 # modification, are permitted provided that the following conditions are 9 # met: 10 # 11 # 1. Redistributions of source code must retain the above copyright notice, 12 # this list of conditions and the following disclaimer. 13 # 14 # 2. Redistributions in binary form must reproduce the above copyright 15 # notice, this list of conditions and the following disclaimer in the 16 # documentation and/or other materials provided with the distribution. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 19 # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 23 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 27 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """HTTP module. 31 32 """ 33 34 import logging 35 import mimetools 36 import OpenSSL 37 import select 38 import socket 39 import errno 40 41 from cStringIO import StringIO 42 43 from ganeti import constants 44 from ganeti import utils 45 46 47 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION 48 49 HTTP_OK = 200 50 HTTP_NO_CONTENT = 204 51 HTTP_NOT_MODIFIED = 304 52 53 HTTP_0_9 = "HTTP/0.9" 54 HTTP_1_0 = "HTTP/1.0" 55 HTTP_1_1 = "HTTP/1.1" 56 57 HTTP_GET = "GET" 58 HTTP_HEAD = "HEAD" 59 HTTP_POST = "POST" 60 HTTP_PUT = "PUT" 61 HTTP_DELETE = "DELETE" 62 63 HTTP_ETAG = "ETag" 64 HTTP_HOST = "Host" 65 HTTP_SERVER = "Server" 66 HTTP_DATE = "Date" 67 HTTP_USER_AGENT = "User-Agent" 68 HTTP_CONTENT_TYPE = "Content-Type" 69 HTTP_CONTENT_LENGTH = "Content-Length" 70 HTTP_CONNECTION = "Connection" 71 HTTP_KEEP_ALIVE = "Keep-Alive" 72 HTTP_WWW_AUTHENTICATE = "WWW-Authenticate" 73 HTTP_AUTHORIZATION = "Authorization" 74 HTTP_AUTHENTICATION_INFO = "Authentication-Info" 75 HTTP_ALLOW = "Allow" 76 77 HTTP_APP_OCTET_STREAM = "application/octet-stream" 78 HTTP_APP_JSON = "application/json" 79 80 _SSL_UNEXPECTED_EOF = "Unexpected EOF" 81 82 # Socket operations 83 (SOCKOP_SEND, 84 SOCKOP_RECV, 85 SOCKOP_SHUTDOWN, 86 SOCKOP_HANDSHAKE) = range(4) 87 88 # send/receive quantum 89 SOCK_BUF_SIZE = 32768 90 91

92 -class HttpError(Exception):

93 """Internal exception for HTTP errors. 94 95 This should only be used for internal error reporting. 96 97 """

98 99

100 -class HttpConnectionClosed(Exception):

101 """Internal exception for a closed connection. 102 103 This should only be used for internal error reporting. Only use 104 it if there's no other way to report this condition. 105 106 """

107 108

109 -class HttpSessionHandshakeUnexpectedEOF(HttpError):

110 """Internal exception for errors during SSL handshake. 111 112 This should only be used for internal error reporting. 113 114 """

115 116

117 -class HttpSocketTimeout(Exception):

118 """Internal exception for socket timeouts. 119 120 This should only be used for internal error reporting. 121 122 """

123 124

125 -class HttpException(Exception):

126 code = None 127 message = None 128

129 - def __init__(self, message=None, headers=None):

130 Exception.__init__(self) 131 self.message = message 132 self.headers = headers

133 134

135 -class HttpBadRequest(HttpException):

136 """400 Bad Request 137 138 RFC2616, 10.4.1: The request could not be understood by the server 139 due to malformed syntax. The client SHOULD NOT repeat the request 140 without modifications. 141 142 """ 143 code = 400

144 145

146 -class HttpUnauthorized(HttpException):

147 """401 Unauthorized 148 149 RFC2616, section 10.4.2: The request requires user 150 authentication. The response MUST include a WWW-Authenticate header 151 field (section 14.47) containing a challenge applicable to the 152 requested resource. 153 154 """ 155 code = 401

156 157

158 -class HttpForbidden(HttpException):

159 """403 Forbidden 160 161 RFC2616, 10.4.4: The server understood the request, but is refusing 162 to fulfill it. Authorization will not help and the request SHOULD 163 NOT be repeated. 164 165 """ 166 code = 403

167 168

169 -class HttpNotFound(HttpException):

170 """404 Not Found 171 172 RFC2616, 10.4.5: The server has not found anything matching the 173 Request-URI. No indication is given of whether the condition is 174 temporary or permanent. 175 176 """ 177 code = 404

178 179

180 -class HttpMethodNotAllowed(HttpException):

181 """405 Method Not Allowed 182 183 RFC2616, 10.4.6: The method specified in the Request-Line is not 184 allowed for the resource identified by the Request-URI. The response 185 MUST include an Allow header containing a list of valid methods for 186 the requested resource. 187 188 """ 189 code = 405

190 191

192 -class HttpNotAcceptable(HttpException):

193 """406 Not Acceptable 194 195 RFC2616, 10.4.7: The resource identified by the request is only capable of 196 generating response entities which have content characteristics not 197 acceptable according to the accept headers sent in the request. 198 199 """ 200 code = 406

201 202

203 -class HttpRequestTimeout(HttpException):

204 """408 Request Timeout 205 206 RFC2616, 10.4.9: The client did not produce a request within the 207 time that the server was prepared to wait. The client MAY repeat the 208 request without modifications at any later time. 209 210 """ 211 code = 408

212 213

214 -class HttpConflict(HttpException):

215 """409 Conflict 216 217 RFC2616, 10.4.10: The request could not be completed due to a 218 conflict with the current state of the resource. This code is only 219 allowed in situations where it is expected that the user might be 220 able to resolve the conflict and resubmit the request. 221 222 """ 223 code = 409

224 225

226 -class HttpGone(HttpException):

227 """410 Gone 228 229 RFC2616, 10.4.11: The requested resource is no longer available at 230 the server and no forwarding address is known. This condition is 231 expected to be considered permanent. 232 233 """ 234 code = 410

235 236

237 -class HttpLengthRequired(HttpException):

238 """411 Length Required 239 240 RFC2616, 10.4.12: The server refuses to accept the request without a 241 defined Content-Length. The client MAY repeat the request if it adds 242 a valid Content-Length header field containing the length of the 243 message-body in the request message. 244 245 """ 246 code = 411

247 248

249 -class HttpPreconditionFailed(HttpException):

250 """412 Precondition Failed 251 252 RFC2616, 10.4.13: The precondition given in one or more of the 253 request-header fields evaluated to false when it was tested on the 254 server. 255 256 """ 257 code = 412

258 259

260 -class HttpUnsupportedMediaType(HttpException):

261 """415 Unsupported Media Type 262 263 RFC2616, 10.4.16: The server is refusing to service the request because the 264 entity of the request is in a format not supported by the requested resource 265 for the requested method. 266 267 """ 268 code = 415

269 270

271 -class HttpInternalServerError(HttpException):

272 """500 Internal Server Error 273 274 RFC2616, 10.5.1: The server encountered an unexpected condition 275 which prevented it from fulfilling the request. 276 277 """ 278 code = 500

279 280

281 -class HttpNotImplemented(HttpException):

282 """501 Not Implemented 283 284 RFC2616, 10.5.2: The server does not support the functionality 285 required to fulfill the request. 286 287 """ 288 code = 501

289 290

291 -class HttpBadGateway(HttpException):

292 """502 Bad Gateway 293 294 RFC2616, 10.5.3: The server, while acting as a gateway or proxy, 295 received an invalid response from the upstream server it accessed in 296 attempting to fulfill the request. 297 298 """ 299 code = 502

300 301

302 -class HttpServiceUnavailable(HttpException):

303 """503 Service Unavailable 304 305 RFC2616, 10.5.4: The server is currently unable to handle the 306 request due to a temporary overloading or maintenance of the server. 307 308 """ 309 code = 503

310 311

312 -class HttpGatewayTimeout(HttpException):

313 """504 Gateway Timeout 314 315 RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did 316 not receive a timely response from the upstream server specified by 317 the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server 318 (e.g. DNS) it needed to access in attempting to complete the 319 request. 320 321 """ 322 code = 504

323 324

325 -class HttpVersionNotSupported(HttpException):

326 """505 HTTP Version Not Supported 327 328 RFC2616, 10.5.6: The server does not support, or refuses to support, 329 the HTTP protocol version that was used in the request message. 330 331 """ 332 code = 505

333 334

335 -def ParseHeaders(buf):

336 """Parses HTTP headers. 337 338 @note: This is just a trivial wrapper around C{mimetools.Message} 339 340 """ 341 return mimetools.Message(buf, 0)

342 343

344 -def SocketOperation(sock, op, arg1, timeout):

345 """Wrapper around socket functions. 346 347 This function abstracts error handling for socket operations, especially 348 for the complicated interaction with OpenSSL. 349 350 @type sock: socket 351 @param sock: Socket for the operation 352 @type op: int 353 @param op: Operation to execute (SOCKOP_* constants) 354 @type arg1: any 355 @param arg1: Parameter for function (if needed) 356 @type timeout: None or float 357 @param timeout: Timeout in seconds or None 358 @return: Return value of socket function 359 360 """ 361 # TODO: event_poll/event_check/override 362 if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE): 363 event_poll = select.POLLOUT 364 365 elif op == SOCKOP_RECV: 366 event_poll = select.POLLIN 367 368 elif op == SOCKOP_SHUTDOWN: 369 event_poll = None 370 371 # The timeout is only used when OpenSSL requests polling for a condition. 372 # It is not advisable to have no timeout for shutdown. 373 assert timeout 374 375 else: 376 raise AssertionError("Invalid socket operation") 377 378 # Handshake is only supported by SSL sockets 379 if (op == SOCKOP_HANDSHAKE and 380 not isinstance(sock, OpenSSL.SSL.ConnectionType)): 381 return 382 383 # No override by default 384 event_override = 0 385 386 while True: 387 # Poll only for certain operations and when asked for by an override 388 if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE): 389 if event_override: 390 wait_for_event = event_override 391 else: 392 wait_for_event = event_poll 393 394 event = utils.WaitForFdCondition(sock, wait_for_event, timeout) 395 if event is None: 396 raise HttpSocketTimeout() 397 398 if event & (select.POLLNVAL | select.POLLHUP | select.POLLERR): 399 # Let the socket functions handle these 400 break 401 402 if not event & wait_for_event: 403 continue 404 405 # Reset override 406 event_override = 0 407 408 try: 409 try: 410 if op == SOCKOP_SEND: 411 return sock.send(arg1) 412 413 elif op == SOCKOP_RECV: 414 return sock.recv(arg1) 415 416 elif op == SOCKOP_SHUTDOWN: 417 if isinstance(sock, OpenSSL.SSL.ConnectionType): 418 # PyOpenSSL's shutdown() doesn't take arguments 419 return sock.shutdown() 420 else: 421 return sock.shutdown(arg1) 422 423 elif op == SOCKOP_HANDSHAKE: 424 return sock.do_handshake() 425 426 except OpenSSL.SSL.WantWriteError: 427 # OpenSSL wants to write, poll for POLLOUT 428 event_override = select.POLLOUT 429 continue 430 431 except OpenSSL.SSL.WantReadError: 432 # OpenSSL wants to read, poll for POLLIN 433 event_override = select.POLLIN | select.POLLPRI 434 continue 435 436 except OpenSSL.SSL.WantX509LookupError: 437 continue 438 439 except OpenSSL.SSL.ZeroReturnError, err: 440 # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only 441 # occurs if a closure alert has occurred in the protocol, i.e. the 442 # connection has been closed cleanly. Note that this does not 443 # necessarily mean that the transport layer (e.g. a socket) has been 444 # closed. 445 if op == SOCKOP_SEND: 446 # Can happen during a renegotiation 447 raise HttpConnectionClosed(err.args) 448 elif op == SOCKOP_RECV: 449 return "" 450 451 # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN 452 raise socket.error(err.args) 453 454 except OpenSSL.SSL.SysCallError, err: 455 if op == SOCKOP_SEND: 456 # arg1 is the data when writing 457 if err.args and err.args[0] == -1 and arg1 == "": 458 # errors when writing empty strings are expected 459 # and can be ignored 460 return 0 461 462 if err.args == (-1, _SSL_UNEXPECTED_EOF): 463 if op == SOCKOP_RECV: 464 return "" 465 elif op == SOCKOP_HANDSHAKE: 466 # Can happen if peer disconnects directly after the connection is 467 # opened. 468 raise HttpSessionHandshakeUnexpectedEOF(err.args) 469 470 raise socket.error(err.args) 471 472 except OpenSSL.SSL.Error, err: 473 raise socket.error(err.args) 474 475 except socket.error, err: 476 if err.args and err.args[0] == errno.EAGAIN: 477 # Ignore EAGAIN 478 continue 479 480 raise

481 482

483 -def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force):

484 """Closes the connection. 485 486 @type sock: socket 487 @param sock: Socket to be shut down 488 @type close_timeout: float 489 @param close_timeout: How long to wait for the peer to close 490 the connection 491 @type write_timeout: float 492 @param write_timeout: Write timeout for shutdown 493 @type msgreader: http.HttpMessageReader 494 @param msgreader: Request message reader, used to determine whether 495 peer should close connection 496 @type force: bool 497 @param force: Whether to forcibly close the connection without 498 waiting for peer 499 500 """ 501 #print msgreader.peer_will_close, force 502 if msgreader and msgreader.peer_will_close and not force: 503 # Wait for peer to close 504 try: 505 # Check whether it's actually closed 506 if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout): 507 return 508 except (socket.error, HttpError, HttpSocketTimeout): 509 # Ignore errors at this stage 510 pass 511 512 # Close the connection from our side 513 try: 514 # We don't care about the return value, see NOTES in SSL_shutdown(3). 515 SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR, 516 write_timeout) 517 except HttpSocketTimeout: 518 raise HttpError("Timeout while shutting down connection") 519 except socket.error, err: 520 # Ignore ENOTCONN 521 if not (err.args and err.args[0] == errno.ENOTCONN): 522 raise HttpError("Error while shutting down connection: %s" % err)

523 524

525 -def Handshake(sock, write_timeout):

526 """Shakes peer's hands. 527 528 @type sock: socket 529 @param sock: Socket to be shut down 530 @type write_timeout: float 531 @param write_timeout: Write timeout for handshake 532 533 """ 534 try: 535 return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout) 536 except HttpSocketTimeout: 537 raise HttpError("Timeout during SSL handshake") 538 except socket.error, err: 539 raise HttpError("Error in SSL handshake: %s" % err)

540 541

542 -class HttpSslParams(object):

543 """Data class for SSL key and certificate. 544 545 """

546 - def __init__(self, ssl_key_path, ssl_cert_path):

547 """Initializes this class. 548 549 @type ssl_key_path: string 550 @param ssl_key_path: Path to file containing SSL key in PEM format 551 @type ssl_cert_path: string 552 @param ssl_cert_path: Path to file containing SSL certificate 553 in PEM format 554 555 """ 556 self.ssl_key_pem = utils.ReadFile(ssl_key_path) 557 self.ssl_cert_pem = utils.ReadFile(ssl_cert_path) 558 self.ssl_cert_path = ssl_cert_path

559

560 - def GetKey(self):

561 return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, 562 self.ssl_key_pem)

563

564 - def GetCertificate(self):

565 return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, 566 self.ssl_cert_pem)

567 568

569 -class HttpBase(object):

570 """Base class for HTTP server and client. 571 572 """

573 - def __init__(self):

574 self.using_ssl = None 575 self._ssl_params = None 576 self._ssl_key = None 577 self._ssl_cert = None

578

579 - def _CreateSocket(self, ssl_params, ssl_verify_peer, family):

580 """Creates a TCP socket and initializes SSL if needed. 581 582 @type ssl_params: HttpSslParams 583 @param ssl_params: SSL key and certificate 584 @type ssl_verify_peer: bool 585 @param ssl_verify_peer: Whether to require client certificate 586 and compare it with our certificate 587 @type family: int 588 @param family: socket.AF_INET | socket.AF_INET6 589 590 """ 591 assert family in (socket.AF_INET, socket.AF_INET6) 592 593 self._ssl_params = ssl_params 594 sock = socket.socket(family, socket.SOCK_STREAM) 595 596 # Should we enable SSL? 597 self.using_ssl = ssl_params is not None 598 599 if not self.using_ssl: 600 return sock 601 602 self._ssl_key = ssl_params.GetKey() 603 self._ssl_cert = ssl_params.GetCertificate() 604 605 ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) 606 ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2) 607 608 ciphers = self.GetSslCiphers() 609 logging.debug("Setting SSL cipher string %s", ciphers) 610 ctx.set_cipher_list(ciphers) 611 612 ctx.use_privatekey(self._ssl_key) 613 ctx.use_certificate(self._ssl_cert) 614 ctx.check_privatekey() 615 616 if ssl_verify_peer: 617 ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | 618 OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, 619 self._SSLVerifyCallback) 620 621 # Also add our certificate as a trusted CA to be sent to the client. 622 # This is required at least for GnuTLS clients to work. 623 try: 624 # This will fail for PyOpenssl versions before 0.10 625 ctx.add_client_ca(self._ssl_cert) 626 except AttributeError: 627 # Fall back to letting OpenSSL read the certificate file directly. 628 ctx.load_client_ca(ssl_params.ssl_cert_path) 629 630 return OpenSSL.SSL.Connection(ctx, sock)

631

632 - def GetSslCiphers(self): # pylint: disable=R0201

633 """Returns the ciphers string for SSL. 634 635 """ 636 return constants.OPENSSL_CIPHERS

637

638 - def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok):

639 """Verify the certificate provided by the peer 640 641 We only compare fingerprints. The client must use the same certificate as 642 we do on our side. 643 644 """ 645 # some parameters are unused, but this is the API 646 # pylint: disable=W0613 647 assert self._ssl_params, "SSL not initialized" 648 649 return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and 650 self._ssl_cert.digest("md5") == cert.digest("md5"))

651 652

653 -class HttpMessage(object):

654 """Data structure for HTTP message. 655 656 """

657 - def __init__(self):

658 self.start_line = None 659 self.headers = None 660 self.body = None

661 662

663 -class HttpClientToServerStartLine(object):

664 """Data structure for HTTP request start line. 665 666 """

667 - def __init__(self, method, path, version):

668 self.method = method 669 self.path = path 670 self.version = version

671

672 - def __str__(self):

673 return "%s %s %s" % (self.method, self.path, self.version)

674 675

676 -class HttpServerToClientStartLine(object):

677 """Data structure for HTTP response start line. 678 679 """

680 - def __init__(self, version, code, reason):

681 self.version = version 682 self.code = code 683 self.reason = reason

684

685 - def __str__(self):

686 return "%s %s %s" % (self.version, self.code, self.reason)

687 688

689 -class HttpMessageWriter(object):

690 """Writes an HTTP message to a socket. 691 692 """

693 - def __init__(self, sock, msg, write_timeout):

694 """Initializes this class and writes an HTTP message to a socket. 695 696 @type sock: socket 697 @param sock: Socket to be written to 698 @type msg: http.HttpMessage 699 @param msg: HTTP message to be written 700 @type write_timeout: float 701 @param write_timeout: Write timeout for socket 702 703 """ 704 self._msg = msg 705 706 self._PrepareMessage() 707 708 buf = self._FormatMessage() 709 710 pos = 0 711 end = len(buf) 712 while pos < end: 713 # Send only SOCK_BUF_SIZE bytes at a time 714 data = buf[pos:(pos + SOCK_BUF_SIZE)] 715 716 sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout) 717 718 # Remove sent bytes 719 pos += sent 720 721 assert pos == end, "Message wasn't sent completely"

722

723 - def _PrepareMessage(self):

724 """Prepares the HTTP message by setting mandatory headers. 725 726 """ 727 # RFC2616, section 4.3: "The presence of a message-body in a request is 728 # signaled by the inclusion of a Content-Length or Transfer-Encoding header 729 # field in the request's message-headers." 730 if self._msg.body: 731 self._msg.headers[HTTP_CONTENT_LENGTH] = len(self._msg.body)

732

733 - def _FormatMessage(self):

734 """Serializes the HTTP message into a string. 735 736 """ 737 buf = StringIO() 738 739 # Add start line 740 buf.write(str(self._msg.start_line)) 741 buf.write("\r\n") 742 743 # Add headers 744 if self._msg.start_line.version != HTTP_0_9: 745 for name, value in self._msg.headers.iteritems(): 746 buf.write("%s: %s\r\n" % (name, value)) 747 748 buf.write("\r\n") 749 750 # Add message body if needed 751 if self.HasMessageBody(): 752 buf.write(self._msg.body) 753 754 elif self._msg.body: 755 logging.warning("Ignoring message body") 756 757 return buf.getvalue()

758

759 - def HasMessageBody(self):

760 """Checks whether the HTTP message contains a body. 761 762 Can be overridden by subclasses. 763 764 """ 765 return bool(self._msg.body)

766 767

768 -class HttpMessageReader(object):

769 """Reads HTTP message from socket. 770 771 """ 772 # Length limits 773 START_LINE_LENGTH_MAX = None 774 HEADER_LENGTH_MAX = None 775 776 # Parser state machine 777 PS_START_LINE = "start-line" 778 PS_HEADERS = "headers" 779 PS_BODY = "entity-body" 780 PS_COMPLETE = "complete" 781

782 - def __init__(self, sock, msg, read_timeout):

783 """Reads an HTTP message from a socket. 784 785 @type sock: socket 786 @param sock: Socket to be read from 787 @type msg: http.HttpMessage 788 @param msg: Object for the read message 789 @type read_timeout: float 790 @param read_timeout: Read timeout for socket 791 792 """ 793 self.sock = sock 794 self.msg = msg 795 796 self.start_line_buffer = None 797 self.header_buffer = StringIO() 798 self.body_buffer = StringIO() 799 self.parser_status = self.PS_START_LINE 800 self.content_length = None 801 self.peer_will_close = None 802 803 buf = "" 804 eof = False 805 while self.parser_status != self.PS_COMPLETE: 806 # TODO: Don't read more than necessary (Content-Length), otherwise 807 # data might be lost and/or an error could occur 808 data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout) 809 810 if data: 811 buf += data 812 else: 813 eof = True 814 815 # Do some parsing and error checking while more data arrives 816 buf = self._ContinueParsing(buf, eof) 817 818 # Must be done only after the buffer has been evaluated 819 # TODO: Content-Length < len(data read) and connection closed 820 if (eof and 821 self.parser_status in (self.PS_START_LINE, 822 self.PS_HEADERS)): 823 raise HttpError("Connection closed prematurely") 824 825 # Parse rest 826 buf = self._ContinueParsing(buf, True) 827 828 assert self.parser_status == self.PS_COMPLETE 829 assert not buf, "Parser didn't read full response" 830 831 # Body is complete 832 msg.body = self.body_buffer.getvalue()

833

834 - def _ContinueParsing(self, buf, eof):

835 """Main function for HTTP message state machine. 836 837 @type buf: string 838 @param buf: Receive buffer 839 @type eof: bool 840 @param eof: Whether we've reached EOF on the socket 841 @rtype: string 842 @return: Updated receive buffer 843 844 """ 845 # TODO: Use offset instead of slicing when possible 846 if self.parser_status == self.PS_START_LINE: 847 # Expect start line 848 while True: 849 idx = buf.find("\r\n") 850 851 # RFC2616, section 4.1: "In the interest of robustness, servers SHOULD 852 # ignore any empty line(s) received where a Request-Line is expected. 853 # In other words, if the server is reading the protocol stream at the 854 # beginning of a message and receives a CRLF first, it should ignore 855 # the CRLF." 856 if idx == 0: 857 # TODO: Limit number of CRLFs/empty lines for safety? 858 buf = buf[2:] 859 continue 860 861 if idx > 0: 862 self.start_line_buffer = buf[:idx] 863 864 self._CheckStartLineLength(len(self.start_line_buffer)) 865 866 # Remove status line, including CRLF 867 buf = buf[idx + 2:] 868 869 self.msg.start_line = self.ParseStartLine(self.start_line_buffer) 870 871 self.parser_status = self.PS_HEADERS 872 else: 873 # Check whether incoming data is getting too large, otherwise we just 874 # fill our read buffer. 875 self._CheckStartLineLength(len(buf)) 876 877 break 878 879 # TODO: Handle messages without headers 880 if self.parser_status == self.PS_HEADERS: 881 # Wait for header end 882 idx = buf.find("\r\n\r\n") 883 if idx >= 0: 884 self.header_buffer.write(buf[:idx + 2]) 885 886 self._CheckHeaderLength(self.header_buffer.tell()) 887 888 # Remove headers, including CRLF 889 buf = buf[idx + 4:] 890 891 self._ParseHeaders() 892 893 self.parser_status = self.PS_BODY 894 else: 895 # Check whether incoming data is getting too large, otherwise we just 896 # fill our read buffer. 897 self._CheckHeaderLength(len(buf)) 898 899 if self.parser_status == self.PS_BODY: 900 # TODO: Implement max size for body_buffer 901 self.body_buffer.write(buf) 902 buf = "" 903 904 # Check whether we've read everything 905 # 906 # RFC2616, section 4.4: "When a message-body is included with a message, 907 # the transfer-length of that body is determined by one of the following 908 # [...] 5. By the server closing the connection. (Closing the connection 909 # cannot be used to indicate the end of a request body, since that would 910 # leave no possibility for the server to send back a response.)" 911 # 912 # TODO: Error when buffer length > Content-Length header 913 if (eof or 914 self.content_length is None or 915 (self.content_length is not None and 916 self.body_buffer.tell() >= self.content_length)): 917 self.parser_status = self.PS_COMPLETE 918 919 return buf

920

921 - def _CheckStartLineLength(self, length):

922 """Limits the start line buffer size. 923 924 @type length: int 925 @param length: Buffer size 926 927 """ 928 if (self.START_LINE_LENGTH_MAX is not None and 929 length > self.START_LINE_LENGTH_MAX): 930 raise HttpError("Start line longer than %d chars" % 931 self.START_LINE_LENGTH_MAX)

932

933 - def _CheckHeaderLength(self, length):

934 """Limits the header buffer size. 935 936 @type length: int 937 @param length: Buffer size 938 939 """ 940 if (self.HEADER_LENGTH_MAX is not None and 941 length > self.HEADER_LENGTH_MAX): 942 raise HttpError("Headers longer than %d chars" % self.HEADER_LENGTH_MAX)

943

944 - def ParseStartLine(self, start_line):

945 """Parses the start line of a message. 946 947 Must be overridden by subclass. 948 949 @type start_line: string 950 @param start_line: Start line string 951 952 """ 953 raise NotImplementedError()

954

955 - def _WillPeerCloseConnection(self):

956 """Evaluate whether peer will close the connection. 957 958 @rtype: bool 959 @return: Whether peer will close the connection 960 961 """ 962 # RFC2616, section 14.10: "HTTP/1.1 defines the "close" connection option 963 # for the sender to signal that the connection will be closed after 964 # completion of the response. For example, 965 # 966 # Connection: close 967 # 968 # in either the request or the response header fields indicates that the 969 # connection SHOULD NOT be considered `persistent' (section 8.1) after the 970 # current request/response is complete." 971 972 hdr_connection = self.msg.headers.get(HTTP_CONNECTION, None) 973 if hdr_connection: 974 hdr_connection = hdr_connection.lower() 975 976 # An HTTP/1.1 server is assumed to stay open unless explicitly closed. 977 if self.msg.start_line.version == HTTP_1_1: 978 return (hdr_connection and "close" in hdr_connection) 979 980 # Some HTTP/1.0 implementations have support for persistent connections, 981 # using rules different than HTTP/1.1. 982 983 # For older HTTP, Keep-Alive indicates persistent connection. 984 if self.msg.headers.get(HTTP_KEEP_ALIVE): 985 return False 986 987 # At least Akamai returns a "Connection: Keep-Alive" header, which was 988 # supposed to be sent by the client. 989 if hdr_connection and "keep-alive" in hdr_connection: 990 return False 991 992 return True

993

994 - def _ParseHeaders(self):

995 """Parses the headers. 996 997 This function also adjusts internal variables based on header values. 998 999 RFC2616, section 4.3: The presence of a message-body in a request is 1000 signaled by the inclusion of a Content-Length or Transfer-Encoding header 1001 field in the request's message-headers. 1002 1003 """ 1004 # Parse headers 1005 self.header_buffer.seek(0, 0) 1006 self.msg.headers = ParseHeaders(self.header_buffer) 1007 1008 self.peer_will_close = self._WillPeerCloseConnection() 1009 1010 # Do we have a Content-Length header? 1011 hdr_content_length = self.msg.headers.get(HTTP_CONTENT_LENGTH, None) 1012 if hdr_content_length: 1013 try: 1014 self.content_length = int(hdr_content_length) 1015 except (TypeError, ValueError): 1016 self.content_length = None 1017 if self.content_length is not None and self.content_length < 0: 1018 self.content_length = None 1019 1020 # if the connection remains open and a content-length was not provided, 1021 # then assume that the connection WILL close. 1022 if self.content_length is None: 1023 self.peer_will_close = True

1024

Source Code for Package ganeti.http