ganeti.http

1 # 2 # 3 4 # Copyright (C) 2007, 2008, 2010 Google Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 # 11 # This program is distributed in the hope that it will be useful, but 12 # WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 # General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19 # 02110-1301, USA. 20 21 """HTTP module. 22 23 """ 24 25 import logging 26 import mimetools 27 import OpenSSL 28 import select 29 import socket 30 import errno 31 32 from cStringIO import StringIO 33 34 from ganeti import constants 35 from ganeti import utils 36 37 38 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION 39 40 HTTP_OK = 200 41 HTTP_NO_CONTENT = 204 42 HTTP_NOT_MODIFIED = 304 43 44 HTTP_0_9 = "HTTP/0.9" 45 HTTP_1_0 = "HTTP/1.0" 46 HTTP_1_1 = "HTTP/1.1" 47 48 HTTP_GET = "GET" 49 HTTP_HEAD = "HEAD" 50 HTTP_POST = "POST" 51 HTTP_PUT = "PUT" 52 HTTP_DELETE = "DELETE" 53 54 HTTP_ETAG = "ETag" 55 HTTP_HOST = "Host" 56 HTTP_SERVER = "Server" 57 HTTP_DATE = "Date" 58 HTTP_USER_AGENT = "User-Agent" 59 HTTP_CONTENT_TYPE = "Content-Type" 60 HTTP_CONTENT_LENGTH = "Content-Length" 61 HTTP_CONNECTION = "Connection" 62 HTTP_KEEP_ALIVE = "Keep-Alive" 63 HTTP_WWW_AUTHENTICATE = "WWW-Authenticate" 64 HTTP_AUTHORIZATION = "Authorization" 65 HTTP_AUTHENTICATION_INFO = "Authentication-Info" 66 HTTP_ALLOW = "Allow" 67 68 HTTP_APP_OCTET_STREAM = "application/octet-stream" 69 HTTP_APP_JSON = "application/json" 70 71 _SSL_UNEXPECTED_EOF = "Unexpected EOF" 72 73 # Socket operations 74 (SOCKOP_SEND, 75 SOCKOP_RECV, 76 SOCKOP_SHUTDOWN, 77 SOCKOP_HANDSHAKE) = range(4) 78 79 # send/receive quantum 80 SOCK_BUF_SIZE = 32768 81 82

83 -class HttpError(Exception):

84 """Internal exception for HTTP errors. 85 86 This should only be used for internal error reporting. 87 88 """

89 90

91 -class HttpConnectionClosed(Exception):

92 """Internal exception for a closed connection. 93 94 This should only be used for internal error reporting. Only use 95 it if there's no other way to report this condition. 96 97 """

98 99

100 -class HttpSessionHandshakeUnexpectedEOF(HttpError):

101 """Internal exception for errors during SSL handshake. 102 103 This should only be used for internal error reporting. 104 105 """

106 107

108 -class HttpSocketTimeout(Exception):

109 """Internal exception for socket timeouts. 110 111 This should only be used for internal error reporting. 112 113 """

114 115

116 -class HttpException(Exception):

117 code = None 118 message = None 119

120 - def __init__(self, message=None, headers=None):

121 Exception.__init__(self) 122 self.message = message 123 self.headers = headers

124 125

126 -class HttpBadRequest(HttpException):

127 """400 Bad Request 128 129 RFC2616, 10.4.1: The request could not be understood by the server 130 due to malformed syntax. The client SHOULD NOT repeat the request 131 without modifications. 132 133 """ 134 code = 400

135 136

137 -class HttpUnauthorized(HttpException):

138 """401 Unauthorized 139 140 RFC2616, section 10.4.2: The request requires user 141 authentication. The response MUST include a WWW-Authenticate header 142 field (section 14.47) containing a challenge applicable to the 143 requested resource. 144 145 """ 146 code = 401

147 148

149 -class HttpForbidden(HttpException):

150 """403 Forbidden 151 152 RFC2616, 10.4.4: The server understood the request, but is refusing 153 to fulfill it. Authorization will not help and the request SHOULD 154 NOT be repeated. 155 156 """ 157 code = 403

158 159

160 -class HttpNotFound(HttpException):

161 """404 Not Found 162 163 RFC2616, 10.4.5: The server has not found anything matching the 164 Request-URI. No indication is given of whether the condition is 165 temporary or permanent. 166 167 """ 168 code = 404

169 170

171 -class HttpMethodNotAllowed(HttpException):

172 """405 Method Not Allowed 173 174 RFC2616, 10.4.6: The method specified in the Request-Line is not 175 allowed for the resource identified by the Request-URI. The response 176 MUST include an Allow header containing a list of valid methods for 177 the requested resource. 178 179 """ 180 code = 405

181 182

183 -class HttpNotAcceptable(HttpException):

184 """406 Not Acceptable 185 186 RFC2616, 10.4.7: The resource identified by the request is only capable of 187 generating response entities which have content characteristics not 188 acceptable according to the accept headers sent in the request. 189 190 """ 191 code = 406

192 193

194 -class HttpRequestTimeout(HttpException):

195 """408 Request Timeout 196 197 RFC2616, 10.4.9: The client did not produce a request within the 198 time that the server was prepared to wait. The client MAY repeat the 199 request without modifications at any later time. 200 201 """ 202 code = 408

203 204

205 -class HttpConflict(HttpException):

206 """409 Conflict 207 208 RFC2616, 10.4.10: The request could not be completed due to a 209 conflict with the current state of the resource. This code is only 210 allowed in situations where it is expected that the user might be 211 able to resolve the conflict and resubmit the request. 212 213 """ 214 code = 409

215 216

217 -class HttpGone(HttpException):

218 """410 Gone 219 220 RFC2616, 10.4.11: The requested resource is no longer available at 221 the server and no forwarding address is known. This condition is 222 expected to be considered permanent. 223 224 """ 225 code = 410

226 227

228 -class HttpLengthRequired(HttpException):

229 """411 Length Required 230 231 RFC2616, 10.4.12: The server refuses to accept the request without a 232 defined Content-Length. The client MAY repeat the request if it adds 233 a valid Content-Length header field containing the length of the 234 message-body in the request message. 235 236 """ 237 code = 411

238 239

240 -class HttpPreconditionFailed(HttpException):

241 """412 Precondition Failed 242 243 RFC2616, 10.4.13: The precondition given in one or more of the 244 request-header fields evaluated to false when it was tested on the 245 server. 246 247 """ 248 code = 412

249 250

251 -class HttpUnsupportedMediaType(HttpException):

252 """415 Unsupported Media Type 253 254 RFC2616, 10.4.16: The server is refusing to service the request because the 255 entity of the request is in a format not supported by the requested resource 256 for the requested method. 257 258 """ 259 code = 415

260 261

262 -class HttpInternalServerError(HttpException):

263 """500 Internal Server Error 264 265 RFC2616, 10.5.1: The server encountered an unexpected condition 266 which prevented it from fulfilling the request. 267 268 """ 269 code = 500

270 271

272 -class HttpNotImplemented(HttpException):

273 """501 Not Implemented 274 275 RFC2616, 10.5.2: The server does not support the functionality 276 required to fulfill the request. 277 278 """ 279 code = 501

280 281

282 -class HttpBadGateway(HttpException):

283 """502 Bad Gateway 284 285 RFC2616, 10.5.3: The server, while acting as a gateway or proxy, 286 received an invalid response from the upstream server it accessed in 287 attempting to fulfill the request. 288 289 """ 290 code = 502

291 292

293 -class HttpServiceUnavailable(HttpException):

294 """503 Service Unavailable 295 296 RFC2616, 10.5.4: The server is currently unable to handle the 297 request due to a temporary overloading or maintenance of the server. 298 299 """ 300 code = 503

301 302

303 -class HttpGatewayTimeout(HttpException):

304 """504 Gateway Timeout 305 306 RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did 307 not receive a timely response from the upstream server specified by 308 the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server 309 (e.g. DNS) it needed to access in attempting to complete the 310 request. 311 312 """ 313 code = 504

314 315

316 -class HttpVersionNotSupported(HttpException):

317 """505 HTTP Version Not Supported 318 319 RFC2616, 10.5.6: The server does not support, or refuses to support, 320 the HTTP protocol version that was used in the request message. 321 322 """ 323 code = 505

324 325

326 -def SocketOperation(sock, op, arg1, timeout):

327 """Wrapper around socket functions. 328 329 This function abstracts error handling for socket operations, especially 330 for the complicated interaction with OpenSSL. 331 332 @type sock: socket 333 @param sock: Socket for the operation 334 @type op: int 335 @param op: Operation to execute (SOCKOP_* constants) 336 @type arg1: any 337 @param arg1: Parameter for function (if needed) 338 @type timeout: None or float 339 @param timeout: Timeout in seconds or None 340 @return: Return value of socket function 341 342 """ 343 # TODO: event_poll/event_check/override 344 if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE): 345 event_poll = select.POLLOUT 346 347 elif op == SOCKOP_RECV: 348 event_poll = select.POLLIN 349 350 elif op == SOCKOP_SHUTDOWN: 351 event_poll = None 352 353 # The timeout is only used when OpenSSL requests polling for a condition. 354 # It is not advisable to have no timeout for shutdown. 355 assert timeout 356 357 else: 358 raise AssertionError("Invalid socket operation") 359 360 # Handshake is only supported by SSL sockets 361 if (op == SOCKOP_HANDSHAKE and 362 not isinstance(sock, OpenSSL.SSL.ConnectionType)): 363 return 364 365 # No override by default 366 event_override = 0 367 368 while True: 369 # Poll only for certain operations and when asked for by an override 370 if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE): 371 if event_override: 372 wait_for_event = event_override 373 else: 374 wait_for_event = event_poll 375 376 event = utils.WaitForFdCondition(sock, wait_for_event, timeout) 377 if event is None: 378 raise HttpSocketTimeout() 379 380 if event & (select.POLLNVAL | select.POLLHUP | select.POLLERR): 381 # Let the socket functions handle these 382 break 383 384 if not event & wait_for_event: 385 continue 386 387 # Reset override 388 event_override = 0 389 390 try: 391 try: 392 if op == SOCKOP_SEND: 393 return sock.send(arg1) 394 395 elif op == SOCKOP_RECV: 396 return sock.recv(arg1) 397 398 elif op == SOCKOP_SHUTDOWN: 399 if isinstance(sock, OpenSSL.SSL.ConnectionType): 400 # PyOpenSSL's shutdown() doesn't take arguments 401 return sock.shutdown() 402 else: 403 return sock.shutdown(arg1) 404 405 elif op == SOCKOP_HANDSHAKE: 406 return sock.do_handshake() 407 408 except OpenSSL.SSL.WantWriteError: 409 # OpenSSL wants to write, poll for POLLOUT 410 event_override = select.POLLOUT 411 continue 412 413 except OpenSSL.SSL.WantReadError: 414 # OpenSSL wants to read, poll for POLLIN 415 event_override = select.POLLIN | select.POLLPRI 416 continue 417 418 except OpenSSL.SSL.WantX509LookupError: 419 continue 420 421 except OpenSSL.SSL.ZeroReturnError, err: 422 # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only 423 # occurs if a closure alert has occurred in the protocol, i.e. the 424 # connection has been closed cleanly. Note that this does not 425 # necessarily mean that the transport layer (e.g. a socket) has been 426 # closed. 427 if op == SOCKOP_SEND: 428 # Can happen during a renegotiation 429 raise HttpConnectionClosed(err.args) 430 elif op == SOCKOP_RECV: 431 return "" 432 433 # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN 434 raise socket.error(err.args) 435 436 except OpenSSL.SSL.SysCallError, err: 437 if op == SOCKOP_SEND: 438 # arg1 is the data when writing 439 if err.args and err.args[0] == -1 and arg1 == "": 440 # errors when writing empty strings are expected 441 # and can be ignored 442 return 0 443 444 if err.args == (-1, _SSL_UNEXPECTED_EOF): 445 if op == SOCKOP_RECV: 446 return "" 447 elif op == SOCKOP_HANDSHAKE: 448 # Can happen if peer disconnects directly after the connection is 449 # opened. 450 raise HttpSessionHandshakeUnexpectedEOF(err.args) 451 452 raise socket.error(err.args) 453 454 except OpenSSL.SSL.Error, err: 455 raise socket.error(err.args) 456 457 except socket.error, err: 458 if err.args and err.args[0] == errno.EAGAIN: 459 # Ignore EAGAIN 460 continue 461 462 raise

463 464

465 -def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force):

466 """Closes the connection. 467 468 @type sock: socket 469 @param sock: Socket to be shut down 470 @type close_timeout: float 471 @param close_timeout: How long to wait for the peer to close 472 the connection 473 @type write_timeout: float 474 @param write_timeout: Write timeout for shutdown 475 @type msgreader: http.HttpMessageReader 476 @param msgreader: Request message reader, used to determine whether 477 peer should close connection 478 @type force: bool 479 @param force: Whether to forcibly close the connection without 480 waiting for peer 481 482 """ 483 #print msgreader.peer_will_close, force 484 if msgreader and msgreader.peer_will_close and not force: 485 # Wait for peer to close 486 try: 487 # Check whether it's actually closed 488 if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout): 489 return 490 except (socket.error, HttpError, HttpSocketTimeout): 491 # Ignore errors at this stage 492 pass 493 494 # Close the connection from our side 495 try: 496 # We don't care about the return value, see NOTES in SSL_shutdown(3). 497 SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR, 498 write_timeout) 499 except HttpSocketTimeout: 500 raise HttpError("Timeout while shutting down connection") 501 except socket.error, err: 502 # Ignore ENOTCONN 503 if not (err.args and err.args[0] == errno.ENOTCONN): 504 raise HttpError("Error while shutting down connection: %s" % err)

505 506

507 -def Handshake(sock, write_timeout):

508 """Shakes peer's hands. 509 510 @type sock: socket 511 @param sock: Socket to be shut down 512 @type write_timeout: float 513 @param write_timeout: Write timeout for handshake 514 515 """ 516 try: 517 return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout) 518 except HttpSocketTimeout: 519 raise HttpError("Timeout during SSL handshake") 520 except socket.error, err: 521 raise HttpError("Error in SSL handshake: %s" % err)

522 523

524 -def InitSsl():

525 """Initializes the SSL infrastructure. 526 527 This function is idempotent. 528 529 """ 530 if not OpenSSL.rand.status(): 531 raise EnvironmentError("OpenSSL could not collect enough entropy" 532 " for the PRNG")

533 534 # TODO: Maybe add some additional seeding for OpenSSL's PRNG 535 536

537 -class HttpSslParams(object):

538 """Data class for SSL key and certificate. 539 540 """

541 - def __init__(self, ssl_key_path, ssl_cert_path):

542 """Initializes this class. 543 544 @type ssl_key_path: string 545 @param ssl_key_path: Path to file containing SSL key in PEM format 546 @type ssl_cert_path: string 547 @param ssl_cert_path: Path to file containing SSL certificate 548 in PEM format 549 550 """ 551 self.ssl_key_pem = utils.ReadFile(ssl_key_path) 552 self.ssl_cert_pem = utils.ReadFile(ssl_cert_path) 553 self.ssl_cert_path = ssl_cert_path

554

555 - def GetKey(self):

556 return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, 557 self.ssl_key_pem)

558

559 - def GetCertificate(self):

560 return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, 561 self.ssl_cert_pem)

562 563

564 -class HttpBase(object):

565 """Base class for HTTP server and client. 566 567 """

568 - def __init__(self):

569 self.using_ssl = None 570 self._ssl_params = None 571 self._ssl_key = None 572 self._ssl_cert = None

573

574 - def _CreateSocket(self, ssl_params, ssl_verify_peer, family):

575 """Creates a TCP socket and initializes SSL if needed. 576 577 @type ssl_params: HttpSslParams 578 @param ssl_params: SSL key and certificate 579 @type ssl_verify_peer: bool 580 @param ssl_verify_peer: Whether to require client certificate 581 and compare it with our certificate 582 @type family: int 583 @param family: socket.AF_INET | socket.AF_INET6 584 585 """ 586 assert family in (socket.AF_INET, socket.AF_INET6) 587 588 self._ssl_params = ssl_params 589 sock = socket.socket(family, socket.SOCK_STREAM) 590 591 # Should we enable SSL? 592 self.using_ssl = ssl_params is not None 593 594 if not self.using_ssl: 595 return sock 596 597 self._ssl_key = ssl_params.GetKey() 598 self._ssl_cert = ssl_params.GetCertificate() 599 600 ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) 601 ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2) 602 603 ciphers = self.GetSslCiphers() 604 logging.debug("Setting SSL cipher string %s", ciphers) 605 ctx.set_cipher_list(ciphers) 606 607 ctx.use_privatekey(self._ssl_key) 608 ctx.use_certificate(self._ssl_cert) 609 ctx.check_privatekey() 610 611 if ssl_verify_peer: 612 ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | 613 OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, 614 self._SSLVerifyCallback) 615 616 # Also add our certificate as a trusted CA to be sent to the client. 617 # This is required at least for GnuTLS clients to work. 618 try: 619 # This will fail for PyOpenssl versions before 0.10 620 ctx.add_client_ca(self._ssl_cert) 621 except AttributeError: 622 # Fall back to letting OpenSSL read the certificate file directly. 623 ctx.load_client_ca(ssl_params.ssl_cert_path) 624 625 return OpenSSL.SSL.Connection(ctx, sock)

626

627 - def GetSslCiphers(self): # pylint: disable-msg=R0201

628 """Returns the ciphers string for SSL. 629 630 """ 631 return constants.OPENSSL_CIPHERS

632

633 - def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok):

634 """Verify the certificate provided by the peer 635 636 We only compare fingerprints. The client must use the same certificate as 637 we do on our side. 638 639 """ 640 # some parameters are unused, but this is the API 641 # pylint: disable-msg=W0613 642 assert self._ssl_params, "SSL not initialized" 643 644 return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and 645 self._ssl_cert.digest("md5") == cert.digest("md5"))

646 647

648 -class HttpMessage(object):

649 """Data structure for HTTP message. 650 651 """

652 - def __init__(self):

653 self.start_line = None 654 self.headers = None 655 self.body = None

656 657

658 -class HttpClientToServerStartLine(object):

659 """Data structure for HTTP request start line. 660 661 """

662 - def __init__(self, method, path, version):

663 self.method = method 664 self.path = path 665 self.version = version

666

667 - def __str__(self):

668 return "%s %s %s" % (self.method, self.path, self.version)

669 670

671 -class HttpServerToClientStartLine(object):

672 """Data structure for HTTP response start line. 673 674 """

675 - def __init__(self, version, code, reason):

676 self.version = version 677 self.code = code 678 self.reason = reason

679

680 - def __str__(self):

681 return "%s %s %s" % (self.version, self.code, self.reason)

682 683

684 -class HttpMessageWriter(object):

685 """Writes an HTTP message to a socket. 686 687 """

688 - def __init__(self, sock, msg, write_timeout):

689 """Initializes this class and writes an HTTP message to a socket. 690 691 @type sock: socket 692 @param sock: Socket to be written to 693 @type msg: http.HttpMessage 694 @param msg: HTTP message to be written 695 @type write_timeout: float 696 @param write_timeout: Write timeout for socket 697 698 """ 699 self._msg = msg 700 701 self._PrepareMessage() 702 703 buf = self._FormatMessage() 704 705 pos = 0 706 end = len(buf) 707 while pos < end: 708 # Send only SOCK_BUF_SIZE bytes at a time 709 data = buf[pos:(pos + SOCK_BUF_SIZE)] 710 711 sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout) 712 713 # Remove sent bytes 714 pos += sent 715 716 assert pos == end, "Message wasn't sent completely"

717

718 - def _PrepareMessage(self):

719 """Prepares the HTTP message by setting mandatory headers. 720 721 """ 722 # RFC2616, section 4.3: "The presence of a message-body in a request is 723 # signaled by the inclusion of a Content-Length or Transfer-Encoding header 724 # field in the request's message-headers." 725 if self._msg.body: 726 self._msg.headers[HTTP_CONTENT_LENGTH] = len(self._msg.body)

727

728 - def _FormatMessage(self):

729 """Serializes the HTTP message into a string. 730 731 """ 732 buf = StringIO() 733 734 # Add start line 735 buf.write(str(self._msg.start_line)) 736 buf.write("\r\n") 737 738 # Add headers 739 if self._msg.start_line.version != HTTP_0_9: 740 for name, value in self._msg.headers.iteritems(): 741 buf.write("%s: %s\r\n" % (name, value)) 742 743 buf.write("\r\n") 744 745 # Add message body if needed 746 if self.HasMessageBody(): 747 buf.write(self._msg.body) 748 749 elif self._msg.body: 750 logging.warning("Ignoring message body") 751 752 return buf.getvalue()

753

754 - def HasMessageBody(self):

755 """Checks whether the HTTP message contains a body. 756 757 Can be overridden by subclasses. 758 759 """ 760 return bool(self._msg.body)

761 762

763 -class HttpMessageReader(object):

764 """Reads HTTP message from socket. 765 766 """ 767 # Length limits 768 START_LINE_LENGTH_MAX = None 769 HEADER_LENGTH_MAX = None 770 771 # Parser state machine 772 PS_START_LINE = "start-line" 773 PS_HEADERS = "headers" 774 PS_BODY = "entity-body" 775 PS_COMPLETE = "complete" 776

777 - def __init__(self, sock, msg, read_timeout):

778 """Reads an HTTP message from a socket. 779 780 @type sock: socket 781 @param sock: Socket to be read from 782 @type msg: http.HttpMessage 783 @param msg: Object for the read message 784 @type read_timeout: float 785 @param read_timeout: Read timeout for socket 786 787 """ 788 self.sock = sock 789 self.msg = msg 790 791 self.start_line_buffer = None 792 self.header_buffer = StringIO() 793 self.body_buffer = StringIO() 794 self.parser_status = self.PS_START_LINE 795 self.content_length = None 796 self.peer_will_close = None 797 798 buf = "" 799 eof = False 800 while self.parser_status != self.PS_COMPLETE: 801 # TODO: Don't read more than necessary (Content-Length), otherwise 802 # data might be lost and/or an error could occur 803 data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout) 804 805 if data: 806 buf += data 807 else: 808 eof = True 809 810 # Do some parsing and error checking while more data arrives 811 buf = self._ContinueParsing(buf, eof) 812 813 # Must be done only after the buffer has been evaluated 814 # TODO: Content-Length < len(data read) and connection closed 815 if (eof and 816 self.parser_status in (self.PS_START_LINE, 817 self.PS_HEADERS)): 818 raise HttpError("Connection closed prematurely") 819 820 # Parse rest 821 buf = self._ContinueParsing(buf, True) 822 823 assert self.parser_status == self.PS_COMPLETE 824 assert not buf, "Parser didn't read full response" 825 826 # Body is complete 827 msg.body = self.body_buffer.getvalue()

828

829 - def _ContinueParsing(self, buf, eof):

830 """Main function for HTTP message state machine. 831 832 @type buf: string 833 @param buf: Receive buffer 834 @type eof: bool 835 @param eof: Whether we've reached EOF on the socket 836 @rtype: string 837 @return: Updated receive buffer 838 839 """ 840 # TODO: Use offset instead of slicing when possible 841 if self.parser_status == self.PS_START_LINE: 842 # Expect start line 843 while True: 844 idx = buf.find("\r\n") 845 846 # RFC2616, section 4.1: "In the interest of robustness, servers SHOULD 847 # ignore any empty line(s) received where a Request-Line is expected. 848 # In other words, if the server is reading the protocol stream at the 849 # beginning of a message and receives a CRLF first, it should ignore 850 # the CRLF." 851 if idx == 0: 852 # TODO: Limit number of CRLFs/empty lines for safety? 853 buf = buf[2:] 854 continue 855 856 if idx > 0: 857 self.start_line_buffer = buf[:idx] 858 859 self._CheckStartLineLength(len(self.start_line_buffer)) 860 861 # Remove status line, including CRLF 862 buf = buf[idx + 2:] 863 864 self.msg.start_line = self.ParseStartLine(self.start_line_buffer) 865 866 self.parser_status = self.PS_HEADERS 867 else: 868 # Check whether incoming data is getting too large, otherwise we just 869 # fill our read buffer. 870 self._CheckStartLineLength(len(buf)) 871 872 break 873 874 # TODO: Handle messages without headers 875 if self.parser_status == self.PS_HEADERS: 876 # Wait for header end 877 idx = buf.find("\r\n\r\n") 878 if idx >= 0: 879 self.header_buffer.write(buf[:idx + 2]) 880 881 self._CheckHeaderLength(self.header_buffer.tell()) 882 883 # Remove headers, including CRLF 884 buf = buf[idx + 4:] 885 886 self._ParseHeaders() 887 888 self.parser_status = self.PS_BODY 889 else: 890 # Check whether incoming data is getting too large, otherwise we just 891 # fill our read buffer. 892 self._CheckHeaderLength(len(buf)) 893 894 if self.parser_status == self.PS_BODY: 895 # TODO: Implement max size for body_buffer 896 self.body_buffer.write(buf) 897 buf = "" 898 899 # Check whether we've read everything 900 # 901 # RFC2616, section 4.4: "When a message-body is included with a message, 902 # the transfer-length of that body is determined by one of the following 903 # [...] 5. By the server closing the connection. (Closing the connection 904 # cannot be used to indicate the end of a request body, since that would 905 # leave no possibility for the server to send back a response.)" 906 # 907 # TODO: Error when buffer length > Content-Length header 908 if (eof or 909 self.content_length is None or 910 (self.content_length is not None and 911 self.body_buffer.tell() >= self.content_length)): 912 self.parser_status = self.PS_COMPLETE 913 914 return buf

915

916 - def _CheckStartLineLength(self, length):

917 """Limits the start line buffer size. 918 919 @type length: int 920 @param length: Buffer size 921 922 """ 923 if (self.START_LINE_LENGTH_MAX is not None and 924 length > self.START_LINE_LENGTH_MAX): 925 raise HttpError("Start line longer than %d chars" % 926 self.START_LINE_LENGTH_MAX)

927

928 - def _CheckHeaderLength(self, length):

929 """Limits the header buffer size. 930 931 @type length: int 932 @param length: Buffer size 933 934 """ 935 if (self.HEADER_LENGTH_MAX is not None and 936 length > self.HEADER_LENGTH_MAX): 937 raise HttpError("Headers longer than %d chars" % self.HEADER_LENGTH_MAX)

938

939 - def ParseStartLine(self, start_line):

940 """Parses the start line of a message. 941 942 Must be overridden by subclass. 943 944 @type start_line: string 945 @param start_line: Start line string 946 947 """ 948 raise NotImplementedError()

949

950 - def _WillPeerCloseConnection(self):

951 """Evaluate whether peer will close the connection. 952 953 @rtype: bool 954 @return: Whether peer will close the connection 955 956 """ 957 # RFC2616, section 14.10: "HTTP/1.1 defines the "close" connection option 958 # for the sender to signal that the connection will be closed after 959 # completion of the response. For example, 960 # 961 # Connection: close 962 # 963 # in either the request or the response header fields indicates that the 964 # connection SHOULD NOT be considered `persistent' (section 8.1) after the 965 # current request/response is complete." 966 967 hdr_connection = self.msg.headers.get(HTTP_CONNECTION, None) 968 if hdr_connection: 969 hdr_connection = hdr_connection.lower() 970 971 # An HTTP/1.1 server is assumed to stay open unless explicitly closed. 972 if self.msg.start_line.version == HTTP_1_1: 973 return (hdr_connection and "close" in hdr_connection) 974 975 # Some HTTP/1.0 implementations have support for persistent connections, 976 # using rules different than HTTP/1.1. 977 978 # For older HTTP, Keep-Alive indicates persistent connection. 979 if self.msg.headers.get(HTTP_KEEP_ALIVE): 980 return False 981 982 # At least Akamai returns a "Connection: Keep-Alive" header, which was 983 # supposed to be sent by the client. 984 if hdr_connection and "keep-alive" in hdr_connection: 985 return False 986 987 return True

988

989 - def _ParseHeaders(self):

990 """Parses the headers. 991 992 This function also adjusts internal variables based on header values. 993 994 RFC2616, section 4.3: The presence of a message-body in a request is 995 signaled by the inclusion of a Content-Length or Transfer-Encoding header 996 field in the request's message-headers. 997 998 """ 999 # Parse headers 1000 self.header_buffer.seek(0, 0) 1001 self.msg.headers = mimetools.Message(self.header_buffer, 0) 1002 1003 self.peer_will_close = self._WillPeerCloseConnection() 1004 1005 # Do we have a Content-Length header? 1006 hdr_content_length = self.msg.headers.get(HTTP_CONTENT_LENGTH, None) 1007 if hdr_content_length: 1008 try: 1009 self.content_length = int(hdr_content_length) 1010 except (TypeError, ValueError): 1011 self.content_length = None 1012 if self.content_length is not None and self.content_length < 0: 1013 self.content_length = None 1014 1015 # if the connection remains open and a content-length was not provided, 1016 # then assume that the connection WILL close. 1017 if self.content_length is None: 1018 self.peer_will_close = True

1019

Source Code for Package ganeti.http