ganeti.http

1 # 2 # 3 4 # Copyright (C) 2007, 2008, 2010, 2012 Google Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 # 11 # This program is distributed in the hope that it will be useful, but 12 # WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 # General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19 # 02110-1301, USA. 20 21 """HTTP module. 22 23 """ 24 25 import logging 26 import mimetools 27 import OpenSSL 28 import select 29 import socket 30 import errno 31 32 from cStringIO import StringIO 33 34 from ganeti import constants 35 from ganeti import utils 36 37 38 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION 39 40 HTTP_OK = 200 41 HTTP_NO_CONTENT = 204 42 HTTP_NOT_MODIFIED = 304 43 44 HTTP_0_9 = "HTTP/0.9" 45 HTTP_1_0 = "HTTP/1.0" 46 HTTP_1_1 = "HTTP/1.1" 47 48 HTTP_GET = "GET" 49 HTTP_HEAD = "HEAD" 50 HTTP_POST = "POST" 51 HTTP_PUT = "PUT" 52 HTTP_DELETE = "DELETE" 53 54 HTTP_ETAG = "ETag" 55 HTTP_HOST = "Host" 56 HTTP_SERVER = "Server" 57 HTTP_DATE = "Date" 58 HTTP_USER_AGENT = "User-Agent" 59 HTTP_CONTENT_TYPE = "Content-Type" 60 HTTP_CONTENT_LENGTH = "Content-Length" 61 HTTP_CONNECTION = "Connection" 62 HTTP_KEEP_ALIVE = "Keep-Alive" 63 HTTP_WWW_AUTHENTICATE = "WWW-Authenticate" 64 HTTP_AUTHORIZATION = "Authorization" 65 HTTP_AUTHENTICATION_INFO = "Authentication-Info" 66 HTTP_ALLOW = "Allow" 67 68 HTTP_APP_OCTET_STREAM = "application/octet-stream" 69 HTTP_APP_JSON = "application/json" 70 71 _SSL_UNEXPECTED_EOF = "Unexpected EOF" 72 73 # Socket operations 74 (SOCKOP_SEND, 75 SOCKOP_RECV, 76 SOCKOP_SHUTDOWN, 77 SOCKOP_HANDSHAKE) = range(4) 78 79 # send/receive quantum 80 SOCK_BUF_SIZE = 32768 81 82

83 -class HttpError(Exception):

84 """Internal exception for HTTP errors. 85 86 This should only be used for internal error reporting. 87 88 """

89 90

91 -class HttpConnectionClosed(Exception):

92 """Internal exception for a closed connection. 93 94 This should only be used for internal error reporting. Only use 95 it if there's no other way to report this condition. 96 97 """

98 99

100 -class HttpSessionHandshakeUnexpectedEOF(HttpError):

101 """Internal exception for errors during SSL handshake. 102 103 This should only be used for internal error reporting. 104 105 """

106 107

108 -class HttpSocketTimeout(Exception):

109 """Internal exception for socket timeouts. 110 111 This should only be used for internal error reporting. 112 113 """

114 115

116 -class HttpException(Exception):

117 code = None 118 message = None 119

120 - def __init__(self, message=None, headers=None):

121 Exception.__init__(self) 122 self.message = message 123 self.headers = headers

124 125

126 -class HttpBadRequest(HttpException):

127 """400 Bad Request 128 129 RFC2616, 10.4.1: The request could not be understood by the server 130 due to malformed syntax. The client SHOULD NOT repeat the request 131 without modifications. 132 133 """ 134 code = 400

135 136

137 -class HttpUnauthorized(HttpException):

138 """401 Unauthorized 139 140 RFC2616, section 10.4.2: The request requires user 141 authentication. The response MUST include a WWW-Authenticate header 142 field (section 14.47) containing a challenge applicable to the 143 requested resource. 144 145 """ 146 code = 401

147 148

149 -class HttpForbidden(HttpException):

150 """403 Forbidden 151 152 RFC2616, 10.4.4: The server understood the request, but is refusing 153 to fulfill it. Authorization will not help and the request SHOULD 154 NOT be repeated. 155 156 """ 157 code = 403

158 159

160 -class HttpNotFound(HttpException):

161 """404 Not Found 162 163 RFC2616, 10.4.5: The server has not found anything matching the 164 Request-URI. No indication is given of whether the condition is 165 temporary or permanent. 166 167 """ 168 code = 404

169 170

171 -class HttpMethodNotAllowed(HttpException):

172 """405 Method Not Allowed 173 174 RFC2616, 10.4.6: The method specified in the Request-Line is not 175 allowed for the resource identified by the Request-URI. The response 176 MUST include an Allow header containing a list of valid methods for 177 the requested resource. 178 179 """ 180 code = 405

181 182

183 -class HttpNotAcceptable(HttpException):

184 """406 Not Acceptable 185 186 RFC2616, 10.4.7: The resource identified by the request is only capable of 187 generating response entities which have content characteristics not 188 acceptable according to the accept headers sent in the request. 189 190 """ 191 code = 406

192 193

194 -class HttpRequestTimeout(HttpException):

195 """408 Request Timeout 196 197 RFC2616, 10.4.9: The client did not produce a request within the 198 time that the server was prepared to wait. The client MAY repeat the 199 request without modifications at any later time. 200 201 """ 202 code = 408

203 204

205 -class HttpConflict(HttpException):

206 """409 Conflict 207 208 RFC2616, 10.4.10: The request could not be completed due to a 209 conflict with the current state of the resource. This code is only 210 allowed in situations where it is expected that the user might be 211 able to resolve the conflict and resubmit the request. 212 213 """ 214 code = 409

215 216

217 -class HttpGone(HttpException):

218 """410 Gone 219 220 RFC2616, 10.4.11: The requested resource is no longer available at 221 the server and no forwarding address is known. This condition is 222 expected to be considered permanent. 223 224 """ 225 code = 410

226 227

228 -class HttpLengthRequired(HttpException):

229 """411 Length Required 230 231 RFC2616, 10.4.12: The server refuses to accept the request without a 232 defined Content-Length. The client MAY repeat the request if it adds 233 a valid Content-Length header field containing the length of the 234 message-body in the request message. 235 236 """ 237 code = 411

238 239

240 -class HttpPreconditionFailed(HttpException):

241 """412 Precondition Failed 242 243 RFC2616, 10.4.13: The precondition given in one or more of the 244 request-header fields evaluated to false when it was tested on the 245 server. 246 247 """ 248 code = 412

249 250

251 -class HttpUnsupportedMediaType(HttpException):

252 """415 Unsupported Media Type 253 254 RFC2616, 10.4.16: The server is refusing to service the request because the 255 entity of the request is in a format not supported by the requested resource 256 for the requested method. 257 258 """ 259 code = 415

260 261

262 -class HttpInternalServerError(HttpException):

263 """500 Internal Server Error 264 265 RFC2616, 10.5.1: The server encountered an unexpected condition 266 which prevented it from fulfilling the request. 267 268 """ 269 code = 500

270 271

272 -class HttpNotImplemented(HttpException):

273 """501 Not Implemented 274 275 RFC2616, 10.5.2: The server does not support the functionality 276 required to fulfill the request. 277 278 """ 279 code = 501

280 281

282 -class HttpBadGateway(HttpException):

283 """502 Bad Gateway 284 285 RFC2616, 10.5.3: The server, while acting as a gateway or proxy, 286 received an invalid response from the upstream server it accessed in 287 attempting to fulfill the request. 288 289 """ 290 code = 502

291 292

293 -class HttpServiceUnavailable(HttpException):

294 """503 Service Unavailable 295 296 RFC2616, 10.5.4: The server is currently unable to handle the 297 request due to a temporary overloading or maintenance of the server. 298 299 """ 300 code = 503

301 302

303 -class HttpGatewayTimeout(HttpException):

304 """504 Gateway Timeout 305 306 RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did 307 not receive a timely response from the upstream server specified by 308 the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server 309 (e.g. DNS) it needed to access in attempting to complete the 310 request. 311 312 """ 313 code = 504

314 315

316 -class HttpVersionNotSupported(HttpException):

317 """505 HTTP Version Not Supported 318 319 RFC2616, 10.5.6: The server does not support, or refuses to support, 320 the HTTP protocol version that was used in the request message. 321 322 """ 323 code = 505

324 325

326 -def ParseHeaders(buf):

327 """Parses HTTP headers. 328 329 @note: This is just a trivial wrapper around C{mimetools.Message} 330 331 """ 332 return mimetools.Message(buf, 0)

333 334

335 -def SocketOperation(sock, op, arg1, timeout):

336 """Wrapper around socket functions. 337 338 This function abstracts error handling for socket operations, especially 339 for the complicated interaction with OpenSSL. 340 341 @type sock: socket 342 @param sock: Socket for the operation 343 @type op: int 344 @param op: Operation to execute (SOCKOP_* constants) 345 @type arg1: any 346 @param arg1: Parameter for function (if needed) 347 @type timeout: None or float 348 @param timeout: Timeout in seconds or None 349 @return: Return value of socket function 350 351 """ 352 # TODO: event_poll/event_check/override 353 if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE): 354 event_poll = select.POLLOUT 355 356 elif op == SOCKOP_RECV: 357 event_poll = select.POLLIN 358 359 elif op == SOCKOP_SHUTDOWN: 360 event_poll = None 361 362 # The timeout is only used when OpenSSL requests polling for a condition. 363 # It is not advisable to have no timeout for shutdown. 364 assert timeout 365 366 else: 367 raise AssertionError("Invalid socket operation") 368 369 # Handshake is only supported by SSL sockets 370 if (op == SOCKOP_HANDSHAKE and 371 not isinstance(sock, OpenSSL.SSL.ConnectionType)): 372 return 373 374 # No override by default 375 event_override = 0 376 377 while True: 378 # Poll only for certain operations and when asked for by an override 379 if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE): 380 if event_override: 381 wait_for_event = event_override 382 else: 383 wait_for_event = event_poll 384 385 event = utils.WaitForFdCondition(sock, wait_for_event, timeout) 386 if event is None: 387 raise HttpSocketTimeout() 388 389 if event & (select.POLLNVAL | select.POLLHUP | select.POLLERR): 390 # Let the socket functions handle these 391 break 392 393 if not event & wait_for_event: 394 continue 395 396 # Reset override 397 event_override = 0 398 399 try: 400 try: 401 if op == SOCKOP_SEND: 402 return sock.send(arg1) 403 404 elif op == SOCKOP_RECV: 405 return sock.recv(arg1) 406 407 elif op == SOCKOP_SHUTDOWN: 408 if isinstance(sock, OpenSSL.SSL.ConnectionType): 409 # PyOpenSSL's shutdown() doesn't take arguments 410 return sock.shutdown() 411 else: 412 return sock.shutdown(arg1) 413 414 elif op == SOCKOP_HANDSHAKE: 415 return sock.do_handshake() 416 417 except OpenSSL.SSL.WantWriteError: 418 # OpenSSL wants to write, poll for POLLOUT 419 event_override = select.POLLOUT 420 continue 421 422 except OpenSSL.SSL.WantReadError: 423 # OpenSSL wants to read, poll for POLLIN 424 event_override = select.POLLIN | select.POLLPRI 425 continue 426 427 except OpenSSL.SSL.WantX509LookupError: 428 continue 429 430 except OpenSSL.SSL.ZeroReturnError, err: 431 # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only 432 # occurs if a closure alert has occurred in the protocol, i.e. the 433 # connection has been closed cleanly. Note that this does not 434 # necessarily mean that the transport layer (e.g. a socket) has been 435 # closed. 436 if op == SOCKOP_SEND: 437 # Can happen during a renegotiation 438 raise HttpConnectionClosed(err.args) 439 elif op == SOCKOP_RECV: 440 return "" 441 442 # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN 443 raise socket.error(err.args) 444 445 except OpenSSL.SSL.SysCallError, err: 446 if op == SOCKOP_SEND: 447 # arg1 is the data when writing 448 if err.args and err.args[0] == -1 and arg1 == "": 449 # errors when writing empty strings are expected 450 # and can be ignored 451 return 0 452 453 if err.args == (-1, _SSL_UNEXPECTED_EOF): 454 if op == SOCKOP_RECV: 455 return "" 456 elif op == SOCKOP_HANDSHAKE: 457 # Can happen if peer disconnects directly after the connection is 458 # opened. 459 raise HttpSessionHandshakeUnexpectedEOF(err.args) 460 461 raise socket.error(err.args) 462 463 except OpenSSL.SSL.Error, err: 464 raise socket.error(err.args) 465 466 except socket.error, err: 467 if err.args and err.args[0] == errno.EAGAIN: 468 # Ignore EAGAIN 469 continue 470 471 raise

472 473

474 -def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force):

475 """Closes the connection. 476 477 @type sock: socket 478 @param sock: Socket to be shut down 479 @type close_timeout: float 480 @param close_timeout: How long to wait for the peer to close 481 the connection 482 @type write_timeout: float 483 @param write_timeout: Write timeout for shutdown 484 @type msgreader: http.HttpMessageReader 485 @param msgreader: Request message reader, used to determine whether 486 peer should close connection 487 @type force: bool 488 @param force: Whether to forcibly close the connection without 489 waiting for peer 490 491 """ 492 #print msgreader.peer_will_close, force 493 if msgreader and msgreader.peer_will_close and not force: 494 # Wait for peer to close 495 try: 496 # Check whether it's actually closed 497 if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout): 498 return 499 except (socket.error, HttpError, HttpSocketTimeout): 500 # Ignore errors at this stage 501 pass 502 503 # Close the connection from our side 504 try: 505 # We don't care about the return value, see NOTES in SSL_shutdown(3). 506 SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR, 507 write_timeout) 508 except HttpSocketTimeout: 509 raise HttpError("Timeout while shutting down connection") 510 except socket.error, err: 511 # Ignore ENOTCONN 512 if not (err.args and err.args[0] == errno.ENOTCONN): 513 raise HttpError("Error while shutting down connection: %s" % err)

514 515

516 -def Handshake(sock, write_timeout):

517 """Shakes peer's hands. 518 519 @type sock: socket 520 @param sock: Socket to be shut down 521 @type write_timeout: float 522 @param write_timeout: Write timeout for handshake 523 524 """ 525 try: 526 return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout) 527 except HttpSocketTimeout: 528 raise HttpError("Timeout during SSL handshake") 529 except socket.error, err: 530 raise HttpError("Error in SSL handshake: %s" % err)

531 532

533 -class HttpSslParams(object):

534 """Data class for SSL key and certificate. 535 536 """

537 - def __init__(self, ssl_key_path, ssl_cert_path):

538 """Initializes this class. 539 540 @type ssl_key_path: string 541 @param ssl_key_path: Path to file containing SSL key in PEM format 542 @type ssl_cert_path: string 543 @param ssl_cert_path: Path to file containing SSL certificate 544 in PEM format 545 546 """ 547 self.ssl_key_pem = utils.ReadFile(ssl_key_path) 548 self.ssl_cert_pem = utils.ReadFile(ssl_cert_path) 549 self.ssl_cert_path = ssl_cert_path

550

551 - def GetKey(self):

552 return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, 553 self.ssl_key_pem)

554

555 - def GetCertificate(self):

556 return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, 557 self.ssl_cert_pem)

558 559

560 -class HttpBase(object):

561 """Base class for HTTP server and client. 562 563 """

564 - def __init__(self):

565 self.using_ssl = None 566 self._ssl_params = None 567 self._ssl_key = None 568 self._ssl_cert = None

569

570 - def _CreateSocket(self, ssl_params, ssl_verify_peer, family):

571 """Creates a TCP socket and initializes SSL if needed. 572 573 @type ssl_params: HttpSslParams 574 @param ssl_params: SSL key and certificate 575 @type ssl_verify_peer: bool 576 @param ssl_verify_peer: Whether to require client certificate 577 and compare it with our certificate 578 @type family: int 579 @param family: socket.AF_INET | socket.AF_INET6 580 581 """ 582 assert family in (socket.AF_INET, socket.AF_INET6) 583 584 self._ssl_params = ssl_params 585 sock = socket.socket(family, socket.SOCK_STREAM) 586 587 # Should we enable SSL? 588 self.using_ssl = ssl_params is not None 589 590 if not self.using_ssl: 591 return sock 592 593 self._ssl_key = ssl_params.GetKey() 594 self._ssl_cert = ssl_params.GetCertificate() 595 596 ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) 597 ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2) 598 599 ciphers = self.GetSslCiphers() 600 logging.debug("Setting SSL cipher string %s", ciphers) 601 ctx.set_cipher_list(ciphers) 602 603 ctx.use_privatekey(self._ssl_key) 604 ctx.use_certificate(self._ssl_cert) 605 ctx.check_privatekey() 606 607 if ssl_verify_peer: 608 ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | 609 OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, 610 self._SSLVerifyCallback) 611 612 # Also add our certificate as a trusted CA to be sent to the client. 613 # This is required at least for GnuTLS clients to work. 614 try: 615 # This will fail for PyOpenssl versions before 0.10 616 ctx.add_client_ca(self._ssl_cert) 617 except AttributeError: 618 # Fall back to letting OpenSSL read the certificate file directly. 619 ctx.load_client_ca(ssl_params.ssl_cert_path) 620 621 return OpenSSL.SSL.Connection(ctx, sock)

622

623 - def GetSslCiphers(self): # pylint: disable=R0201

624 """Returns the ciphers string for SSL. 625 626 """ 627 return constants.OPENSSL_CIPHERS

628

629 - def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok):

630 """Verify the certificate provided by the peer 631 632 We only compare fingerprints. The client must use the same certificate as 633 we do on our side. 634 635 """ 636 # some parameters are unused, but this is the API 637 # pylint: disable=W0613 638 assert self._ssl_params, "SSL not initialized" 639 640 return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and 641 self._ssl_cert.digest("md5") == cert.digest("md5"))

642 643

644 -class HttpMessage(object):

645 """Data structure for HTTP message. 646 647 """

648 - def __init__(self):

649 self.start_line = None 650 self.headers = None 651 self.body = None

652 653

654 -class HttpClientToServerStartLine(object):

655 """Data structure for HTTP request start line. 656 657 """

658 - def __init__(self, method, path, version):

659 self.method = method 660 self.path = path 661 self.version = version

662

663 - def __str__(self):

664 return "%s %s %s" % (self.method, self.path, self.version)

665 666

667 -class HttpServerToClientStartLine(object):

668 """Data structure for HTTP response start line. 669 670 """

671 - def __init__(self, version, code, reason):

672 self.version = version 673 self.code = code 674 self.reason = reason

675

676 - def __str__(self):

677 return "%s %s %s" % (self.version, self.code, self.reason)

678 679

680 -class HttpMessageWriter(object):

681 """Writes an HTTP message to a socket. 682 683 """

684 - def __init__(self, sock, msg, write_timeout):

685 """Initializes this class and writes an HTTP message to a socket. 686 687 @type sock: socket 688 @param sock: Socket to be written to 689 @type msg: http.HttpMessage 690 @param msg: HTTP message to be written 691 @type write_timeout: float 692 @param write_timeout: Write timeout for socket 693 694 """ 695 self._msg = msg 696 697 self._PrepareMessage() 698 699 buf = self._FormatMessage() 700 701 pos = 0 702 end = len(buf) 703 while pos < end: 704 # Send only SOCK_BUF_SIZE bytes at a time 705 data = buf[pos:(pos + SOCK_BUF_SIZE)] 706 707 sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout) 708 709 # Remove sent bytes 710 pos += sent 711 712 assert pos == end, "Message wasn't sent completely"

713

714 - def _PrepareMessage(self):

715 """Prepares the HTTP message by setting mandatory headers. 716 717 """ 718 # RFC2616, section 4.3: "The presence of a message-body in a request is 719 # signaled by the inclusion of a Content-Length or Transfer-Encoding header 720 # field in the request's message-headers." 721 if self._msg.body: 722 self._msg.headers[HTTP_CONTENT_LENGTH] = len(self._msg.body)

723

724 - def _FormatMessage(self):

725 """Serializes the HTTP message into a string. 726 727 """ 728 buf = StringIO() 729 730 # Add start line 731 buf.write(str(self._msg.start_line)) 732 buf.write("\r\n") 733 734 # Add headers 735 if self._msg.start_line.version != HTTP_0_9: 736 for name, value in self._msg.headers.iteritems(): 737 buf.write("%s: %s\r\n" % (name, value)) 738 739 buf.write("\r\n") 740 741 # Add message body if needed 742 if self.HasMessageBody(): 743 buf.write(self._msg.body) 744 745 elif self._msg.body: 746 logging.warning("Ignoring message body") 747 748 return buf.getvalue()

749

750 - def HasMessageBody(self):

751 """Checks whether the HTTP message contains a body. 752 753 Can be overridden by subclasses. 754 755 """ 756 return bool(self._msg.body)

757 758

759 -class HttpMessageReader(object):

760 """Reads HTTP message from socket. 761 762 """ 763 # Length limits 764 START_LINE_LENGTH_MAX = None 765 HEADER_LENGTH_MAX = None 766 767 # Parser state machine 768 PS_START_LINE = "start-line" 769 PS_HEADERS = "headers" 770 PS_BODY = "entity-body" 771 PS_COMPLETE = "complete" 772

773 - def __init__(self, sock, msg, read_timeout):

774 """Reads an HTTP message from a socket. 775 776 @type sock: socket 777 @param sock: Socket to be read from 778 @type msg: http.HttpMessage 779 @param msg: Object for the read message 780 @type read_timeout: float 781 @param read_timeout: Read timeout for socket 782 783 """ 784 self.sock = sock 785 self.msg = msg 786 787 self.start_line_buffer = None 788 self.header_buffer = StringIO() 789 self.body_buffer = StringIO() 790 self.parser_status = self.PS_START_LINE 791 self.content_length = None 792 self.peer_will_close = None 793 794 buf = "" 795 eof = False 796 while self.parser_status != self.PS_COMPLETE: 797 # TODO: Don't read more than necessary (Content-Length), otherwise 798 # data might be lost and/or an error could occur 799 data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout) 800 801 if data: 802 buf += data 803 else: 804 eof = True 805 806 # Do some parsing and error checking while more data arrives 807 buf = self._ContinueParsing(buf, eof) 808 809 # Must be done only after the buffer has been evaluated 810 # TODO: Content-Length < len(data read) and connection closed 811 if (eof and 812 self.parser_status in (self.PS_START_LINE, 813 self.PS_HEADERS)): 814 raise HttpError("Connection closed prematurely") 815 816 # Parse rest 817 buf = self._ContinueParsing(buf, True) 818 819 assert self.parser_status == self.PS_COMPLETE 820 assert not buf, "Parser didn't read full response" 821 822 # Body is complete 823 msg.body = self.body_buffer.getvalue()

824

825 - def _ContinueParsing(self, buf, eof):

826 """Main function for HTTP message state machine. 827 828 @type buf: string 829 @param buf: Receive buffer 830 @type eof: bool 831 @param eof: Whether we've reached EOF on the socket 832 @rtype: string 833 @return: Updated receive buffer 834 835 """ 836 # TODO: Use offset instead of slicing when possible 837 if self.parser_status == self.PS_START_LINE: 838 # Expect start line 839 while True: 840 idx = buf.find("\r\n") 841 842 # RFC2616, section 4.1: "In the interest of robustness, servers SHOULD 843 # ignore any empty line(s) received where a Request-Line is expected. 844 # In other words, if the server is reading the protocol stream at the 845 # beginning of a message and receives a CRLF first, it should ignore 846 # the CRLF." 847 if idx == 0: 848 # TODO: Limit number of CRLFs/empty lines for safety? 849 buf = buf[2:] 850 continue 851 852 if idx > 0: 853 self.start_line_buffer = buf[:idx] 854 855 self._CheckStartLineLength(len(self.start_line_buffer)) 856 857 # Remove status line, including CRLF 858 buf = buf[idx + 2:] 859 860 self.msg.start_line = self.ParseStartLine(self.start_line_buffer) 861 862 self.parser_status = self.PS_HEADERS 863 else: 864 # Check whether incoming data is getting too large, otherwise we just 865 # fill our read buffer. 866 self._CheckStartLineLength(len(buf)) 867 868 break 869 870 # TODO: Handle messages without headers 871 if self.parser_status == self.PS_HEADERS: 872 # Wait for header end 873 idx = buf.find("\r\n\r\n") 874 if idx >= 0: 875 self.header_buffer.write(buf[:idx + 2]) 876 877 self._CheckHeaderLength(self.header_buffer.tell()) 878 879 # Remove headers, including CRLF 880 buf = buf[idx + 4:] 881 882 self._ParseHeaders() 883 884 self.parser_status = self.PS_BODY 885 else: 886 # Check whether incoming data is getting too large, otherwise we just 887 # fill our read buffer. 888 self._CheckHeaderLength(len(buf)) 889 890 if self.parser_status == self.PS_BODY: 891 # TODO: Implement max size for body_buffer 892 self.body_buffer.write(buf) 893 buf = "" 894 895 # Check whether we've read everything 896 # 897 # RFC2616, section 4.4: "When a message-body is included with a message, 898 # the transfer-length of that body is determined by one of the following 899 # [...] 5. By the server closing the connection. (Closing the connection 900 # cannot be used to indicate the end of a request body, since that would 901 # leave no possibility for the server to send back a response.)" 902 # 903 # TODO: Error when buffer length > Content-Length header 904 if (eof or 905 self.content_length is None or 906 (self.content_length is not None and 907 self.body_buffer.tell() >= self.content_length)): 908 self.parser_status = self.PS_COMPLETE 909 910 return buf

911

912 - def _CheckStartLineLength(self, length):

913 """Limits the start line buffer size. 914 915 @type length: int 916 @param length: Buffer size 917 918 """ 919 if (self.START_LINE_LENGTH_MAX is not None and 920 length > self.START_LINE_LENGTH_MAX): 921 raise HttpError("Start line longer than %d chars" % 922 self.START_LINE_LENGTH_MAX)

923

924 - def _CheckHeaderLength(self, length):

925 """Limits the header buffer size. 926 927 @type length: int 928 @param length: Buffer size 929 930 """ 931 if (self.HEADER_LENGTH_MAX is not None and 932 length > self.HEADER_LENGTH_MAX): 933 raise HttpError("Headers longer than %d chars" % self.HEADER_LENGTH_MAX)

934

935 - def ParseStartLine(self, start_line):

936 """Parses the start line of a message. 937 938 Must be overridden by subclass. 939 940 @type start_line: string 941 @param start_line: Start line string 942 943 """ 944 raise NotImplementedError()

945

946 - def _WillPeerCloseConnection(self):

947 """Evaluate whether peer will close the connection. 948 949 @rtype: bool 950 @return: Whether peer will close the connection 951 952 """ 953 # RFC2616, section 14.10: "HTTP/1.1 defines the "close" connection option 954 # for the sender to signal that the connection will be closed after 955 # completion of the response. For example, 956 # 957 # Connection: close 958 # 959 # in either the request or the response header fields indicates that the 960 # connection SHOULD NOT be considered `persistent' (section 8.1) after the 961 # current request/response is complete." 962 963 hdr_connection = self.msg.headers.get(HTTP_CONNECTION, None) 964 if hdr_connection: 965 hdr_connection = hdr_connection.lower() 966 967 # An HTTP/1.1 server is assumed to stay open unless explicitly closed. 968 if self.msg.start_line.version == HTTP_1_1: 969 return (hdr_connection and "close" in hdr_connection) 970 971 # Some HTTP/1.0 implementations have support for persistent connections, 972 # using rules different than HTTP/1.1. 973 974 # For older HTTP, Keep-Alive indicates persistent connection. 975 if self.msg.headers.get(HTTP_KEEP_ALIVE): 976 return False 977 978 # At least Akamai returns a "Connection: Keep-Alive" header, which was 979 # supposed to be sent by the client. 980 if hdr_connection and "keep-alive" in hdr_connection: 981 return False 982 983 return True

984

985 - def _ParseHeaders(self):

986 """Parses the headers. 987 988 This function also adjusts internal variables based on header values. 989 990 RFC2616, section 4.3: The presence of a message-body in a request is 991 signaled by the inclusion of a Content-Length or Transfer-Encoding header 992 field in the request's message-headers. 993 994 """ 995 # Parse headers 996 self.header_buffer.seek(0, 0) 997 self.msg.headers = ParseHeaders(self.header_buffer) 998 999 self.peer_will_close = self._WillPeerCloseConnection() 1000 1001 # Do we have a Content-Length header? 1002 hdr_content_length = self.msg.headers.get(HTTP_CONTENT_LENGTH, None) 1003 if hdr_content_length: 1004 try: 1005 self.content_length = int(hdr_content_length) 1006 except (TypeError, ValueError): 1007 self.content_length = None 1008 if self.content_length is not None and self.content_length < 0: 1009 self.content_length = None 1010 1011 # if the connection remains open and a content-length was not provided, 1012 # then assume that the connection WILL close. 1013 if self.content_length is None: 1014 self.peer_will_close = True

1015

Source Code for Package ganeti.http