ganeti.http

1 # 2 # 3 4 # Copyright (C) 2007, 2008 Google Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 # 11 # This program is distributed in the hope that it will be useful, but 12 # WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 # General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19 # 02110-1301, USA. 20 21 """HTTP module. 22 23 """ 24 25 import logging 26 import mimetools 27 import OpenSSL 28 import select 29 import socket 30 import errno 31 32 from cStringIO import StringIO 33 34 from ganeti import constants 35 from ganeti import utils 36 37 38 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION 39 40 HTTP_OK = 200 41 HTTP_NO_CONTENT = 204 42 HTTP_NOT_MODIFIED = 304 43 44 HTTP_0_9 = "HTTP/0.9" 45 HTTP_1_0 = "HTTP/1.0" 46 HTTP_1_1 = "HTTP/1.1" 47 48 HTTP_GET = "GET" 49 HTTP_HEAD = "HEAD" 50 HTTP_POST = "POST" 51 HTTP_PUT = "PUT" 52 HTTP_DELETE = "DELETE" 53 54 HTTP_ETAG = "ETag" 55 HTTP_HOST = "Host" 56 HTTP_SERVER = "Server" 57 HTTP_DATE = "Date" 58 HTTP_USER_AGENT = "User-Agent" 59 HTTP_CONTENT_TYPE = "Content-Type" 60 HTTP_CONTENT_LENGTH = "Content-Length" 61 HTTP_CONNECTION = "Connection" 62 HTTP_KEEP_ALIVE = "Keep-Alive" 63 HTTP_WWW_AUTHENTICATE = "WWW-Authenticate" 64 HTTP_AUTHORIZATION = "Authorization" 65 HTTP_AUTHENTICATION_INFO = "Authentication-Info" 66 HTTP_ALLOW = "Allow" 67 68 HTTP_APP_OCTET_STREAM = "application/octet-stream" 69 HTTP_APP_JSON = "application/json" 70 71 _SSL_UNEXPECTED_EOF = "Unexpected EOF" 72 73 # Socket operations 74 (SOCKOP_SEND, 75 SOCKOP_RECV, 76 SOCKOP_SHUTDOWN, 77 SOCKOP_HANDSHAKE) = range(4) 78 79 # send/receive quantum 80 SOCK_BUF_SIZE = 32768 81 82

83 -class HttpError(Exception):

84 """Internal exception for HTTP errors. 85 86 This should only be used for internal error reporting. 87 88 """

89 90

91 -class HttpConnectionClosed(Exception):

92 """Internal exception for a closed connection. 93 94 This should only be used for internal error reporting. Only use 95 it if there's no other way to report this condition. 96 97 """

98 99

100 -class HttpSessionHandshakeUnexpectedEOF(HttpError):

101 """Internal exception for errors during SSL handshake. 102 103 This should only be used for internal error reporting. 104 105 """

106 107

108 -class HttpSocketTimeout(Exception):

109 """Internal exception for socket timeouts. 110 111 This should only be used for internal error reporting. 112 113 """

114 115

116 -class HttpException(Exception):

117 code = None 118 message = None 119

120 - def __init__(self, message=None, headers=None):

121 Exception.__init__(self) 122 self.message = message 123 self.headers = headers

124 125

126 -class HttpBadRequest(HttpException):

127 """400 Bad Request 128 129 RFC2616, 10.4.1: The request could not be understood by the server 130 due to malformed syntax. The client SHOULD NOT repeat the request 131 without modifications. 132 133 """ 134 code = 400

135 136

137 -class HttpUnauthorized(HttpException):

138 """401 Unauthorized 139 140 RFC2616, section 10.4.2: The request requires user 141 authentication. The response MUST include a WWW-Authenticate header 142 field (section 14.47) containing a challenge applicable to the 143 requested resource. 144 145 """ 146 code = 401

147 148

149 -class HttpForbidden(HttpException):

150 """403 Forbidden 151 152 RFC2616, 10.4.4: The server understood the request, but is refusing 153 to fulfill it. Authorization will not help and the request SHOULD 154 NOT be repeated. 155 156 """ 157 code = 403

158 159

160 -class HttpNotFound(HttpException):

161 """404 Not Found 162 163 RFC2616, 10.4.5: The server has not found anything matching the 164 Request-URI. No indication is given of whether the condition is 165 temporary or permanent. 166 167 """ 168 code = 404

169 170

171 -class HttpMethodNotAllowed(HttpException):

172 """405 Method Not Allowed 173 174 RFC2616, 10.4.6: The method specified in the Request-Line is not 175 allowed for the resource identified by the Request-URI. The response 176 MUST include an Allow header containing a list of valid methods for 177 the requested resource. 178 179 """ 180 code = 405

181 182

183 -class HttpNotAcceptable(HttpException):

184 """406 Not Acceptable 185 186 RFC2616, 10.4.7: The resource identified by the request is only capable of 187 generating response entities which have content characteristics not 188 acceptable according to the accept headers sent in the request. 189 190 """ 191 code = 406

192 193

194 -class HttpRequestTimeout(HttpException):

195 """408 Request Timeout 196 197 RFC2616, 10.4.9: The client did not produce a request within the 198 time that the server was prepared to wait. The client MAY repeat the 199 request without modifications at any later time. 200 201 """ 202 code = 408

203 204

205 -class HttpConflict(HttpException):

206 """409 Conflict 207 208 RFC2616, 10.4.10: The request could not be completed due to a 209 conflict with the current state of the resource. This code is only 210 allowed in situations where it is expected that the user might be 211 able to resolve the conflict and resubmit the request. 212 213 """ 214 code = 409

215 216

217 -class HttpGone(HttpException):

218 """410 Gone 219 220 RFC2616, 10.4.11: The requested resource is no longer available at 221 the server and no forwarding address is known. This condition is 222 expected to be considered permanent. 223 224 """ 225 code = 410

226 227

228 -class HttpLengthRequired(HttpException):

229 """411 Length Required 230 231 RFC2616, 10.4.12: The server refuses to accept the request without a 232 defined Content-Length. The client MAY repeat the request if it adds 233 a valid Content-Length header field containing the length of the 234 message-body in the request message. 235 236 """ 237 code = 411

238 239

240 -class HttpPreconditionFailed(HttpException):

241 """412 Precondition Failed 242 243 RFC2616, 10.4.13: The precondition given in one or more of the 244 request-header fields evaluated to false when it was tested on the 245 server. 246 247 """ 248 code = 412

249 250

251 -class HttpUnsupportedMediaType(HttpException):

252 """415 Unsupported Media Type 253 254 RFC2616, 10.4.16: The server is refusing to service the request because the 255 entity of the request is in a format not supported by the requested resource 256 for the requested method. 257 258 """ 259 code = 415

260 261

262 -class HttpInternalServerError(HttpException):

263 """500 Internal Server Error 264 265 RFC2616, 10.5.1: The server encountered an unexpected condition 266 which prevented it from fulfilling the request. 267 268 """ 269 code = 500

270 271

272 -class HttpNotImplemented(HttpException):

273 """501 Not Implemented 274 275 RFC2616, 10.5.2: The server does not support the functionality 276 required to fulfill the request. 277 278 """ 279 code = 501

280 281

282 -class HttpBadGateway(HttpException):

283 """502 Bad Gateway 284 285 RFC2616, 10.5.3: The server, while acting as a gateway or proxy, 286 received an invalid response from the upstream server it accessed in 287 attempting to fulfill the request. 288 289 """ 290 code = 502

291 292

293 -class HttpServiceUnavailable(HttpException):

294 """503 Service Unavailable 295 296 RFC2616, 10.5.4: The server is currently unable to handle the 297 request due to a temporary overloading or maintenance of the server. 298 299 """ 300 code = 503

301 302

303 -class HttpGatewayTimeout(HttpException):

304 """504 Gateway Timeout 305 306 RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did 307 not receive a timely response from the upstream server specified by 308 the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server 309 (e.g. DNS) it needed to access in attempting to complete the 310 request. 311 312 """ 313 code = 504

314 315

316 -class HttpVersionNotSupported(HttpException):

317 """505 HTTP Version Not Supported 318 319 RFC2616, 10.5.6: The server does not support, or refuses to support, 320 the HTTP protocol version that was used in the request message. 321 322 """ 323 code = 505

324 325

326 -def SocketOperation(sock, op, arg1, timeout):

327 """Wrapper around socket functions. 328 329 This function abstracts error handling for socket operations, especially 330 for the complicated interaction with OpenSSL. 331 332 @type sock: socket 333 @param sock: Socket for the operation 334 @type op: int 335 @param op: Operation to execute (SOCKOP_* constants) 336 @type arg1: any 337 @param arg1: Parameter for function (if needed) 338 @type timeout: None or float 339 @param timeout: Timeout in seconds or None 340 @return: Return value of socket function 341 342 """ 343 # TODO: event_poll/event_check/override 344 if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE): 345 event_poll = select.POLLOUT 346 347 elif op == SOCKOP_RECV: 348 event_poll = select.POLLIN 349 350 elif op == SOCKOP_SHUTDOWN: 351 event_poll = None 352 353 # The timeout is only used when OpenSSL requests polling for a condition. 354 # It is not advisable to have no timeout for shutdown. 355 assert timeout 356 357 else: 358 raise AssertionError("Invalid socket operation") 359 360 # Handshake is only supported by SSL sockets 361 if (op == SOCKOP_HANDSHAKE and 362 not isinstance(sock, OpenSSL.SSL.ConnectionType)): 363 return 364 365 # No override by default 366 event_override = 0 367 368 while True: 369 # Poll only for certain operations and when asked for by an override 370 if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE): 371 if event_override: 372 wait_for_event = event_override 373 else: 374 wait_for_event = event_poll 375 376 event = utils.WaitForFdCondition(sock, wait_for_event, timeout) 377 if event is None: 378 raise HttpSocketTimeout() 379 380 if event & (select.POLLNVAL | select.POLLHUP | select.POLLERR): 381 # Let the socket functions handle these 382 break 383 384 if not event & wait_for_event: 385 continue 386 387 # Reset override 388 event_override = 0 389 390 try: 391 try: 392 if op == SOCKOP_SEND: 393 return sock.send(arg1) 394 395 elif op == SOCKOP_RECV: 396 return sock.recv(arg1) 397 398 elif op == SOCKOP_SHUTDOWN: 399 if isinstance(sock, OpenSSL.SSL.ConnectionType): 400 # PyOpenSSL's shutdown() doesn't take arguments 401 return sock.shutdown() 402 else: 403 return sock.shutdown(arg1) 404 405 elif op == SOCKOP_HANDSHAKE: 406 return sock.do_handshake() 407 408 except OpenSSL.SSL.WantWriteError: 409 # OpenSSL wants to write, poll for POLLOUT 410 event_override = select.POLLOUT 411 continue 412 413 except OpenSSL.SSL.WantReadError: 414 # OpenSSL wants to read, poll for POLLIN 415 event_override = select.POLLIN | select.POLLPRI 416 continue 417 418 except OpenSSL.SSL.WantX509LookupError: 419 continue 420 421 except OpenSSL.SSL.ZeroReturnError, err: 422 # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only 423 # occurs if a closure alert has occurred in the protocol, i.e. the 424 # connection has been closed cleanly. Note that this does not 425 # necessarily mean that the transport layer (e.g. a socket) has been 426 # closed. 427 if op == SOCKOP_SEND: 428 # Can happen during a renegotiation 429 raise HttpConnectionClosed(err.args) 430 elif op == SOCKOP_RECV: 431 return "" 432 433 # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN 434 raise socket.error(err.args) 435 436 except OpenSSL.SSL.SysCallError, err: 437 if op == SOCKOP_SEND: 438 # arg1 is the data when writing 439 if err.args and err.args[0] == -1 and arg1 == "": 440 # errors when writing empty strings are expected 441 # and can be ignored 442 return 0 443 444 if err.args == (-1, _SSL_UNEXPECTED_EOF): 445 if op == SOCKOP_RECV: 446 return "" 447 elif op == SOCKOP_HANDSHAKE: 448 # Can happen if peer disconnects directly after the connection is 449 # opened. 450 raise HttpSessionHandshakeUnexpectedEOF(err.args) 451 452 raise socket.error(err.args) 453 454 except OpenSSL.SSL.Error, err: 455 raise socket.error(err.args) 456 457 except socket.error, err: 458 if err.args and err.args[0] == errno.EAGAIN: 459 # Ignore EAGAIN 460 continue 461 462 raise

463 464

465 -def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force):

466 """Closes the connection. 467 468 @type sock: socket 469 @param sock: Socket to be shut down 470 @type close_timeout: float 471 @param close_timeout: How long to wait for the peer to close 472 the connection 473 @type write_timeout: float 474 @param write_timeout: Write timeout for shutdown 475 @type msgreader: http.HttpMessageReader 476 @param msgreader: Request message reader, used to determine whether 477 peer should close connection 478 @type force: bool 479 @param force: Whether to forcibly close the connection without 480 waiting for peer 481 482 """ 483 #print msgreader.peer_will_close, force 484 if msgreader and msgreader.peer_will_close and not force: 485 # Wait for peer to close 486 try: 487 # Check whether it's actually closed 488 if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout): 489 return 490 except (socket.error, HttpError, HttpSocketTimeout): 491 # Ignore errors at this stage 492 pass 493 494 # Close the connection from our side 495 try: 496 # We don't care about the return value, see NOTES in SSL_shutdown(3). 497 SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR, 498 write_timeout) 499 except HttpSocketTimeout: 500 raise HttpError("Timeout while shutting down connection") 501 except socket.error, err: 502 # Ignore ENOTCONN 503 if not (err.args and err.args[0] == errno.ENOTCONN): 504 raise HttpError("Error while shutting down connection: %s" % err)

505 506

507 -def Handshake(sock, write_timeout):

508 """Shakes peer's hands. 509 510 @type sock: socket 511 @param sock: Socket to be shut down 512 @type write_timeout: float 513 @param write_timeout: Write timeout for handshake 514 515 """ 516 try: 517 return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout) 518 except HttpSocketTimeout: 519 raise HttpError("Timeout during SSL handshake") 520 except socket.error, err: 521 raise HttpError("Error in SSL handshake: %s" % err)

522 523

524 -def InitSsl():

525 """Initializes the SSL infrastructure. 526 527 This function is idempotent. 528 529 """ 530 if not OpenSSL.rand.status(): 531 raise EnvironmentError("OpenSSL could not collect enough entropy" 532 " for the PRNG")

533 534 # TODO: Maybe add some additional seeding for OpenSSL's PRNG 535 536

537 -class HttpSslParams(object):

538 """Data class for SSL key and certificate. 539 540 """

541 - def __init__(self, ssl_key_path, ssl_cert_path):

542 """Initializes this class. 543 544 @type ssl_key_path: string 545 @param ssl_key_path: Path to file containing SSL key in PEM format 546 @type ssl_cert_path: string 547 @param ssl_cert_path: Path to file containing SSL certificate 548 in PEM format 549 550 """ 551 self.ssl_key_pem = utils.ReadFile(ssl_key_path) 552 self.ssl_cert_pem = utils.ReadFile(ssl_cert_path) 553 self.ssl_cert_path = ssl_cert_path

554

555 - def GetKey(self):

556 return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, 557 self.ssl_key_pem)

558

559 - def GetCertificate(self):

560 return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, 561 self.ssl_cert_pem)

562 563

564 -class HttpBase(object):

565 """Base class for HTTP server and client. 566 567 """

568 - def __init__(self):

569 self.using_ssl = None 570 self._ssl_params = None 571 self._ssl_key = None 572 self._ssl_cert = None

573

574 - def _CreateSocket(self, ssl_params, ssl_verify_peer):

575 """Creates a TCP socket and initializes SSL if needed. 576 577 @type ssl_params: HttpSslParams 578 @param ssl_params: SSL key and certificate 579 @type ssl_verify_peer: bool 580 @param ssl_verify_peer: Whether to require client certificate 581 and compare it with our certificate 582 583 """ 584 self._ssl_params = ssl_params 585 586 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 587 588 # Should we enable SSL? 589 self.using_ssl = ssl_params is not None 590 591 if not self.using_ssl: 592 return sock 593 594 self._ssl_key = ssl_params.GetKey() 595 self._ssl_cert = ssl_params.GetCertificate() 596 597 ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) 598 ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2) 599 600 ciphers = self.GetSslCiphers() 601 logging.debug("Setting SSL cipher string %s", ciphers) 602 ctx.set_cipher_list(ciphers) 603 604 ctx.use_privatekey(self._ssl_key) 605 ctx.use_certificate(self._ssl_cert) 606 ctx.check_privatekey() 607 608 if ssl_verify_peer: 609 ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | 610 OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, 611 self._SSLVerifyCallback) 612 613 # Also add our certificate as a trusted CA to be sent to the client. 614 # This is required at least for GnuTLS clients to work. 615 try: 616 # This will fail for PyOpenssl versions before 0.10 617 ctx.add_client_ca(self._ssl_cert) 618 except AttributeError: 619 # Fall back to letting OpenSSL read the certificate file directly. 620 ctx.load_client_ca(ssl_params.ssl_cert_path) 621 622 return OpenSSL.SSL.Connection(ctx, sock)

623

624 - def GetSslCiphers(self): # pylint: disable-msg=R0201

625 """Returns the ciphers string for SSL. 626 627 """ 628 return constants.OPENSSL_CIPHERS

629

630 - def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok):

631 """Verify the certificate provided by the peer 632 633 We only compare fingerprints. The client must use the same certificate as 634 we do on our side. 635 636 """ 637 # some parameters are unused, but this is the API 638 # pylint: disable-msg=W0613 639 assert self._ssl_params, "SSL not initialized" 640 641 return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and 642 self._ssl_cert.digest("md5") == cert.digest("md5"))

643 644

645 -class HttpMessage(object):

646 """Data structure for HTTP message. 647 648 """

649 - def __init__(self):

650 self.start_line = None 651 self.headers = None 652 self.body = None

653 654

655 -class HttpClientToServerStartLine(object):

656 """Data structure for HTTP request start line. 657 658 """

659 - def __init__(self, method, path, version):

660 self.method = method 661 self.path = path 662 self.version = version

663

664 - def __str__(self):

665 return "%s %s %s" % (self.method, self.path, self.version)

666 667

668 -class HttpServerToClientStartLine(object):

669 """Data structure for HTTP response start line. 670 671 """

672 - def __init__(self, version, code, reason):

673 self.version = version 674 self.code = code 675 self.reason = reason

676

677 - def __str__(self):

678 return "%s %s %s" % (self.version, self.code, self.reason)

679 680

681 -class HttpMessageWriter(object):

682 """Writes an HTTP message to a socket. 683 684 """

685 - def __init__(self, sock, msg, write_timeout):

686 """Initializes this class and writes an HTTP message to a socket. 687 688 @type sock: socket 689 @param sock: Socket to be written to 690 @type msg: http.HttpMessage 691 @param msg: HTTP message to be written 692 @type write_timeout: float 693 @param write_timeout: Write timeout for socket 694 695 """ 696 self._msg = msg 697 698 self._PrepareMessage() 699 700 buf = self._FormatMessage() 701 702 pos = 0 703 end = len(buf) 704 while pos < end: 705 # Send only SOCK_BUF_SIZE bytes at a time 706 data = buf[pos:(pos + SOCK_BUF_SIZE)] 707 708 sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout) 709 710 # Remove sent bytes 711 pos += sent 712 713 assert pos == end, "Message wasn't sent completely"

714

715 - def _PrepareMessage(self):

716 """Prepares the HTTP message by setting mandatory headers. 717 718 """ 719 # RFC2616, section 4.3: "The presence of a message-body in a request is 720 # signaled by the inclusion of a Content-Length or Transfer-Encoding header 721 # field in the request's message-headers." 722 if self._msg.body: 723 self._msg.headers[HTTP_CONTENT_LENGTH] = len(self._msg.body)

724

725 - def _FormatMessage(self):

726 """Serializes the HTTP message into a string. 727 728 """ 729 buf = StringIO() 730 731 # Add start line 732 buf.write(str(self._msg.start_line)) 733 buf.write("\r\n") 734 735 # Add headers 736 if self._msg.start_line.version != HTTP_0_9: 737 for name, value in self._msg.headers.iteritems(): 738 buf.write("%s: %s\r\n" % (name, value)) 739 740 buf.write("\r\n") 741 742 # Add message body if needed 743 if self.HasMessageBody(): 744 buf.write(self._msg.body) 745 746 elif self._msg.body: 747 logging.warning("Ignoring message body") 748 749 return buf.getvalue()

750

751 - def HasMessageBody(self):

752 """Checks whether the HTTP message contains a body. 753 754 Can be overridden by subclasses. 755 756 """ 757 return bool(self._msg.body)

758 759

760 -class HttpMessageReader(object):

761 """Reads HTTP message from socket. 762 763 """ 764 # Length limits 765 START_LINE_LENGTH_MAX = None 766 HEADER_LENGTH_MAX = None 767 768 # Parser state machine 769 PS_START_LINE = "start-line" 770 PS_HEADERS = "headers" 771 PS_BODY = "entity-body" 772 PS_COMPLETE = "complete" 773

774 - def __init__(self, sock, msg, read_timeout):

775 """Reads an HTTP message from a socket. 776 777 @type sock: socket 778 @param sock: Socket to be read from 779 @type msg: http.HttpMessage 780 @param msg: Object for the read message 781 @type read_timeout: float 782 @param read_timeout: Read timeout for socket 783 784 """ 785 self.sock = sock 786 self.msg = msg 787 788 self.start_line_buffer = None 789 self.header_buffer = StringIO() 790 self.body_buffer = StringIO() 791 self.parser_status = self.PS_START_LINE 792 self.content_length = None 793 self.peer_will_close = None 794 795 buf = "" 796 eof = False 797 while self.parser_status != self.PS_COMPLETE: 798 # TODO: Don't read more than necessary (Content-Length), otherwise 799 # data might be lost and/or an error could occur 800 data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout) 801 802 if data: 803 buf += data 804 else: 805 eof = True 806 807 # Do some parsing and error checking while more data arrives 808 buf = self._ContinueParsing(buf, eof) 809 810 # Must be done only after the buffer has been evaluated 811 # TODO: Content-Length < len(data read) and connection closed 812 if (eof and 813 self.parser_status in (self.PS_START_LINE, 814 self.PS_HEADERS)): 815 raise HttpError("Connection closed prematurely") 816 817 # Parse rest 818 buf = self._ContinueParsing(buf, True) 819 820 assert self.parser_status == self.PS_COMPLETE 821 assert not buf, "Parser didn't read full response" 822 823 # Body is complete 824 msg.body = self.body_buffer.getvalue()

825

826 - def _ContinueParsing(self, buf, eof):

827 """Main function for HTTP message state machine. 828 829 @type buf: string 830 @param buf: Receive buffer 831 @type eof: bool 832 @param eof: Whether we've reached EOF on the socket 833 @rtype: string 834 @return: Updated receive buffer 835 836 """ 837 # TODO: Use offset instead of slicing when possible 838 if self.parser_status == self.PS_START_LINE: 839 # Expect start line 840 while True: 841 idx = buf.find("\r\n") 842 843 # RFC2616, section 4.1: "In the interest of robustness, servers SHOULD 844 # ignore any empty line(s) received where a Request-Line is expected. 845 # In other words, if the server is reading the protocol stream at the 846 # beginning of a message and receives a CRLF first, it should ignore 847 # the CRLF." 848 if idx == 0: 849 # TODO: Limit number of CRLFs/empty lines for safety? 850 buf = buf[2:] 851 continue 852 853 if idx > 0: 854 self.start_line_buffer = buf[:idx] 855 856 self._CheckStartLineLength(len(self.start_line_buffer)) 857 858 # Remove status line, including CRLF 859 buf = buf[idx + 2:] 860 861 self.msg.start_line = self.ParseStartLine(self.start_line_buffer) 862 863 self.parser_status = self.PS_HEADERS 864 else: 865 # Check whether incoming data is getting too large, otherwise we just 866 # fill our read buffer. 867 self._CheckStartLineLength(len(buf)) 868 869 break 870 871 # TODO: Handle messages without headers 872 if self.parser_status == self.PS_HEADERS: 873 # Wait for header end 874 idx = buf.find("\r\n\r\n") 875 if idx >= 0: 876 self.header_buffer.write(buf[:idx + 2]) 877 878 self._CheckHeaderLength(self.header_buffer.tell()) 879 880 # Remove headers, including CRLF 881 buf = buf[idx + 4:] 882 883 self._ParseHeaders() 884 885 self.parser_status = self.PS_BODY 886 else: 887 # Check whether incoming data is getting too large, otherwise we just 888 # fill our read buffer. 889 self._CheckHeaderLength(len(buf)) 890 891 if self.parser_status == self.PS_BODY: 892 # TODO: Implement max size for body_buffer 893 self.body_buffer.write(buf) 894 buf = "" 895 896 # Check whether we've read everything 897 # 898 # RFC2616, section 4.4: "When a message-body is included with a message, 899 # the transfer-length of that body is determined by one of the following 900 # [...] 5. By the server closing the connection. (Closing the connection 901 # cannot be used to indicate the end of a request body, since that would 902 # leave no possibility for the server to send back a response.)" 903 # 904 # TODO: Error when buffer length > Content-Length header 905 if (eof or 906 self.content_length is None or 907 (self.content_length is not None and 908 self.body_buffer.tell() >= self.content_length)): 909 self.parser_status = self.PS_COMPLETE 910 911 return buf

912

913 - def _CheckStartLineLength(self, length):

914 """Limits the start line buffer size. 915 916 @type length: int 917 @param length: Buffer size 918 919 """ 920 if (self.START_LINE_LENGTH_MAX is not None and 921 length > self.START_LINE_LENGTH_MAX): 922 raise HttpError("Start line longer than %d chars" % 923 self.START_LINE_LENGTH_MAX)

924

925 - def _CheckHeaderLength(self, length):

926 """Limits the header buffer size. 927 928 @type length: int 929 @param length: Buffer size 930 931 """ 932 if (self.HEADER_LENGTH_MAX is not None and 933 length > self.HEADER_LENGTH_MAX): 934 raise HttpError("Headers longer than %d chars" % self.HEADER_LENGTH_MAX)

935

936 - def ParseStartLine(self, start_line):

937 """Parses the start line of a message. 938 939 Must be overridden by subclass. 940 941 @type start_line: string 942 @param start_line: Start line string 943 944 """ 945 raise NotImplementedError()

946

947 - def _WillPeerCloseConnection(self):

948 """Evaluate whether peer will close the connection. 949 950 @rtype: bool 951 @return: Whether peer will close the connection 952 953 """ 954 # RFC2616, section 14.10: "HTTP/1.1 defines the "close" connection option 955 # for the sender to signal that the connection will be closed after 956 # completion of the response. For example, 957 # 958 # Connection: close 959 # 960 # in either the request or the response header fields indicates that the 961 # connection SHOULD NOT be considered `persistent' (section 8.1) after the 962 # current request/response is complete." 963 964 hdr_connection = self.msg.headers.get(HTTP_CONNECTION, None) 965 if hdr_connection: 966 hdr_connection = hdr_connection.lower() 967 968 # An HTTP/1.1 server is assumed to stay open unless explicitly closed. 969 if self.msg.start_line.version == HTTP_1_1: 970 return (hdr_connection and "close" in hdr_connection) 971 972 # Some HTTP/1.0 implementations have support for persistent connections, 973 # using rules different than HTTP/1.1. 974 975 # For older HTTP, Keep-Alive indicates persistent connection. 976 if self.msg.headers.get(HTTP_KEEP_ALIVE): 977 return False 978 979 # At least Akamai returns a "Connection: Keep-Alive" header, which was 980 # supposed to be sent by the client. 981 if hdr_connection and "keep-alive" in hdr_connection: 982 return False 983 984 return True

985

986 - def _ParseHeaders(self):

987 """Parses the headers. 988 989 This function also adjusts internal variables based on header values. 990 991 RFC2616, section 4.3: The presence of a message-body in a request is 992 signaled by the inclusion of a Content-Length or Transfer-Encoding header 993 field in the request's message-headers. 994 995 """ 996 # Parse headers 997 self.header_buffer.seek(0, 0) 998 self.msg.headers = mimetools.Message(self.header_buffer, 0) 999 1000 self.peer_will_close = self._WillPeerCloseConnection() 1001 1002 # Do we have a Content-Length header? 1003 hdr_content_length = self.msg.headers.get(HTTP_CONTENT_LENGTH, None) 1004 if hdr_content_length: 1005 try: 1006 self.content_length = int(hdr_content_length) 1007 except (TypeError, ValueError): 1008 self.content_length = None 1009 if self.content_length is not None and self.content_length < 0: 1010 self.content_length = None 1011 1012 # if the connection remains open and a content-length was not provided, 1013 # then assume that the connection WILL close. 1014 if self.content_length is None: 1015 self.peer_will_close = True

1016

Source Code for Package ganeti.http