ganeti.http

1 # 2 # 3 4 # Copyright (C) 2007, 2008 Google Inc. 5 # 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or 9 # (at your option) any later version. 10 # 11 # This program is distributed in the hope that it will be useful, but 12 # WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 # General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 19 # 02110-1301, USA. 20 21 """HTTP module. 22 23 """ 24 25 import logging 26 import mimetools 27 import OpenSSL 28 import select 29 import socket 30 import errno 31 32 from cStringIO import StringIO 33 34 from ganeti import constants 35 from ganeti import serializer 36 from ganeti import utils 37 38 39 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION 40 41 HTTP_OK = 200 42 HTTP_NO_CONTENT = 204 43 HTTP_NOT_MODIFIED = 304 44 45 HTTP_0_9 = "HTTP/0.9" 46 HTTP_1_0 = "HTTP/1.0" 47 HTTP_1_1 = "HTTP/1.1" 48 49 HTTP_GET = "GET" 50 HTTP_HEAD = "HEAD" 51 HTTP_POST = "POST" 52 HTTP_PUT = "PUT" 53 HTTP_DELETE = "DELETE" 54 55 HTTP_ETAG = "ETag" 56 HTTP_HOST = "Host" 57 HTTP_SERVER = "Server" 58 HTTP_DATE = "Date" 59 HTTP_USER_AGENT = "User-Agent" 60 HTTP_CONTENT_TYPE = "Content-Type" 61 HTTP_CONTENT_LENGTH = "Content-Length" 62 HTTP_CONNECTION = "Connection" 63 HTTP_KEEP_ALIVE = "Keep-Alive" 64 HTTP_WWW_AUTHENTICATE = "WWW-Authenticate" 65 HTTP_AUTHORIZATION = "Authorization" 66 HTTP_AUTHENTICATION_INFO = "Authentication-Info" 67 HTTP_ALLOW = "Allow" 68 69 _SSL_UNEXPECTED_EOF = "Unexpected EOF" 70 71 # Socket operations 72 (SOCKOP_SEND, 73 SOCKOP_RECV, 74 SOCKOP_SHUTDOWN, 75 SOCKOP_HANDSHAKE) = range(4) 76 77 # send/receive quantum 78 SOCK_BUF_SIZE = 32768 79 80

81 -class HttpError(Exception):

82 """Internal exception for HTTP errors. 83 84 This should only be used for internal error reporting. 85 86 """

87 88

89 -class HttpConnectionClosed(Exception):

90 """Internal exception for a closed connection. 91 92 This should only be used for internal error reporting. Only use 93 it if there's no other way to report this condition. 94 95 """

96 97

98 -class HttpSessionHandshakeUnexpectedEOF(HttpError):

99 """Internal exception for errors during SSL handshake. 100 101 This should only be used for internal error reporting. 102 103 """

104 105

106 -class HttpSocketTimeout(Exception):

107 """Internal exception for socket timeouts. 108 109 This should only be used for internal error reporting. 110 111 """

112 113

114 -class HttpException(Exception):

115 code = None 116 message = None 117

118 - def __init__(self, message=None, headers=None):

119 Exception.__init__(self) 120 self.message = message 121 self.headers = headers

122 123

124 -class HttpBadRequest(HttpException):

125 """400 Bad Request 126 127 RFC2616, 10.4.1: The request could not be understood by the server 128 due to malformed syntax. The client SHOULD NOT repeat the request 129 without modifications. 130 131 """ 132 code = 400

133 134

135 -class HttpUnauthorized(HttpException):

136 """401 Unauthorized 137 138 RFC2616, section 10.4.2: The request requires user 139 authentication. The response MUST include a WWW-Authenticate header 140 field (section 14.47) containing a challenge applicable to the 141 requested resource. 142 143 """ 144 code = 401

145 146

147 -class HttpForbidden(HttpException):

148 """403 Forbidden 149 150 RFC2616, 10.4.4: The server understood the request, but is refusing 151 to fulfill it. Authorization will not help and the request SHOULD 152 NOT be repeated. 153 154 """ 155 code = 403

156 157

158 -class HttpNotFound(HttpException):

159 """404 Not Found 160 161 RFC2616, 10.4.5: The server has not found anything matching the 162 Request-URI. No indication is given of whether the condition is 163 temporary or permanent. 164 165 """ 166 code = 404

167 168

169 -class HttpMethodNotAllowed(HttpException):

170 """405 Method Not Allowed 171 172 RFC2616, 10.4.6: The method specified in the Request-Line is not 173 allowed for the resource identified by the Request-URI. The response 174 MUST include an Allow header containing a list of valid methods for 175 the requested resource. 176 177 """ 178 code = 405

179 180

181 -class HttpRequestTimeout(HttpException):

182 """408 Request Timeout 183 184 RFC2616, 10.4.9: The client did not produce a request within the 185 time that the server was prepared to wait. The client MAY repeat the 186 request without modifications at any later time. 187 188 """ 189 code = 408

190 191

192 -class HttpConflict(HttpException):

193 """409 Conflict 194 195 RFC2616, 10.4.10: The request could not be completed due to a 196 conflict with the current state of the resource. This code is only 197 allowed in situations where it is expected that the user might be 198 able to resolve the conflict and resubmit the request. 199 200 """ 201 code = 409

202 203

204 -class HttpGone(HttpException):

205 """410 Gone 206 207 RFC2616, 10.4.11: The requested resource is no longer available at 208 the server and no forwarding address is known. This condition is 209 expected to be considered permanent. 210 211 """ 212 code = 410

213 214

215 -class HttpLengthRequired(HttpException):

216 """411 Length Required 217 218 RFC2616, 10.4.12: The server refuses to accept the request without a 219 defined Content-Length. The client MAY repeat the request if it adds 220 a valid Content-Length header field containing the length of the 221 message-body in the request message. 222 223 """ 224 code = 411

225 226

227 -class HttpPreconditionFailed(HttpException):

228 """412 Precondition Failed 229 230 RFC2616, 10.4.13: The precondition given in one or more of the 231 request-header fields evaluated to false when it was tested on the 232 server. 233 234 """ 235 code = 412

236 237

238 -class HttpInternalServerError(HttpException):

239 """500 Internal Server Error 240 241 RFC2616, 10.5.1: The server encountered an unexpected condition 242 which prevented it from fulfilling the request. 243 244 """ 245 code = 500

246 247

248 -class HttpNotImplemented(HttpException):

249 """501 Not Implemented 250 251 RFC2616, 10.5.2: The server does not support the functionality 252 required to fulfill the request. 253 254 """ 255 code = 501

256 257

258 -class HttpBadGateway(HttpException):

259 """502 Bad Gateway 260 261 RFC2616, 10.5.3: The server, while acting as a gateway or proxy, 262 received an invalid response from the upstream server it accessed in 263 attempting to fulfill the request. 264 265 """ 266 code = 502

267 268

269 -class HttpServiceUnavailable(HttpException):

270 """503 Service Unavailable 271 272 RFC2616, 10.5.4: The server is currently unable to handle the 273 request due to a temporary overloading or maintenance of the server. 274 275 """ 276 code = 503

277 278

279 -class HttpGatewayTimeout(HttpException):

280 """504 Gateway Timeout 281 282 RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did 283 not receive a timely response from the upstream server specified by 284 the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server 285 (e.g. DNS) it needed to access in attempting to complete the 286 request. 287 288 """ 289 code = 504

290 291

292 -class HttpVersionNotSupported(HttpException):

293 """505 HTTP Version Not Supported 294 295 RFC2616, 10.5.6: The server does not support, or refuses to support, 296 the HTTP protocol version that was used in the request message. 297 298 """ 299 code = 505

300 301

302 -class HttpJsonConverter:

303 CONTENT_TYPE = "application/json" 304

305 - def Encode(self, data):

306 return serializer.DumpJson(data)

307

308 - def Decode(self, data):

309 return serializer.LoadJson(data)

310 311

312 -def WaitForSocketCondition(sock, event, timeout):

313 """Waits for a condition to occur on the socket. 314 315 @type sock: socket 316 @param sock: Wait for events on this socket 317 @type event: int 318 @param event: ORed condition (see select module) 319 @type timeout: float or None 320 @param timeout: Timeout in seconds 321 @rtype: int or None 322 @return: None for timeout, otherwise occured conditions 323 324 """ 325 check = (event | select.POLLPRI | 326 select.POLLNVAL | select.POLLHUP | select.POLLERR) 327 328 if timeout is not None: 329 # Poller object expects milliseconds 330 timeout *= 1000 331 332 poller = select.poll() 333 poller.register(sock, event) 334 try: 335 while True: 336 # TODO: If the main thread receives a signal and we have no timeout, we 337 # could wait forever. This should check a global "quit" flag or 338 # something every so often. 339 io_events = poller.poll(timeout) 340 if not io_events: 341 # Timeout 342 return None 343 for (evfd, evcond) in io_events: 344 if evcond & check: 345 return evcond 346 finally: 347 poller.unregister(sock)

348 349

350 -def SocketOperation(sock, op, arg1, timeout):

351 """Wrapper around socket functions. 352 353 This function abstracts error handling for socket operations, especially 354 for the complicated interaction with OpenSSL. 355 356 @type sock: socket 357 @param sock: Socket for the operation 358 @type op: int 359 @param op: Operation to execute (SOCKOP_* constants) 360 @type arg1: any 361 @param arg1: Parameter for function (if needed) 362 @type timeout: None or float 363 @param timeout: Timeout in seconds or None 364 @return: Return value of socket function 365 366 """ 367 # TODO: event_poll/event_check/override 368 if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE): 369 event_poll = select.POLLOUT 370 371 elif op == SOCKOP_RECV: 372 event_poll = select.POLLIN 373 374 elif op == SOCKOP_SHUTDOWN: 375 event_poll = None 376 377 # The timeout is only used when OpenSSL requests polling for a condition. 378 # It is not advisable to have no timeout for shutdown. 379 assert timeout 380 381 else: 382 raise AssertionError("Invalid socket operation") 383 384 # Handshake is only supported by SSL sockets 385 if (op == SOCKOP_HANDSHAKE and 386 not isinstance(sock, OpenSSL.SSL.ConnectionType)): 387 return 388 389 # No override by default 390 event_override = 0 391 392 while True: 393 # Poll only for certain operations and when asked for by an override 394 if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE): 395 if event_override: 396 wait_for_event = event_override 397 else: 398 wait_for_event = event_poll 399 400 event = WaitForSocketCondition(sock, wait_for_event, timeout) 401 if event is None: 402 raise HttpSocketTimeout() 403 404 if (op == SOCKOP_RECV and 405 event & (select.POLLNVAL | select.POLLHUP | select.POLLERR)): 406 return "" 407 408 if not event & wait_for_event: 409 continue 410 411 # Reset override 412 event_override = 0 413 414 try: 415 try: 416 if op == SOCKOP_SEND: 417 return sock.send(arg1) 418 419 elif op == SOCKOP_RECV: 420 return sock.recv(arg1) 421 422 elif op == SOCKOP_SHUTDOWN: 423 if isinstance(sock, OpenSSL.SSL.ConnectionType): 424 # PyOpenSSL's shutdown() doesn't take arguments 425 return sock.shutdown() 426 else: 427 return sock.shutdown(arg1) 428 429 elif op == SOCKOP_HANDSHAKE: 430 return sock.do_handshake() 431 432 except OpenSSL.SSL.WantWriteError: 433 # OpenSSL wants to write, poll for POLLOUT 434 event_override = select.POLLOUT 435 continue 436 437 except OpenSSL.SSL.WantReadError: 438 # OpenSSL wants to read, poll for POLLIN 439 event_override = select.POLLIN | select.POLLPRI 440 continue 441 442 except OpenSSL.SSL.WantX509LookupError: 443 continue 444 445 except OpenSSL.SSL.ZeroReturnError, err: 446 # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only 447 # occurs if a closure alert has occurred in the protocol, i.e. the 448 # connection has been closed cleanly. Note that this does not 449 # necessarily mean that the transport layer (e.g. a socket) has been 450 # closed. 451 if op == SOCKOP_SEND: 452 # Can happen during a renegotiation 453 raise HttpConnectionClosed(err.args) 454 elif op == SOCKOP_RECV: 455 return "" 456 457 # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN 458 raise socket.error(err.args) 459 460 except OpenSSL.SSL.SysCallError, err: 461 if op == SOCKOP_SEND: 462 # arg1 is the data when writing 463 if err.args and err.args[0] == -1 and arg1 == "": 464 # errors when writing empty strings are expected 465 # and can be ignored 466 return 0 467 468 if err.args == (-1, _SSL_UNEXPECTED_EOF): 469 if op == SOCKOP_RECV: 470 return "" 471 elif op == SOCKOP_HANDSHAKE: 472 # Can happen if peer disconnects directly after the connection is 473 # opened. 474 raise HttpSessionHandshakeUnexpectedEOF(err.args) 475 476 raise socket.error(err.args) 477 478 except OpenSSL.SSL.Error, err: 479 raise socket.error(err.args) 480 481 except socket.error, err: 482 if err.args and err.args[0] == errno.EAGAIN: 483 # Ignore EAGAIN 484 continue 485 486 raise

487 488

489 -def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force):

490 """Closes the connection. 491 492 @type sock: socket 493 @param sock: Socket to be shut down 494 @type close_timeout: float 495 @param close_timeout: How long to wait for the peer to close 496 the connection 497 @type write_timeout: float 498 @param write_timeout: Write timeout for shutdown 499 @type msgreader: http.HttpMessageReader 500 @param msgreader: Request message reader, used to determine whether 501 peer should close connection 502 @type force: bool 503 @param force: Whether to forcibly close the connection without 504 waiting for peer 505 506 """ 507 #print msgreader.peer_will_close, force 508 if msgreader and msgreader.peer_will_close and not force: 509 # Wait for peer to close 510 try: 511 # Check whether it's actually closed 512 if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout): 513 return 514 except (socket.error, HttpError, HttpSocketTimeout): 515 # Ignore errors at this stage 516 pass 517 518 # Close the connection from our side 519 try: 520 # We don't care about the return value, see NOTES in SSL_shutdown(3). 521 SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR, 522 write_timeout) 523 except HttpSocketTimeout: 524 raise HttpError("Timeout while shutting down connection") 525 except socket.error, err: 526 # Ignore ENOTCONN 527 if not (err.args and err.args[0] == errno.ENOTCONN): 528 raise HttpError("Error while shutting down connection: %s" % err)

529 530

531 -def Handshake(sock, write_timeout):

532 """Shakes peer's hands. 533 534 @type sock: socket 535 @param sock: Socket to be shut down 536 @type write_timeout: float 537 @param write_timeout: Write timeout for handshake 538 539 """ 540 try: 541 return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout) 542 except HttpSocketTimeout: 543 raise HttpError("Timeout during SSL handshake") 544 except socket.error, err: 545 raise HttpError("Error in SSL handshake: %s" % err)

546 547

548 -class HttpSslParams(object):

549 """Data class for SSL key and certificate. 550 551 """

552 - def __init__(self, ssl_key_path, ssl_cert_path):

553 """Initializes this class. 554 555 @type ssl_key_path: string 556 @param ssl_key_path: Path to file containing SSL key in PEM format 557 @type ssl_cert_path: string 558 @param ssl_cert_path: Path to file containing SSL certificate 559 in PEM format 560 561 """ 562 self.ssl_key_pem = utils.ReadFile(ssl_key_path) 563 self.ssl_cert_pem = utils.ReadFile(ssl_cert_path)

564

565 - def GetKey(self):

566 return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, 567 self.ssl_key_pem)

568

569 - def GetCertificate(self):

570 return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, 571 self.ssl_cert_pem)

572 573

574 -class HttpBase(object):

575 """Base class for HTTP server and client. 576 577 """

578 - def __init__(self):

579 self.using_ssl = None 580 self._ssl_params = None 581 self._ssl_key = None 582 self._ssl_cert = None

583

584 - def _CreateSocket(self, ssl_params, ssl_verify_peer):

585 """Creates a TCP socket and initializes SSL if needed. 586 587 @type ssl_params: HttpSslParams 588 @param ssl_params: SSL key and certificate 589 @type ssl_verify_peer: bool 590 @param ssl_verify_peer: Whether to require client certificate 591 and compare it with our certificate 592 593 """ 594 self._ssl_params = ssl_params 595 596 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 597 598 # Should we enable SSL? 599 self.using_ssl = ssl_params is not None 600 601 if not self.using_ssl: 602 return sock 603 604 self._ssl_key = ssl_params.GetKey() 605 self._ssl_cert = ssl_params.GetCertificate() 606 607 ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) 608 ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2) 609 610 ctx.use_privatekey(self._ssl_key) 611 ctx.use_certificate(self._ssl_cert) 612 ctx.check_privatekey() 613 614 if ssl_verify_peer: 615 ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | 616 OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, 617 self._SSLVerifyCallback) 618 619 return OpenSSL.SSL.Connection(ctx, sock)

620

621 - def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok):

622 """Verify the certificate provided by the peer 623 624 We only compare fingerprints. The client must use the same certificate as 625 we do on our side. 626 627 """ 628 assert self._ssl_params, "SSL not initialized" 629 630 return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and 631 self._ssl_cert.digest("md5") == cert.digest("md5"))

632 633

634 -class HttpMessage(object):

635 """Data structure for HTTP message. 636 637 """

638 - def __init__(self):

639 self.start_line = None 640 self.headers = None 641 self.body = None 642 self.decoded_body = None

643 644

645 -class HttpClientToServerStartLine(object):

646 """Data structure for HTTP request start line. 647 648 """

649 - def __init__(self, method, path, version):

650 self.method = method 651 self.path = path 652 self.version = version

653

654 - def __str__(self):

655 return "%s %s %s" % (self.method, self.path, self.version)

656 657

658 -class HttpServerToClientStartLine(object):

659 """Data structure for HTTP response start line. 660 661 """

662 - def __init__(self, version, code, reason):

663 self.version = version 664 self.code = code 665 self.reason = reason

666

667 - def __str__(self):

668 return "%s %s %s" % (self.version, self.code, self.reason)

669 670

671 -class HttpMessageWriter(object):

672 """Writes an HTTP message to a socket. 673 674 """

675 - def __init__(self, sock, msg, write_timeout):

676 """Initializes this class and writes an HTTP message to a socket. 677 678 @type sock: socket 679 @param sock: Socket to be written to 680 @type msg: http.HttpMessage 681 @param msg: HTTP message to be written 682 @type write_timeout: float 683 @param write_timeout: Write timeout for socket 684 685 """ 686 self._msg = msg 687 688 self._PrepareMessage() 689 690 buf = self._FormatMessage() 691 692 pos = 0 693 end = len(buf) 694 while pos < end: 695 # Send only SOCK_BUF_SIZE bytes at a time 696 data = buf[pos:(pos + SOCK_BUF_SIZE)] 697 698 sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout) 699 700 # Remove sent bytes 701 pos += sent 702 703 assert pos == end, "Message wasn't sent completely"

704

705 - def _PrepareMessage(self):

706 """Prepares the HTTP message by setting mandatory headers. 707 708 """ 709 # RFC2616, section 4.3: "The presence of a message-body in a request is 710 # signaled by the inclusion of a Content-Length or Transfer-Encoding header 711 # field in the request's message-headers." 712 if self._msg.body: 713 self._msg.headers[HTTP_CONTENT_LENGTH] = len(self._msg.body)

714

715 - def _FormatMessage(self):

716 """Serializes the HTTP message into a string. 717 718 """ 719 buf = StringIO() 720 721 # Add start line 722 buf.write(str(self._msg.start_line)) 723 buf.write("\r\n") 724 725 # Add headers 726 if self._msg.start_line.version != HTTP_0_9: 727 for name, value in self._msg.headers.iteritems(): 728 buf.write("%s: %s\r\n" % (name, value)) 729 730 buf.write("\r\n") 731 732 # Add message body if needed 733 if self.HasMessageBody(): 734 buf.write(self._msg.body) 735 736 elif self._msg.body: 737 logging.warning("Ignoring message body") 738 739 return buf.getvalue()

740

741 - def HasMessageBody(self):

742 """Checks whether the HTTP message contains a body. 743 744 Can be overridden by subclasses. 745 746 """ 747 return bool(self._msg.body)

748 749

750 -class HttpMessageReader(object):

751 """Reads HTTP message from socket. 752 753 """ 754 # Length limits 755 START_LINE_LENGTH_MAX = None 756 HEADER_LENGTH_MAX = None 757 758 # Parser state machine 759 PS_START_LINE = "start-line" 760 PS_HEADERS = "headers" 761 PS_BODY = "entity-body" 762 PS_COMPLETE = "complete" 763

764 - def __init__(self, sock, msg, read_timeout):

765 """Reads an HTTP message from a socket. 766 767 @type sock: socket 768 @param sock: Socket to be read from 769 @type msg: http.HttpMessage 770 @param msg: Object for the read message 771 @type read_timeout: float 772 @param read_timeout: Read timeout for socket 773 774 """ 775 self.sock = sock 776 self.msg = msg 777 778 self.start_line_buffer = None 779 self.header_buffer = StringIO() 780 self.body_buffer = StringIO() 781 self.parser_status = self.PS_START_LINE 782 self.content_length = None 783 self.peer_will_close = None 784 785 buf = "" 786 eof = False 787 while self.parser_status != self.PS_COMPLETE: 788 # TODO: Don't read more than necessary (Content-Length), otherwise 789 # data might be lost and/or an error could occur 790 data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout) 791 792 if data: 793 buf += data 794 else: 795 eof = True 796 797 # Do some parsing and error checking while more data arrives 798 buf = self._ContinueParsing(buf, eof) 799 800 # Must be done only after the buffer has been evaluated 801 # TODO: Connection-length < len(data read) and connection closed 802 if (eof and 803 self.parser_status in (self.PS_START_LINE, 804 self.PS_HEADERS)): 805 raise HttpError("Connection closed prematurely") 806 807 # Parse rest 808 buf = self._ContinueParsing(buf, True) 809 810 assert self.parser_status == self.PS_COMPLETE 811 assert not buf, "Parser didn't read full response" 812 813 msg.body = self.body_buffer.getvalue() 814 815 # TODO: Content-type, error handling 816 if msg.body: 817 msg.decoded_body = HttpJsonConverter().Decode(msg.body) 818 else: 819 msg.decoded_body = None 820 821 if msg.decoded_body: 822 logging.debug("Message body: %s", msg.decoded_body)

823

824 - def _ContinueParsing(self, buf, eof):

825 """Main function for HTTP message state machine. 826 827 @type buf: string 828 @param buf: Receive buffer 829 @type eof: bool 830 @param eof: Whether we've reached EOF on the socket 831 @rtype: string 832 @return: Updated receive buffer 833 834 """ 835 # TODO: Use offset instead of slicing when possible 836 if self.parser_status == self.PS_START_LINE: 837 # Expect start line 838 while True: 839 idx = buf.find("\r\n") 840 841 # RFC2616, section 4.1: "In the interest of robustness, servers SHOULD 842 # ignore any empty line(s) received where a Request-Line is expected. 843 # In other words, if the server is reading the protocol stream at the 844 # beginning of a message and receives a CRLF first, it should ignore 845 # the CRLF." 846 if idx == 0: 847 # TODO: Limit number of CRLFs/empty lines for safety? 848 buf = buf[:2] 849 continue 850 851 if idx > 0: 852 self.start_line_buffer = buf[:idx] 853 854 self._CheckStartLineLength(len(self.start_line_buffer)) 855 856 # Remove status line, including CRLF 857 buf = buf[idx + 2:] 858 859 self.msg.start_line = self.ParseStartLine(self.start_line_buffer) 860 861 self.parser_status = self.PS_HEADERS 862 else: 863 # Check whether incoming data is getting too large, otherwise we just 864 # fill our read buffer. 865 self._CheckStartLineLength(len(buf)) 866 867 break 868 869 # TODO: Handle messages without headers 870 if self.parser_status == self.PS_HEADERS: 871 # Wait for header end 872 idx = buf.find("\r\n\r\n") 873 if idx >= 0: 874 self.header_buffer.write(buf[:idx + 2]) 875 876 self._CheckHeaderLength(self.header_buffer.tell()) 877 878 # Remove headers, including CRLF 879 buf = buf[idx + 4:] 880 881 self._ParseHeaders() 882 883 self.parser_status = self.PS_BODY 884 else: 885 # Check whether incoming data is getting too large, otherwise we just 886 # fill our read buffer. 887 self._CheckHeaderLength(len(buf)) 888 889 if self.parser_status == self.PS_BODY: 890 # TODO: Implement max size for body_buffer 891 self.body_buffer.write(buf) 892 buf = "" 893 894 # Check whether we've read everything 895 # 896 # RFC2616, section 4.4: "When a message-body is included with a message, 897 # the transfer-length of that body is determined by one of the following 898 # [...] 5. By the server closing the connection. (Closing the connection 899 # cannot be used to indicate the end of a request body, since that would 900 # leave no possibility for the server to send back a response.)" 901 # 902 # TODO: Error when buffer length > Content-Length header 903 if (eof or 904 self.content_length is None or 905 (self.content_length is not None and 906 self.body_buffer.tell() >= self.content_length)): 907 self.parser_status = self.PS_COMPLETE 908 909 return buf

910

911 - def _CheckStartLineLength(self, length):

912 """Limits the start line buffer size. 913 914 @type length: int 915 @param length: Buffer size 916 917 """ 918 if (self.START_LINE_LENGTH_MAX is not None and 919 length > self.START_LINE_LENGTH_MAX): 920 raise HttpError("Start line longer than %d chars" % 921 self.START_LINE_LENGTH_MAX)

922

923 - def _CheckHeaderLength(self, length):

924 """Limits the header buffer size. 925 926 @type length: int 927 @param length: Buffer size 928 929 """ 930 if (self.HEADER_LENGTH_MAX is not None and 931 length > self.HEADER_LENGTH_MAX): 932 raise HttpError("Headers longer than %d chars" % self.HEADER_LENGTH_MAX)

933

934 - def ParseStartLine(self, start_line):

935 """Parses the start line of a message. 936 937 Must be overridden by subclass. 938 939 @type start_line: string 940 @param start_line: Start line string 941 942 """ 943 raise NotImplementedError()

944

945 - def _WillPeerCloseConnection(self):

946 """Evaluate whether peer will close the connection. 947 948 @rtype: bool 949 @return: Whether peer will close the connection 950 951 """ 952 # RFC2616, section 14.10: "HTTP/1.1 defines the "close" connection option 953 # for the sender to signal that the connection will be closed after 954 # completion of the response. For example, 955 # 956 # Connection: close 957 # 958 # in either the request or the response header fields indicates that the 959 # connection SHOULD NOT be considered `persistent' (section 8.1) after the 960 # current request/response is complete." 961 962 hdr_connection = self.msg.headers.get(HTTP_CONNECTION, None) 963 if hdr_connection: 964 hdr_connection = hdr_connection.lower() 965 966 # An HTTP/1.1 server is assumed to stay open unless explicitly closed. 967 if self.msg.start_line.version == HTTP_1_1: 968 return (hdr_connection and "close" in hdr_connection) 969 970 # Some HTTP/1.0 implementations have support for persistent connections, 971 # using rules different than HTTP/1.1. 972 973 # For older HTTP, Keep-Alive indicates persistent connection. 974 if self.msg.headers.get(HTTP_KEEP_ALIVE): 975 return False 976 977 # At least Akamai returns a "Connection: Keep-Alive" header, which was 978 # supposed to be sent by the client. 979 if hdr_connection and "keep-alive" in hdr_connection: 980 return False 981 982 return True

983

984 - def _ParseHeaders(self):

985 """Parses the headers. 986 987 This function also adjusts internal variables based on header values. 988 989 RFC2616, section 4.3: The presence of a message-body in a request is 990 signaled by the inclusion of a Content-Length or Transfer-Encoding header 991 field in the request's message-headers. 992 993 """ 994 # Parse headers 995 self.header_buffer.seek(0, 0) 996 self.msg.headers = mimetools.Message(self.header_buffer, 0) 997 998 self.peer_will_close = self._WillPeerCloseConnection() 999 1000 # Do we have a Content-Length header? 1001 hdr_content_length = self.msg.headers.get(HTTP_CONTENT_LENGTH, None) 1002 if hdr_content_length: 1003 try: 1004 self.content_length = int(hdr_content_length) 1005 except (TypeError, ValueError): 1006 self.content_length = None 1007 if self.content_length is not None and self.content_length < 0: 1008 self.content_length = None 1009 1010 # if the connection remains open and a content-length was not provided, 1011 # then assume that the connection WILL close. 1012 if self.content_length is None: 1013 self.peer_will_close = True

1014

Source Code for Package ganeti.http