code.delx.au - gnu-emacs/blob - lisp/url/url-http.el

   1 ;;; url-http.el --- HTTP retrieval routines
   2
   3 ;; Copyright (C) 1999, 2001, 2004-2016 Free Software Foundation, Inc.
   4
   5 ;; Author: Bill Perry <wmperry@gnu.org>
   6 ;; Maintainer: emacs-devel@gnu.org
   7 ;; Keywords: comm, data, processes
   8
   9 ;; This file is part of GNU Emacs.
  10 ;;
  11 ;; GNU Emacs is free software: you can redistribute it and/or modify
  12 ;; it under the terms of the GNU General Public License as published by
  13 ;; the Free Software Foundation, either version 3 of the License, or
  14 ;; (at your option) any later version.
  15
  16 ;; GNU Emacs is distributed in the hope that it will be useful,
  17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 ;; GNU General Public License for more details.
  20
  21 ;; You should have received a copy of the GNU General Public License
  22 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
  23
  24 ;;; Commentary:
  25
  26 ;;; Code:
  27
  28 (require 'cl-lib)
  29 (require 'nsm)
  30 (eval-when-compile
  31   (require 'subr-x))
  32
  33 (defvar url-callback-arguments)
  34 (defvar url-callback-function)
  35 (defvar url-current-object)
  36 (defvar url-http-after-change-function)
  37 (defvar url-http-chunked-counter)
  38 (defvar url-http-chunked-length)
  39 (defvar url-http-chunked-start)
  40 (defvar url-http-connection-opened)
  41 (defvar url-http-content-length)
  42 (defvar url-http-content-type)
  43 (defvar url-http-data)
  44 (defvar url-http-end-of-headers)
  45 (defvar url-http-extra-headers)
  46 (defvar url-http-noninteractive)
  47 (defvar url-http-method)
  48 (defvar url-http-no-retry)
  49 (defvar url-http-process)
  50 (defvar url-http-proxy)
  51 (defvar url-http-response-status)
  52 (defvar url-http-response-version)
  53 (defvar url-http-target-url)
  54 (defvar url-http-transfer-encoding)
  55 (defvar url-show-status)
  56
  57 (require 'url-gw)
  58 (require 'url-parse)
  59 (require 'url-cookie)
  60 (require 'mail-parse)
  61 (require 'url-auth)
  62 (require 'url)
  63 (autoload 'url-cache-create-filename "url-cache")
  64
  65 (defconst url-http-default-port 80 "Default HTTP port.")
  66 (defconst url-http-asynchronous-p t "HTTP retrievals are asynchronous.")
  67 (defalias 'url-http-expand-file-name 'url-default-expander)
  68
  69 (defvar url-http-real-basic-auth-storage nil)
  70 (defvar url-http-proxy-basic-auth-storage nil)
  71
  72 (defvar url-http-open-connections (make-hash-table :test 'equal
  73                                                    :size 17)
  74   "A hash table of all open network connections.")
  75
  76 (defvar url-http-version "1.1"
  77   "What version of HTTP we advertise, as a string.
  78 Valid values are 1.1 and 1.0.
  79 This is only useful when debugging the HTTP subsystem.
  80
  81 Setting this to 1.0 will tell servers not to send chunked encoding,
  82 and other HTTP/1.1 specific features.")
  83
  84 (defvar url-http-attempt-keepalives t
  85   "Whether to use a single TCP connection multiple times in HTTP.
  86 This is only useful when debugging the HTTP subsystem.  Setting to
  87 nil will explicitly close the connection to the server after every
  88 request.")
  89
  90 (defconst url-http-codes
  91   '((100 continue                        "Continue with request")
  92     (101 switching-protocols             "Switching protocols")
  93     (102 processing                      "Processing (Added by DAV)")
  94     (200 OK                              "OK")
  95     (201 created                         "Created")
  96     (202 accepted                        "Accepted")
  97     (203 non-authoritative               "Non-authoritative information")
  98     (204 no-content                      "No content")
  99     (205 reset-content                   "Reset content")
 100     (206 partial-content                 "Partial content")
 101     (207 multi-status                    "Multi-status (Added by DAV)")
 102     (300 multiple-choices                "Multiple choices")
 103     (301 moved-permanently               "Moved permanently")
 104     (302 found                           "Found")
 105     (303 see-other                       "See other")
 106     (304 not-modified                    "Not modified")
 107     (305 use-proxy                       "Use proxy")
 108     (307 temporary-redirect              "Temporary redirect")
 109     (400 bad-request                     "Bad Request")
 110     (401 unauthorized                    "Unauthorized")
 111     (402 payment-required                "Payment required")
 112     (403 forbidden                       "Forbidden")
 113     (404 not-found                       "Not found")
 114     (405 method-not-allowed              "Method not allowed")
 115     (406 not-acceptable                  "Not acceptable")
 116     (407 proxy-authentication-required   "Proxy authentication required")
 117     (408 request-timeout                 "Request time-out")
 118     (409 conflict                        "Conflict")
 119     (410 gone                            "Gone")
 120     (411 length-required                 "Length required")
 121     (412 precondition-failed             "Precondition failed")
 122     (413 request-entity-too-large        "Request entity too large")
 123     (414 request-uri-too-large           "Request-URI too large")
 124     (415 unsupported-media-type          "Unsupported media type")
 125     (416 requested-range-not-satisfiable "Requested range not satisfiable")
 126     (417 expectation-failed              "Expectation failed")
 127     (422 unprocessable-entity            "Unprocessable Entity (Added by DAV)")
 128     (423 locked                          "Locked")
 129     (424 failed-Dependency               "Failed Dependency")
 130     (500 internal-server-error           "Internal server error")
 131     (501 not-implemented                 "Not implemented")
 132     (502 bad-gateway                     "Bad gateway")
 133     (503 service-unavailable             "Service unavailable")
 134     (504 gateway-timeout                 "Gateway time-out")
 135     (505 http-version-not-supported      "HTTP version not supported")
 136     (507 insufficient-storage            "Insufficient storage"))
 137   "The HTTP return codes and their text.")
 138
 139 (defconst url-https-default-port 443 "Default HTTPS port.")
 140
 141 ;(eval-when-compile
 142 ;; These are all macros so that they are hidden from external sight
 143 ;; when the file is byte-compiled.
 144 ;;
 145 ;; This allows us to expose just the entry points we want.
 146
 147 ;; These routines will allow us to implement persistent HTTP
 148 ;; connections.
 149 (defsubst url-http-debug (&rest args)
 150   (if quit-flag
 151       (let ((proc (get-buffer-process (current-buffer))))
 152         ;; The user hit C-g, honor it!  Some things can get in an
 153         ;; incredibly tight loop (chunked encoding)
 154         (if proc
 155             (progn
 156               (set-process-sentinel proc nil)
 157               (set-process-filter proc nil)))
 158         (error "Transfer interrupted!")))
 159   (apply 'url-debug 'http args))
 160
 161 (defun url-http-mark-connection-as-busy (host port proc)
 162   (url-http-debug "Marking connection as busy: %s:%d %S" host port proc)
 163   (set-process-query-on-exit-flag proc t)
 164   (puthash (cons host port)
 165               (delq proc (gethash (cons host port) url-http-open-connections))
 166               url-http-open-connections)
 167   proc)
 168
 169 (defun url-http-mark-connection-as-free (host port proc)
 170   (url-http-debug "Marking connection as free: %s:%d %S" host port proc)
 171   (when (memq (process-status proc) '(open run connect))
 172     (set-process-buffer proc nil)
 173     (set-process-sentinel proc 'url-http-idle-sentinel)
 174     (set-process-query-on-exit-flag proc nil)
 175     (puthash (cons host port)
 176              (cons proc (gethash (cons host port) url-http-open-connections))
 177              url-http-open-connections))
 178   nil)
 179
 180 (defun url-http-find-free-connection (host port &optional gateway-method)
 181   (let ((conns (gethash (cons host port) url-http-open-connections))
 182         (connection nil))
 183     (while (and conns (not connection))
 184       (if (not (memq (process-status (car conns)) '(run open connect)))
 185           (progn
 186             (url-http-debug "Cleaning up dead process: %s:%d %S"
 187                             host port (car conns))
 188             (url-http-idle-sentinel (car conns) nil))
 189         (setq connection (car conns))
 190         (url-http-debug "Found existing connection: %s:%d %S" host port connection))
 191       (pop conns))
 192     (if connection
 193         (url-http-debug "Reusing existing connection: %s:%d" host port)
 194       (url-http-debug "Contacting host: %s:%d" host port))
 195     (url-lazy-message "Contacting host: %s:%d" host port)
 196
 197     (unless connection
 198       (let ((buf (generate-new-buffer " *url-http-temp*")))
 199         ;; `url-open-stream' needs a buffer in which to do things
 200         ;; like authentication.  But we use another buffer afterwards.
 201         (unwind-protect
 202             (let ((proc (url-open-stream host buf
 203                                          (if url-using-proxy
 204                                              (url-host url-using-proxy)
 205                                            host)
 206                                          (if url-using-proxy
 207                                              (url-port url-using-proxy)
 208                                            port)
 209                                          gateway-method)))
 210               ;; url-open-stream might return nil.
 211               (when (processp proc)
 212                 ;; Drop the temp buffer link before killing the buffer.
 213                 (set-process-buffer proc nil)
 214                 (setq connection proc)))
 215           ;; If there was an error on connect, make sure we don't
 216           ;; get queried.
 217           (when (get-buffer-process buf)
 218             (set-process-query-on-exit-flag (get-buffer-process buf) nil))
 219           (kill-buffer buf))))
 220
 221     (if connection
 222         (url-http-mark-connection-as-busy host port connection))))
 223
 224 ;; Building an HTTP request
 225 (defun url-http-user-agent-string ()
 226   (if (or (eq url-privacy-level 'paranoid)
 227           (and (listp url-privacy-level)
 228                (memq 'agent url-privacy-level)))
 229       ""
 230     (if (functionp url-user-agent)
 231         (funcall url-user-agent)
 232       url-user-agent)))
 233
 234 (defun url-http-create-request (&optional ref-url)
 235   "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
 236   (let* ((extra-headers)
 237          (request nil)
 238          (no-cache (cdr-safe (assoc "Pragma" url-http-extra-headers)))
 239          (using-proxy url-http-proxy)
 240          (proxy-auth (if (or (cdr-safe (assoc "Proxy-Authorization"
 241                                               url-http-extra-headers))
 242                              (not using-proxy))
 243                          nil
 244                        (let ((url-basic-auth-storage
 245                               'url-http-proxy-basic-auth-storage))
 246                          (url-get-authentication url-http-proxy nil 'any nil))))
 247          (real-fname (url-filename url-http-target-url))
 248          (host (url-host url-http-target-url))
 249          (auth (if (cdr-safe (assoc "Authorization" url-http-extra-headers))
 250                    nil
 251                  (url-get-authentication (or
 252                                           (and (boundp 'proxy-info)
 253                                                proxy-info)
 254                                           url-http-target-url) nil 'any nil))))
 255     (if (equal "" real-fname)
 256         (setq real-fname "/"))
 257     (setq no-cache (and no-cache (string-match "no-cache" no-cache)))
 258     (if auth
 259         (setq auth (concat "Authorization: " auth "\r\n")))
 260     (if proxy-auth
 261         (setq proxy-auth (concat "Proxy-Authorization: " proxy-auth "\r\n")))
 262
 263     ;; Protection against stupid values in the referrer
 264     (if (and ref-url (stringp ref-url) (or (string= ref-url "file:nil")
 265                                            (string= ref-url "")))
 266         (setq ref-url nil))
 267
 268     ;; We do not want to expose the referrer if the user is paranoid.
 269     (if (or (memq url-privacy-level '(low high paranoid))
 270             (and (listp url-privacy-level)
 271                  (memq 'lastloc url-privacy-level)))
 272         (setq ref-url nil))
 273
 274     ;; url-http-extra-headers contains an assoc-list of
 275     ;; header/value pairs that we need to put into the request.
 276     (setq extra-headers (mapconcat
 277                          (lambda (x)
 278                            (concat (car x) ": " (cdr x)))
 279                          url-http-extra-headers "\r\n"))
 280     (if (not (equal extra-headers ""))
 281         (setq extra-headers (concat extra-headers "\r\n")))
 282
 283     ;; This was done with a call to `format'.  Concatenating parts has
 284     ;; the advantage of keeping the parts of each header together and
 285     ;; allows us to elide null lines directly, at the cost of making
 286     ;; the layout less clear.
 287     (setq request
 288           ;; We used to concat directly, but if one of the strings happens
 289           ;; to being multibyte (even if it only contains pure ASCII) then
 290           ;; every string gets converted with `string-MAKE-multibyte' which
 291           ;; turns the 127-255 codes into things like latin-1 accented chars
 292           ;; (it would work right if it used `string-TO-multibyte' instead).
 293           ;; So to avoid the problem we force every string to be unibyte.
 294           (mapconcat
 295            ;; FIXME: Instead of `string-AS-unibyte' we'd want
 296            ;; `string-to-unibyte', so as to properly signal an error if one
 297            ;; of the strings contains a multibyte char.
 298            'string-as-unibyte
 299            (delq nil
 300             (list
 301              ;; The request
 302              (or url-http-method "GET") " "
 303              (if using-proxy (url-recreate-url url-http-target-url) real-fname)
 304              " HTTP/" url-http-version "\r\n"
 305              ;; Version of MIME we speak
 306              "MIME-Version: 1.0\r\n"
 307              ;; (maybe) Try to keep the connection open
 308              "Connection: " (if (or using-proxy
 309                                     (not url-http-attempt-keepalives))
 310                                 "close" "keep-alive") "\r\n"
 311                                 ;; HTTP extensions we support
 312              (if url-extensions-header
 313                  (format
 314                   "Extension: %s\r\n" url-extensions-header))
 315              ;; Who we want to talk to
 316              (if (/= (url-port url-http-target-url)
 317                      (url-scheme-get-property
 318                       (url-type url-http-target-url) 'default-port))
 319                  (format
 320                   "Host: %s:%d\r\n" host (url-port url-http-target-url))
 321                (format "Host: %s\r\n" host))
 322              ;; Who its from
 323              (if url-personal-mail-address
 324                  (concat
 325                   "From: " url-personal-mail-address "\r\n"))
 326              ;; Encodings we understand
 327              (if (or url-mime-encoding-string
 328                      ;; MS-Windows loads zlib dynamically, so recheck
 329                      ;; in case they made it available since
 330                      ;; initialization in url-vars.el.
 331                      (and (eq 'system-type 'windows-nt)
 332                           (fboundp 'zlib-available-p)
 333                           (zlib-available-p)
 334                           (setq url-mime-encoding-string "gzip")))
 335                  (concat
 336                   "Accept-encoding: " url-mime-encoding-string "\r\n"))
 337              (if url-mime-charset-string
 338                  (concat
 339                   "Accept-charset: " url-mime-charset-string "\r\n"))
 340              ;; Languages we understand
 341              (if url-mime-language-string
 342                  (concat
 343                   "Accept-language: " url-mime-language-string "\r\n"))
 344              ;; Types we understand
 345              "Accept: " (or url-mime-accept-string "*/*") "\r\n"
 346              ;; User agent
 347              (url-http-user-agent-string)
 348              ;; Proxy Authorization
 349              proxy-auth
 350              ;; Authorization
 351              auth
 352              ;; Cookies
 353              (when (url-use-cookies url-http-target-url)
 354                (url-cookie-generate-header-lines
 355                 host real-fname
 356                 (equal "https" (url-type url-http-target-url))))
 357              ;; If-modified-since
 358              (if (and (not no-cache)
 359                       (member url-http-method '("GET" nil)))
 360                  (let ((tm (url-is-cached url-http-target-url)))
 361                    (if tm
 362                        (concat "If-modified-since: "
 363                                (url-get-normalized-date tm) "\r\n"))))
 364              ;; Whence we came
 365              (if ref-url (concat
 366                           "Referer: " ref-url "\r\n"))
 367              extra-headers
 368              ;; Length of data
 369              (if url-http-data
 370                  (concat
 371                   "Content-length: " (number-to-string
 372                                       (length url-http-data))
 373                   "\r\n"))
 374              ;; End request
 375              "\r\n"
 376              ;; Any data
 377              url-http-data))
 378            ""))
 379     (url-http-debug "Request is: \n%s" request)
 380     request))
 381
 382 ;; Parsing routines
 383 (defun url-http-clean-headers ()
 384   "Remove trailing \r from header lines.
 385 This allows us to use `mail-fetch-field', etc.
 386 Return the number of characters removed."
 387   (let ((end (marker-position url-http-end-of-headers)))
 388     (goto-char (point-min))
 389     (while (re-search-forward "\r$" url-http-end-of-headers t)
 390       (replace-match ""))
 391     (- end url-http-end-of-headers)))
 392
 393 (defun url-http-handle-authentication (proxy)
 394   (url-http-debug "Handling %s authentication" (if proxy "proxy" "normal"))
 395   (let ((auths (or (nreverse
 396                     (mail-fetch-field
 397                      (if proxy "proxy-authenticate" "www-authenticate")
 398                      nil nil t))
 399                   '("basic")))
 400         (type nil)
 401         (url (url-recreate-url url-current-object))
 402         (auth-url (url-recreate-url
 403                    (if (and proxy (boundp 'url-http-proxy))
 404                        url-http-proxy
 405                      url-current-object)))
 406         (url-basic-auth-storage (if proxy
 407                                     ;; Cheating, but who cares? :)
 408                                     'url-http-proxy-basic-auth-storage
 409                                   'url-http-real-basic-auth-storage))
 410         auth
 411         (strength 0))
 412
 413     ;; find strongest supported auth
 414     (dolist (this-auth auths)
 415       (setq this-auth (url-eat-trailing-space
 416                        (url-strip-leading-spaces
 417                         this-auth)))
 418       (let* ((this-type
 419               (downcase (if (string-match "[ \t]" this-auth)
 420                             (substring this-auth 0 (match-beginning 0))
 421                           this-auth)))
 422              (registered (url-auth-registered this-type))
 423              (this-strength (cddr registered)))
 424         (when (and registered (> this-strength strength))
 425           (setq auth this-auth
 426                 type this-type
 427                 strength this-strength))))
 428
 429     (if (not (url-auth-registered type))
 430         (progn
 431           (widen)
 432           (goto-char (point-max))
 433           (insert "<hr>Sorry, but I do not know how to handle " (or type auth url "")
 434                   " authentication.  If you'd like to write it,"
 435                   " please use M-x report-emacs-bug RET.<hr>")
 436           ;; We used to set a `status' var (declared "special") but I can't
 437           ;; find the corresponding let-binding, so it's probably an error.
 438           ;; FIXME: Maybe it was supposed to set `success', i.e. to return t?
 439           ;; (setq status t)
 440           nil) ;; Not success yet.
 441
 442       (let* ((args (url-parse-args (subst-char-in-string ?, ?\; auth)))
 443              (auth (url-get-authentication auth-url
 444                                            (cdr-safe (assoc "realm" args))
 445                                            type t args)))
 446         (if (not auth)
 447             t                           ;Success.
 448           (push (cons (if proxy "Proxy-Authorization" "Authorization") auth)
 449                 url-http-extra-headers)
 450           (let ((url-request-method url-http-method)
 451                 (url-request-data url-http-data)
 452                 (url-request-extra-headers url-http-extra-headers))
 453             (url-retrieve-internal url url-callback-function
 454                                    url-callback-arguments))
 455           nil))))) ;; Not success yet.
 456
 457 (defun url-http-parse-response ()
 458   "Parse just the response code."
 459   (if (not url-http-end-of-headers)
 460       (error "Trying to parse HTTP response code in odd buffer: %s" (buffer-name)))
 461   (url-http-debug "url-http-parse-response called in (%s)" (buffer-name))
 462   (goto-char (point-min))
 463   (skip-chars-forward " \t\n")          ; Skip any blank crap
 464   (skip-chars-forward "HTTP/")          ; Skip HTTP Version
 465   (setq url-http-response-version
 466         (buffer-substring (point)
 467                           (progn
 468                             (skip-chars-forward "[0-9].")
 469                             (point))))
 470   (setq url-http-response-status (read (current-buffer))))
 471
 472 (defun url-http-handle-cookies ()
 473   "Handle all set-cookie / set-cookie2 headers in an HTTP response.
 474 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
 475 work correctly."
 476   (let ((cookies (nreverse (mail-fetch-field "Set-Cookie" nil nil t)))
 477         (cookies2 (nreverse (mail-fetch-field "Set-Cookie2" nil nil t))))
 478     (and cookies (url-http-debug "Found %d Set-Cookie headers" (length cookies)))
 479     (and cookies2 (url-http-debug "Found %d Set-Cookie2 headers" (length cookies2)))
 480     (while cookies
 481       (url-cookie-handle-set-cookie (pop cookies)))
 482 ;;;     (while cookies2
 483 ;;;       (url-cookie-handle-set-cookie2 (pop cookies)))
 484     )
 485   )
 486
 487 (declare-function gnutls-peer-status "gnutls.c" (proc))
 488 (declare-function gnutls-negotiate "gnutls.el")
 489
 490 (defun url-http-parse-headers ()
 491  "Parse and handle HTTP specific headers.
 492 Return t if and only if the current buffer is still active and
 493 should be shown to the user."
 494   ;; The comments after each status code handled are taken from RFC
 495   ;; 2616 (HTTP/1.1)
 496   (url-http-mark-connection-as-free (url-host url-current-object)
 497                                     (url-port url-current-object)
 498                                     url-http-process)
 499   ;; Pass the https certificate on to the caller.
 500   (when (gnutls-available-p)
 501     (let ((status (gnutls-peer-status url-http-process)))
 502       (when (or status
 503                 (plist-get (car url-callback-arguments) :peer))
 504         (setcar url-callback-arguments
 505                 (plist-put (car url-callback-arguments)
 506                            :peer status)))))
 507   (if (or (not (boundp 'url-http-end-of-headers))
 508           (not url-http-end-of-headers))
 509       (error "Trying to parse headers in odd buffer: %s" (buffer-name)))
 510   (goto-char (point-min))
 511   (url-http-debug "url-http-parse-headers called in (%s)" (buffer-name))
 512   (url-http-parse-response)
 513   (mail-narrow-to-head)
 514   ;;(narrow-to-region (point-min) url-http-end-of-headers)
 515   (let ((connection (mail-fetch-field "Connection")))
 516     ;; In HTTP 1.0, keep the connection only if there is a
 517     ;; "Connection: keep-alive" header.
 518     ;; In HTTP 1.1 (and greater), keep the connection unless there is a
 519     ;; "Connection: close" header
 520     (cond
 521      ((string= url-http-response-version "1.0")
 522       (unless (and connection
 523                    (string= (downcase connection) "keep-alive"))
 524         (delete-process url-http-process)))
 525      (t
 526       (when (and connection
 527                  (string= (downcase connection) "close"))
 528         (delete-process url-http-process)))))
 529   (let* ((buffer (current-buffer))
 530          (class (/ url-http-response-status 100))
 531          (success nil)
 532          ;; other status symbols: jewelry and luxury cars
 533          (status-symbol (cadr (assq url-http-response-status url-http-codes))))
 534     (url-http-debug "Parsed HTTP headers: class=%d status=%d"
 535                     class url-http-response-status)
 536     (when (url-use-cookies url-http-target-url)
 537       (url-http-handle-cookies))
 538
 539     (pcase class
 540       ;; Classes of response codes
 541       ;;
 542       ;; 5xx = Server Error
 543       ;; 4xx = Client Error
 544       ;; 3xx = Redirection
 545       ;; 2xx = Successful
 546       ;; 1xx = Informational
 547       (1                                ; Information messages
 548        ;; 100 = Continue with request
 549        ;; 101 = Switching protocols
 550        ;; 102 = Processing (Added by DAV)
 551        (url-mark-buffer-as-dead buffer)
 552        (error "HTTP responses in class 1xx not supported (%d)"
 553               url-http-response-status))
 554       (2                                ; Success
 555        ;; 200 Ok
 556        ;; 201 Created
 557        ;; 202 Accepted
 558        ;; 203 Non-authoritative information
 559        ;; 204 No content
 560        ;; 205 Reset content
 561        ;; 206 Partial content
 562        ;; 207 Multi-status (Added by DAV)
 563        (pcase status-symbol
 564          ((or `no-content `reset-content)
 565           ;; No new data, just stay at the same document
 566           (url-mark-buffer-as-dead buffer))
 567          (_
 568           ;; Generic success for all others.  Store in the cache, and
 569           ;; mark it as successful.
 570           (widen)
 571           (if (and url-automatic-caching (equal url-http-method "GET"))
 572               (url-store-in-cache buffer))))
 573        (setq success t))
 574       (3                                ; Redirection
 575        ;; 300 Multiple choices
 576        ;; 301 Moved permanently
 577        ;; 302 Found
 578        ;; 303 See other
 579        ;; 304 Not modified
 580        ;; 305 Use proxy
 581        ;; 307 Temporary redirect
 582        (let ((redirect-uri (or (mail-fetch-field "Location")
 583                                (mail-fetch-field "URI"))))
 584          (pcase status-symbol
 585            (`multiple-choices       ; 300
 586             ;; Quoth the spec (section 10.3.1)
 587             ;; -------------------------------
 588             ;; The requested resource corresponds to any one of a set of
 589             ;; representations, each with its own specific location and
 590             ;; agent-driven negotiation information is being provided so
 591             ;; that the user can select a preferred representation and
 592             ;; redirect its request to that location.
 593             ;; [...]
 594             ;; If the server has a preferred choice of representation, it
 595             ;; SHOULD include the specific URI for that representation in
 596             ;; the Location field; user agents MAY use the Location field
 597             ;; value for automatic redirection.
 598             ;; -------------------------------
 599             ;; We do not support agent-driven negotiation, so we just
 600             ;; redirect to the preferred URI if one is provided.
 601             nil)
 602            ((or `moved-permanently `found `temporary-redirect) ; 301 302 307
 603             ;; If the 301|302 status code is received in response to a
 604             ;; request other than GET or HEAD, the user agent MUST NOT
 605             ;; automatically redirect the request unless it can be
 606             ;; confirmed by the user, since this might change the
 607             ;; conditions under which the request was issued.
 608             (unless (member url-http-method '("HEAD" "GET"))
 609               (setq redirect-uri nil)))
 610            (`see-other                  ; 303
 611             ;; The response to the request can be found under a different
 612             ;; URI and SHOULD be retrieved using a GET method on that
 613             ;; resource.
 614             (setq url-http-method "GET"
 615                   url-http-data nil))
 616            (`not-modified               ; 304
 617             ;; The 304 response MUST NOT contain a message-body.
 618             (url-http-debug "Extracting document from cache... (%s)"
 619                             (url-cache-create-filename (url-view-url t)))
 620             (url-cache-extract (url-cache-create-filename (url-view-url t)))
 621             (setq redirect-uri nil
 622                   success t))
 623            (`use-proxy                  ; 305
 624             ;; The requested resource MUST be accessed through the
 625             ;; proxy given by the Location field.  The Location field
 626             ;; gives the URI of the proxy.  The recipient is expected
 627             ;; to repeat this single request via the proxy.  305
 628             ;; responses MUST only be generated by origin servers.
 629             (error "Redirection thru a proxy server not supported: %s"
 630                    redirect-uri))
 631            (_
 632             ;; Treat everything like '300'
 633             nil))
 634          (when redirect-uri
 635            ;; Clean off any whitespace and/or <...> cruft.
 636            (if (string-match "\\([^ \t]+\\)[ \t]" redirect-uri)
 637                (setq redirect-uri (match-string 1 redirect-uri)))
 638            (if (string-match "^<\\(.*\\)>$" redirect-uri)
 639                (setq redirect-uri (match-string 1 redirect-uri)))
 640
 641            ;; Some stupid sites (like sourceforge) send a
 642            ;; non-fully-qualified URL (ie: /), which royally confuses
 643            ;; the URL library.
 644            (if (not (string-match url-nonrelative-link redirect-uri))
 645                ;; Be careful to use the real target URL, otherwise we may
 646                ;; compute the redirection relative to the URL of the proxy.
 647                (setq redirect-uri
 648                      (url-expand-file-name redirect-uri url-http-target-url)))
 649            ;; Do not automatically include an authorization header in the
 650            ;; redirect.  If needed it will be regenerated by the relevant
 651            ;; auth scheme when the new request happens.
 652            (setq url-http-extra-headers
 653                  (cl-remove "Authorization"
 654                             url-http-extra-headers :key 'car :test 'equal))
 655            (let ((url-request-method url-http-method)
 656                  (url-request-data url-http-data)
 657                  (url-request-extra-headers url-http-extra-headers))
 658              ;; Check existing number of redirects
 659              (if (or (< url-max-redirections 0)
 660                      (and (> url-max-redirections 0)
 661                           (let ((events (car url-callback-arguments))
 662                                 (old-redirects 0))
 663                             (while events
 664                               (if (eq (car events) :redirect)
 665                                   (setq old-redirects (1+ old-redirects)))
 666                               (and (setq events (cdr events))
 667                                    (setq events (cdr events))))
 668                             (< old-redirects url-max-redirections))))
 669                  ;; url-max-redirections hasn't been reached, so go
 670                  ;; ahead and redirect.
 671                  (progn
 672                    ;; Remember that the request was redirected.
 673                    (setf (car url-callback-arguments)
 674                          (nconc (list :redirect redirect-uri)
 675                                 (car url-callback-arguments)))
 676                    ;; Put in the current buffer a forwarding pointer to the new
 677                    ;; destination buffer.
 678                    ;; FIXME: This is a hack to fix url-retrieve-synchronously
 679                    ;; without changing the API.  Instead url-retrieve should
 680                    ;; either simply not return the "destination" buffer, or it
 681                    ;; should take an optional `dest-buf' argument.
 682                    (set (make-local-variable 'url-redirect-buffer)
 683                         (url-retrieve-internal
 684                          redirect-uri url-callback-function
 685                          url-callback-arguments
 686                          (url-silent url-current-object)
 687                          (not (url-use-cookies url-current-object))))
 688                    (url-mark-buffer-as-dead buffer))
 689                ;; We hit url-max-redirections, so issue an error and
 690                ;; stop redirecting.
 691                (url-http-debug "Maximum redirections reached")
 692                (setf (car url-callback-arguments)
 693                      (nconc (list :error (list 'error 'http-redirect-limit
 694                                                redirect-uri))
 695                             (car url-callback-arguments)))
 696                (setq success t))))))
 697       (4                                ; Client error
 698        ;; 400 Bad Request
 699        ;; 401 Unauthorized
 700        ;; 402 Payment required
 701        ;; 403 Forbidden
 702        ;; 404 Not found
 703        ;; 405 Method not allowed
 704        ;; 406 Not acceptable
 705        ;; 407 Proxy authentication required
 706        ;; 408 Request time-out
 707        ;; 409 Conflict
 708        ;; 410 Gone
 709        ;; 411 Length required
 710        ;; 412 Precondition failed
 711        ;; 413 Request entity too large
 712        ;; 414 Request-URI too large
 713        ;; 415 Unsupported media type
 714        ;; 416 Requested range not satisfiable
 715        ;; 417 Expectation failed
 716        ;; 422 Unprocessable Entity (Added by DAV)
 717        ;; 423 Locked
 718        ;; 424 Failed Dependency
 719        (setq success
 720              (pcase status-symbol
 721                (`unauthorized                   ; 401
 722                 ;; The request requires user authentication.  The response
 723                 ;; MUST include a WWW-Authenticate header field containing a
 724                 ;; challenge applicable to the requested resource.  The
 725                 ;; client MAY repeat the request with a suitable
 726                 ;; Authorization header field.
 727                 (url-http-handle-authentication nil))
 728                (`payment-required              ; 402
 729                 ;; This code is reserved for future use
 730                 (url-mark-buffer-as-dead buffer)
 731                 (error "Somebody wants you to give them money"))
 732                (`forbidden                      ; 403
 733                 ;; The server understood the request, but is refusing to
 734                 ;; fulfill it.  Authorization will not help and the request
 735                 ;; SHOULD NOT be repeated.
 736                 t)
 737                (`not-found                      ; 404
 738                 ;; Not found
 739                 t)
 740                (`method-not-allowed             ; 405
 741                 ;; The method specified in the Request-Line is not allowed
 742                 ;; for the resource identified by the Request-URI.  The
 743                 ;; response MUST include an Allow header containing a list of
 744                 ;; valid methods for the requested resource.
 745                 t)
 746                (`not-acceptable         ; 406
 747                 ;; The resource identified by the request is only capable of
 748                 ;; generating response entities which have content
 749                 ;; characteristics not acceptable according to the accept
 750                 ;; headers sent in the request.
 751                 t)
 752                (`proxy-authentication-required ; 407
 753                 ;; This code is similar to 401 (Unauthorized), but indicates
 754                 ;; that the client must first authenticate itself with the
 755                 ;; proxy.  The proxy MUST return a Proxy-Authenticate header
 756                 ;; field containing a challenge applicable to the proxy for
 757                 ;; the requested resource.
 758                 (url-http-handle-authentication t))
 759                (`request-timeout                ; 408
 760                 ;; The client did not produce a request within the time that
 761                 ;; the server was prepared to wait.  The client MAY repeat
 762                 ;; the request without modifications at any later time.
 763                 t)
 764                (`conflict                       ; 409
 765                 ;; The request could not be completed due to a conflict with
 766                 ;; the current state of the resource.  This code is only
 767                 ;; allowed in situations where it is expected that the user
 768                 ;; might be able to resolve the conflict and resubmit the
 769                 ;; request.  The response body SHOULD include enough
 770                 ;; information for the user to recognize the source of the
 771                 ;; conflict.
 772                 t)
 773                (`gone                          ; 410
 774                 ;; The requested resource is no longer available at the
 775                 ;; server and no forwarding address is known.
 776                 t)
 777                (`length-required                ; 411
 778                 ;; The server refuses to accept the request without a defined
 779                 ;; Content-Length.  The client MAY repeat the request if it
 780                 ;; adds a valid Content-Length header field containing the
 781                 ;; length of the message-body in the request message.
 782                 ;;
 783                 ;; NOTE - this will never happen because
 784                 ;; `url-http-create-request' automatically calculates the
 785                 ;; content-length.
 786                 t)
 787                (`precondition-failed            ; 412
 788                 ;; The precondition given in one or more of the
 789                 ;; request-header fields evaluated to false when it was
 790                 ;; tested on the server.
 791                 t)
 792                ((or `request-entity-too-large `request-uri-too-large) ; 413 414
 793                 ;; The server is refusing to process a request because the
 794                 ;; request entity|URI is larger than the server is willing or
 795                 ;; able to process.
 796                 t)
 797                (`unsupported-media-type ; 415
 798                 ;; The server is refusing to service the request because the
 799                 ;; entity of the request is in a format not supported by the
 800                 ;; requested resource for the requested method.
 801                 t)
 802                (`requested-range-not-satisfiable ; 416
 803                 ;; A server SHOULD return a response with this status code if
 804                 ;; a request included a Range request-header field, and none
 805                 ;; of the range-specifier values in this field overlap the
 806                 ;; current extent of the selected resource, and the request
 807                 ;; did not include an If-Range request-header field.
 808                 t)
 809                (`expectation-failed             ; 417
 810                 ;; The expectation given in an Expect request-header field
 811                 ;; could not be met by this server, or, if the server is a
 812                 ;; proxy, the server has unambiguous evidence that the
 813                 ;; request could not be met by the next-hop server.
 814                 t)
 815                (_
 816                 ;; The request could not be understood by the server due to
 817                 ;; malformed syntax.  The client SHOULD NOT repeat the
 818                 ;; request without modifications.
 819                 t)))
 820        ;; Tell the callback that an error occurred, and what the
 821        ;; status code was.
 822        (when success
 823          (setf (car url-callback-arguments)
 824                (nconc (list :error (list 'error 'http url-http-response-status))
 825                       (car url-callback-arguments)))))
 826       (5
 827        ;; 500 Internal server error
 828        ;; 501 Not implemented
 829        ;; 502 Bad gateway
 830        ;; 503 Service unavailable
 831        ;; 504 Gateway time-out
 832        ;; 505 HTTP version not supported
 833        ;; 507 Insufficient storage
 834        (setq success t)
 835        (pcase url-http-response-status
 836          (`not-implemented              ; 501
 837           ;; The server does not support the functionality required to
 838           ;; fulfill the request.
 839           nil)
 840          (`bad-gateway                  ; 502
 841           ;; The server, while acting as a gateway or proxy, received
 842           ;; an invalid response from the upstream server it accessed
 843           ;; in attempting to fulfill the request.
 844           nil)
 845          (`service-unavailable          ; 503
 846           ;; The server is currently unable to handle the request due
 847           ;; to a temporary overloading or maintenance of the server.
 848           ;; The implication is that this is a temporary condition
 849           ;; which will be alleviated after some delay.  If known, the
 850           ;; length of the delay MAY be indicated in a Retry-After
 851           ;; header.  If no Retry-After is given, the client SHOULD
 852           ;; handle the response as it would for a 500 response.
 853           nil)
 854          (`gateway-timeout              ; 504
 855           ;; The server, while acting as a gateway or proxy, did not
 856           ;; receive a timely response from the upstream server
 857           ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
 858           ;; auxiliary server (e.g. DNS) it needed to access in
 859           ;; attempting to complete the request.
 860           nil)
 861          (`http-version-not-supported   ; 505
 862           ;; The server does not support, or refuses to support, the
 863           ;; HTTP protocol version that was used in the request
 864           ;; message.
 865           nil)
 866          (`insufficient-storage         ; 507 (DAV)
 867           ;; The method could not be performed on the resource
 868           ;; because the server is unable to store the representation
 869           ;; needed to successfully complete the request.  This
 870           ;; condition is considered to be temporary.  If the request
 871           ;; which received this status code was the result of a user
 872           ;; action, the request MUST NOT be repeated until it is
 873           ;; requested by a separate user action.
 874           nil))
 875        ;; Tell the callback that an error occurred, and what the
 876        ;; status code was.
 877        (when success
 878          (setf (car url-callback-arguments)
 879                (nconc (list :error (list 'error 'http url-http-response-status))
 880                       (car url-callback-arguments)))))
 881       (_
 882        (error "Unknown class of HTTP response code: %d (%d)"
 883               class url-http-response-status)))
 884     (if (not success)
 885         (url-mark-buffer-as-dead buffer)
 886       (url-handle-content-transfer-encoding))
 887     (url-http-debug "Finished parsing HTTP headers: %S" success)
 888     (widen)
 889     (goto-char (point-min))
 890     success))
 891
 892 (declare-function zlib-decompress-region "decompress.c" (start end))
 893
 894 (defun url-handle-content-transfer-encoding ()
 895   (let ((encoding (mail-fetch-field "content-encoding")))
 896     (when (and encoding
 897                (fboundp 'zlib-available-p)
 898                (zlib-available-p)
 899                (equal (downcase encoding) "gzip"))
 900       (save-restriction
 901         (widen)
 902         (goto-char (point-min))
 903         (when (search-forward "\n\n")
 904           (zlib-decompress-region (point) (point-max)))))))
 905
 906 ;; Miscellaneous
 907 (defun url-http-activate-callback ()
 908   "Activate callback specified when this buffer was created."
 909   (url-http-mark-connection-as-free (url-host url-current-object)
 910                                     (url-port url-current-object)
 911                                     url-http-process)
 912   (url-http-debug "Activating callback in buffer (%s): %S %S"
 913                   (buffer-name) url-callback-function url-callback-arguments)
 914   (apply url-callback-function url-callback-arguments))
 915
 916 ;; )
 917
 918 ;; These unfortunately cannot be macros... please ignore them!
 919 (defun url-http-idle-sentinel (proc why)
 920   "Remove (now defunct) process PROC from the list of open connections."
 921   (maphash (lambda (key val)
 922                 (if (memq proc val)
 923                     (puthash key (delq proc val) url-http-open-connections)))
 924               url-http-open-connections))
 925
 926 (defun url-http-end-of-document-sentinel (proc why)
 927   ;; Sentinel used to handle (i) terminated old HTTP/0.9 connections,
 928   ;; and (ii) closed connection due to reusing a HTTP connection which
 929   ;; we believed was still alive, but which the server closed on us.
 930   ;; We handle case (ii) by calling `url-http' again.
 931   (url-http-debug "url-http-end-of-document-sentinel in buffer (%s)"
 932                   (process-buffer proc))
 933   (url-http-idle-sentinel proc why)
 934   (when (buffer-name (process-buffer proc))
 935     (with-current-buffer (process-buffer proc)
 936       (goto-char (point-min))
 937       (cond ((not (looking-at "HTTP/"))
 938              (if url-http-no-retry
 939                  ;; HTTP/0.9 just gets passed back no matter what
 940                  (url-http-activate-callback)
 941                ;; Call `url-http' again if our connection expired.
 942                (erase-buffer)
 943                (let ((url-request-method url-http-method)
 944                      (url-request-extra-headers url-http-extra-headers)
 945                      (url-request-data url-http-data)
 946                      (url-using-proxy (url-find-proxy-for-url
 947                                        url-current-object
 948                                        (url-host url-current-object))))
 949                  (when url-using-proxy
 950                    (setq url-using-proxy
 951                          (url-generic-parse-url url-using-proxy)))
 952                  (url-http url-current-object url-callback-function
 953                            url-callback-arguments (current-buffer)))))
 954             ((url-http-parse-headers)
 955              (url-http-activate-callback))))))
 956
 957 (defun url-http-simple-after-change-function (st nd length)
 958   ;; Function used when we do NOT know how long the document is going to be
 959   ;; Just _very_ simple 'downloaded %d' type of info.
 960   (url-lazy-message "Reading %s..." (file-size-human-readable nd)))
 961
 962 (defun url-http-content-length-after-change-function (st nd length)
 963   "Function used when we DO know how long the document is going to be.
 964 More sophisticated percentage downloaded, etc.
 965 Also does minimal parsing of HTTP headers and will actually cause
 966 the callback to be triggered."
 967   (if url-http-content-type
 968       (url-display-percentage
 969        "Reading [%s]... %s of %s (%d%%)"
 970        (url-percentage (- nd url-http-end-of-headers)
 971                        url-http-content-length)
 972        url-http-content-type
 973        (file-size-human-readable (- nd url-http-end-of-headers))
 974        (file-size-human-readable url-http-content-length)
 975        (url-percentage (- nd url-http-end-of-headers)
 976                        url-http-content-length))
 977     (url-display-percentage
 978      "Reading... %s of %s (%d%%)"
 979      (url-percentage (- nd url-http-end-of-headers)
 980                      url-http-content-length)
 981      (file-size-human-readable (- nd url-http-end-of-headers))
 982      (file-size-human-readable url-http-content-length)
 983      (url-percentage (- nd url-http-end-of-headers)
 984                      url-http-content-length)))
 985
 986   (if (> (- nd url-http-end-of-headers) url-http-content-length)
 987       (progn
 988         ;; Found the end of the document!  Wheee!
 989         (url-display-percentage nil nil)
 990         (url-lazy-message "Reading... done.")
 991         (if (url-http-parse-headers)
 992             (url-http-activate-callback)))))
 993
 994 (defun url-http-chunked-encoding-after-change-function (st nd length)
 995   "Function used when dealing with chunked encoding.
 996 Cannot give a sophisticated percentage, but we need a different
 997 function to look for the special 0-length chunk that signifies
 998 the end of the document."
 999   (save-excursion
1000     (goto-char st)
1001     (let ((read-next-chunk t)
1002           (case-fold-search t)
1003           (regexp nil)
1004           (no-initial-crlf nil))
1005       ;; We need to loop thru looking for more chunks even within
1006       ;; one after-change-function call.
1007       (while read-next-chunk
1008         (setq no-initial-crlf (= 0 url-http-chunked-counter))
1009         (if url-http-content-type
1010             (url-display-percentage nil
1011              "Reading [%s]... chunk #%d"
1012              url-http-content-type url-http-chunked-counter)
1013           (url-display-percentage nil
1014            "Reading... chunk #%d"
1015            url-http-chunked-counter))
1016         (url-http-debug "Reading chunk %d (%d %d %d)"
1017                         url-http-chunked-counter st nd length)
1018         (setq regexp (if no-initial-crlf
1019                          "\\([0-9a-z]+\\).*\r?\n"
1020                        "\r?\n\\([0-9a-z]+\\).*\r?\n"))
1021
1022         (if url-http-chunked-start
1023             ;; We know how long the chunk is supposed to be, skip over
1024             ;; leading crap if possible.
1025             (if (> nd (+ url-http-chunked-start url-http-chunked-length))
1026                 (progn
1027                   (url-http-debug "Got to the end of chunk #%d!"
1028                                   url-http-chunked-counter)
1029                   (goto-char (+ url-http-chunked-start
1030                                 url-http-chunked-length)))
1031               (url-http-debug "Still need %d bytes to hit end of chunk"
1032                               (- (+ url-http-chunked-start
1033                                     url-http-chunked-length)
1034                                  nd))
1035               (setq read-next-chunk nil)))
1036         (if (not read-next-chunk)
1037             (url-http-debug "Still spinning for next chunk...")
1038           (if no-initial-crlf (skip-chars-forward "\r\n"))
1039           (if (not (looking-at regexp))
1040               (progn
1041            ;; Must not have received the entirety of the chunk header,
1042                 ;; need to spin some more.
1043                 (url-http-debug "Did not see start of chunk @ %d!" (point))
1044                 (setq read-next-chunk nil))
1045             (add-text-properties (match-beginning 0) (match-end 0)
1046                                  (list 'start-open t
1047                                        'end-open t
1048                                        'chunked-encoding t
1049                                        'face 'cursor
1050                                        'invisible t))
1051             (setq url-http-chunked-length (string-to-number (buffer-substring
1052                                                              (match-beginning 1)
1053                                                              (match-end 1))
1054                                                             16)
1055                   url-http-chunked-counter (1+ url-http-chunked-counter)
1056                   url-http-chunked-start (set-marker
1057                                           (or url-http-chunked-start
1058                                               (make-marker))
1059                                           (match-end 0)))
1060 ;           (if (not url-http-debug)
1061                 (delete-region (match-beginning 0) (match-end 0));)
1062             (url-http-debug "Saw start of chunk %d (length=%d, start=%d"
1063                             url-http-chunked-counter url-http-chunked-length
1064                             (marker-position url-http-chunked-start))
1065             (if (= 0 url-http-chunked-length)
1066                 (progn
1067                   ;; Found the end of the document!  Wheee!
1068                   (url-http-debug "Saw end of stream chunk!")
1069                   (setq read-next-chunk nil)
1070                   (url-display-percentage nil nil)
1071                   ;; Every chunk, even the last 0-length one, is
1072                   ;; terminated by CRLF.  Skip it.
1073                   (when (looking-at "\r?\n")
1074                     (url-http-debug "Removing terminator of last chunk")
1075                     (delete-region (match-beginning 0) (match-end 0)))
1076                   (if (re-search-forward "^\r?\n" nil t)
1077                       (url-http-debug "Saw end of trailers..."))
1078                   (if (url-http-parse-headers)
1079                       (url-http-activate-callback))))))))))
1080
1081 (defun url-http-wait-for-headers-change-function (st nd length)
1082   ;; This will wait for the headers to arrive and then splice in the
1083   ;; next appropriate after-change-function, etc.
1084   (url-http-debug "url-http-wait-for-headers-change-function (%s)"
1085                   (buffer-name))
1086   (let ((end-of-headers nil)
1087         (old-http nil)
1088         (process-buffer (current-buffer))
1089         (content-length nil))
1090     (when (not (bobp))
1091       (goto-char (point-min))
1092       (if (and (looking-at ".*\n")      ; have one line at least
1093                (not (looking-at "^HTTP/[1-9]\\.[0-9]")))
1094           ;; Not HTTP/x.y data, must be 0.9
1095           ;; God, I wish this could die.
1096           (setq end-of-headers t
1097                 url-http-end-of-headers 0
1098                 old-http t)
1099         ;; Blank line at end of headers.
1100         (when (re-search-forward "^\r?\n" nil t)
1101           (backward-char 1)
1102           ;; Saw the end of the headers
1103           (url-http-debug "Saw end of headers... (%s)" (buffer-name))
1104           (setq url-http-end-of-headers (set-marker (make-marker)
1105                                                     (point))
1106                 end-of-headers t)
1107           (setq nd (- nd (url-http-clean-headers)))))
1108
1109       (if (not end-of-headers)
1110           ;; Haven't seen the end of the headers yet, need to wait
1111           ;; for more data to arrive.
1112           nil
1113         (unless old-http
1114           (url-http-parse-response)
1115           (mail-narrow-to-head)
1116           (setq url-http-transfer-encoding (mail-fetch-field
1117                                             "transfer-encoding")
1118                 url-http-content-type (mail-fetch-field "content-type"))
1119           (if (mail-fetch-field "content-length")
1120               (setq url-http-content-length
1121                     (string-to-number (mail-fetch-field "content-length"))))
1122           (widen))
1123         (when url-http-transfer-encoding
1124           (setq url-http-transfer-encoding
1125                 (downcase url-http-transfer-encoding)))
1126
1127         (cond
1128          ((null url-http-response-status)
1129           ;; We got back a headerless malformed response from the
1130           ;; server.
1131           (url-http-activate-callback))
1132          ((or (= url-http-response-status 204)
1133               (= url-http-response-status 205))
1134           (url-http-debug "%d response must have headers only (%s)."
1135                           url-http-response-status (buffer-name))
1136           (when (url-http-parse-headers)
1137             (url-http-activate-callback)))
1138          ((string= "HEAD" url-http-method)
1139           ;; A HEAD request is _ALWAYS_ terminated by the header
1140           ;; information, regardless of any entity headers,
1141           ;; according to section 4.4 of the HTTP/1.1 draft.
1142           (url-http-debug "HEAD request must have headers only (%s)."
1143                           (buffer-name))
1144           (when (url-http-parse-headers)
1145             (url-http-activate-callback)))
1146          ((string= "CONNECT" url-http-method)
1147           ;; A CONNECT request is finished, but we cannot stick this
1148           ;; back on the free connection list
1149           (url-http-debug "CONNECT request must have headers only.")
1150           (when (url-http-parse-headers)
1151             (url-http-activate-callback)))
1152          ((equal url-http-response-status 304)
1153           ;; Only allowed to have a header section.  We have to handle
1154           ;; this here instead of in url-http-parse-headers because if
1155           ;; you have a cached copy of something without a known
1156           ;; content-length, and try to retrieve it from the cache, we'd
1157           ;; fall into the 'being dumb' section and wait for the
1158           ;; connection to terminate, which means we'd wait for 10
1159           ;; seconds for the keep-alives to time out on some servers.
1160           (when (url-http-parse-headers)
1161             (url-http-activate-callback)))
1162          (old-http
1163           ;; HTTP/0.9 always signaled end-of-connection by closing the
1164           ;; connection.
1165           (url-http-debug
1166            "Saw HTTP/0.9 response, connection closed means end of document.")
1167           (setq url-http-after-change-function
1168                 'url-http-simple-after-change-function))
1169          ((equal url-http-transfer-encoding "chunked")
1170           (url-http-debug "Saw chunked encoding.")
1171           (setq url-http-after-change-function
1172                 'url-http-chunked-encoding-after-change-function)
1173           (when (> nd url-http-end-of-headers)
1174             (url-http-debug
1175              "Calling initial chunked-encoding for extra data at end of headers")
1176             (url-http-chunked-encoding-after-change-function
1177              (marker-position url-http-end-of-headers) nd
1178              (- nd url-http-end-of-headers))))
1179          ((integerp url-http-content-length)
1180           (url-http-debug
1181            "Got a content-length, being smart about document end.")
1182           (setq url-http-after-change-function
1183                 'url-http-content-length-after-change-function)
1184           (cond
1185            ((= 0 url-http-content-length)
1186             ;; We got a NULL body!  Activate the callback
1187             ;; immediately!
1188             (url-http-debug
1189              "Got 0-length content-length, activating callback immediately.")
1190             (when (url-http-parse-headers)
1191               (url-http-activate-callback)))
1192            ((> nd url-http-end-of-headers)
1193             ;; Have some leftover data
1194             (url-http-debug "Calling initial content-length for extra data at end of headers")
1195             (url-http-content-length-after-change-function
1196              (marker-position url-http-end-of-headers)
1197              nd
1198              (- nd url-http-end-of-headers)))
1199            (t
1200             nil)))
1201          (t
1202           (url-http-debug "No content-length, being dumb.")
1203           (setq url-http-after-change-function
1204                 'url-http-simple-after-change-function)))))
1205     ;; We are still at the beginning of the buffer... must just be
1206     ;; waiting for a response.
1207     (url-http-debug "Spinning waiting for headers...")
1208     (when (eq process-buffer (current-buffer))
1209       (goto-char (point-max)))))
1210
1211 (defun url-http (url callback cbargs &optional retry-buffer gateway-method)
1212   "Retrieve URL via HTTP asynchronously.
1213 URL must be a parsed URL.  See `url-generic-parse-url' for details.
1214
1215 When retrieval is completed, execute the function CALLBACK, passing it
1216 an updated value of CBARGS as arguments.  The first element in CBARGS
1217 should be a plist describing what has happened so far during the
1218 request, as described in the docstring of `url-retrieve' (if in
1219 doubt, specify nil).
1220
1221 Optional arg RETRY-BUFFER, if non-nil, specifies the buffer of a
1222 previous `url-http' call, which is being re-attempted.
1223
1224 Optional arg GATEWAY-METHOD specifies the gateway to be used,
1225 overriding the value of `url-gateway-method'."
1226   (cl-check-type url vector "Need a pre-parsed URL.")
1227   (let* ((host (url-host (or url-using-proxy url)))
1228          (port (url-port (or url-using-proxy url)))
1229          (nsm-noninteractive (or url-request-noninteractive
1230                                  (and (boundp 'url-http-noninteractive)
1231                                       url-http-noninteractive)))
1232          (connection (url-http-find-free-connection (url-host url)
1233                                                     (url-port url)
1234                                                     gateway-method))
1235          (mime-accept-string url-mime-accept-string)
1236          (buffer (or retry-buffer
1237                      (generate-new-buffer
1238                       (format " *http %s:%d*" (url-host url) (url-port url))))))
1239     (if (not connection)
1240         ;; Failed to open the connection for some reason
1241         (progn
1242           (kill-buffer buffer)
1243           (setq buffer nil)
1244           (error "Could not create connection to %s:%d" (url-host url)
1245                  (url-port url)))
1246       (with-current-buffer buffer
1247         (mm-disable-multibyte)
1248         (setq url-current-object url
1249               mode-line-format "%b [%s]")
1250
1251         (dolist (var '(url-http-end-of-headers
1252                        url-http-content-type
1253                        url-http-content-length
1254                        url-http-transfer-encoding
1255                        url-http-after-change-function
1256                        url-http-response-version
1257                        url-http-response-status
1258                        url-http-chunked-length
1259                        url-http-chunked-counter
1260                        url-http-chunked-start
1261                        url-callback-function
1262                        url-callback-arguments
1263                        url-show-status
1264                        url-http-process
1265                        url-http-method
1266                        url-http-extra-headers
1267                        url-http-noninteractive
1268                        url-http-data
1269                        url-http-target-url
1270                        url-http-no-retry
1271                        url-http-connection-opened
1272                        url-mime-accept-string
1273                        url-http-proxy))
1274           (set (make-local-variable var) nil))
1275
1276         (setq url-http-method (or url-request-method "GET")
1277               url-http-extra-headers url-request-extra-headers
1278               url-http-noninteractive url-request-noninteractive
1279               url-http-data url-request-data
1280               url-http-process connection
1281               url-http-chunked-length nil
1282               url-http-chunked-start nil
1283               url-http-chunked-counter 0
1284               url-callback-function callback
1285               url-callback-arguments cbargs
1286               url-http-after-change-function 'url-http-wait-for-headers-change-function
1287               url-http-target-url url-current-object
1288               url-http-no-retry retry-buffer
1289               url-http-connection-opened nil
1290               url-mime-accept-string mime-accept-string
1291               url-http-proxy url-using-proxy)
1292
1293         (set-process-buffer connection buffer)
1294         (set-process-filter connection 'url-http-generic-filter)
1295         (pcase (process-status connection)
1296           (`connect
1297            ;; Asynchronous connection
1298            (set-process-sentinel connection 'url-http-async-sentinel))
1299           (`failed
1300            ;; Asynchronous connection failed
1301            (error "Could not create connection to %s:%d" (url-host url)
1302                   (url-port url)))
1303           (_
1304            (if (and url-http-proxy (string= "https"
1305                                             (url-type url-current-object)))
1306                (url-https-proxy-connect connection)
1307              (set-process-sentinel connection
1308                                    'url-http-end-of-document-sentinel)
1309              (process-send-string connection (url-http-create-request)))))))
1310     buffer))
1311
1312 (defun url-https-proxy-connect (connection)
1313   (setq url-http-after-change-function 'url-https-proxy-after-change-function)
1314   (process-send-string connection (format (concat "CONNECT %s:%d HTTP/1.1\r\n"
1315                                                   "Host: %s\r\n"
1316                                                   "\r\n")
1317                                           (url-host url-current-object)
1318                                           (or (url-port url-current-object)
1319                                               url-https-default-port)
1320                                           (url-host url-current-object))))
1321
1322 (defun url-https-proxy-after-change-function (st nd length)
1323   (let* ((process-buffer (current-buffer))
1324          (proc (get-buffer-process process-buffer)))
1325     (goto-char (point-min))
1326     (when (re-search-forward "^\r?\n" nil t)
1327       (backward-char 1)
1328       ;; Saw the end of the headers
1329       (setq url-http-end-of-headers (set-marker (make-marker) (point)))
1330       (url-http-parse-response)
1331       (cond
1332        ((null url-http-response-status)
1333         ;; We got back a headerless malformed response from the
1334         ;; server.
1335         (url-http-activate-callback)
1336         (error "Malformed response from proxy, fail!"))
1337        ((= url-http-response-status 200)
1338         (if (gnutls-available-p)
1339             (condition-case e
1340                 (let ((tls-connection (gnutls-negotiate
1341                                        :process proc
1342                                        :hostname (url-host url-current-object)
1343                                        :verify-error nil)))
1344                   ;; check certificate validity
1345                   (setq tls-connection
1346                         (nsm-verify-connection tls-connection
1347                                                (url-host url-current-object)
1348                                                (url-port url-current-object)))
1349                   (with-current-buffer process-buffer (erase-buffer))
1350                   (set-process-buffer tls-connection process-buffer)
1351                   (setq url-http-after-change-function
1352                         'url-http-wait-for-headers-change-function)
1353                   (set-process-filter tls-connection 'url-http-generic-filter)
1354                   (process-send-string tls-connection
1355                                        (url-http-create-request)))
1356               (gnutls-error
1357                (url-http-activate-callback)
1358                (error "gnutls-error: %s" e))
1359               (error
1360                (url-http-activate-callback)
1361                (error "error: %s" e)))
1362           (error "error: gnutls support needed!")))
1363        (t
1364         (url-http-activate-callback)
1365         (message "error response: %d" url-http-response-status))))))
1366
1367 (defun url-http-async-sentinel (proc why)
1368   ;; We are performing an asynchronous connection, and a status change
1369   ;; has occurred.
1370   (when (buffer-name (process-buffer proc))
1371     (with-current-buffer (process-buffer proc)
1372       (cond
1373        (url-http-connection-opened
1374         (setq url-http-no-retry t)
1375         (url-http-end-of-document-sentinel proc why))
1376        ((string= (substring why 0 4) "open")
1377         (setq url-http-connection-opened t)
1378         (if (and url-http-proxy (string= "https" (url-type url-current-object)))
1379             (url-https-proxy-connect proc)
1380           (condition-case error
1381               (process-send-string proc (url-http-create-request))
1382             (file-error
1383              (setq url-http-connection-opened nil)
1384              (message "HTTP error: %s" error)))))
1385        (t
1386         (setf (car url-callback-arguments)
1387               (nconc (list :error (list 'error 'connection-failed why
1388                                         :host (url-host (or url-http-proxy url-current-object))
1389                                         :service (url-port (or url-http-proxy url-current-object))))
1390                      (car url-callback-arguments)))
1391         (url-http-activate-callback))))))
1392
1393 ;; Since Emacs 19/20 does not allow you to change the
1394 ;; `after-change-functions' hook in the midst of running them, we fake
1395 ;; an after change by hooking into the process filter and inserting
1396 ;; the data ourselves.  This is slightly less efficient, but there
1397 ;; were tons of weird ways the after-change code was biting us in the
1398 ;; shorts.
1399 ;; FIXME this can probably be simplified since the above is no longer true.
1400 (defun url-http-generic-filter (proc data)
1401   ;; Sometimes we get a zero-length data chunk after the process has
1402   ;; been changed to 'free', which means it has no buffer associated
1403   ;; with it.  Do nothing if there is no buffer, or 0 length data.
1404   (and (process-buffer proc)
1405        (/= (length data) 0)
1406        (with-current-buffer (process-buffer proc)
1407          (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1408          (funcall url-http-after-change-function
1409                   (point-max)
1410                   (progn
1411                     (goto-char (point-max))
1412                     (insert data)
1413                     (point-max))
1414                   (length data)))))
1415
1416 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1417 ;;; file-name-handler stuff from here on out
1418 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1419 (defalias 'url-http-symbol-value-in-buffer
1420   (if (fboundp 'symbol-value-in-buffer)
1421       'symbol-value-in-buffer
1422     (lambda (symbol buffer &optional unbound-value)
1423       "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1424       (with-current-buffer buffer
1425         (if (not (boundp symbol))
1426             unbound-value
1427           (symbol-value symbol))))))
1428
1429 (defun url-http-head (url)
1430   (let ((url-request-method "HEAD")
1431         (url-request-data nil))
1432     (url-retrieve-synchronously url)))
1433
1434 (defun url-http-file-exists-p (url)
1435   (let ((buffer (url-http-head url)))
1436     (when buffer
1437       (let ((status (url-http-symbol-value-in-buffer 'url-http-response-status
1438                                                      buffer 500)))
1439         (prog1
1440             (and (integerp status)
1441                  (>= status 200) (< status 300))
1442           (kill-buffer buffer))))))
1443
1444 (defalias 'url-http-file-readable-p 'url-http-file-exists-p)
1445
1446 (defun url-http-head-file-attributes (url &optional id-format)
1447   (let ((buffer (url-http-head url)))
1448     (when buffer
1449       (prog1
1450           (list
1451            nil                          ;dir / link / normal file
1452            1                            ;number of links to file.
1453            0 0                          ;uid ; gid
1454            nil nil nil                  ;atime ; mtime ; ctime
1455            (url-http-symbol-value-in-buffer 'url-http-content-length
1456                                             buffer -1)
1457            (eval-when-compile (make-string 10 ?-))
1458            nil nil nil)          ;whether gid would change ; inode ; device.
1459         (kill-buffer buffer)))))
1460
1461 (declare-function url-dav-file-attributes "url-dav" (url &optional id-format))
1462
1463 (defun url-http-file-attributes (url &optional id-format)
1464   (if (url-dav-supported-p url)
1465       (url-dav-file-attributes url id-format)
1466     (url-http-head-file-attributes url id-format)))
1467
1468 (defun url-http-options (url)
1469   "Return a property list describing options available for URL.
1470 This list is retrieved using the `OPTIONS' HTTP method.
1471
1472 Property list members:
1473
1474 methods
1475   A list of symbols specifying what HTTP methods the resource
1476   supports.
1477
1478 dav
1479   A list of numbers specifying what DAV protocol/schema versions are
1480   supported.
1481
1482 dasl
1483   A list of supported DASL search types supported (string form)
1484
1485 ranges
1486   A list of the units available for use in partial document fetches.
1487
1488 p3p
1489   The `Platform For Privacy Protection' description for the resource.
1490   Currently this is just the raw header contents.  This is likely to
1491   change once P3P is formally supported by the URL package or
1492   Emacs/W3."
1493   (let* ((url-request-method "OPTIONS")
1494          (url-request-data nil)
1495          (buffer (url-retrieve-synchronously url))
1496          (header nil)
1497          (options nil))
1498     (when (and buffer (= 2 (/ (url-http-symbol-value-in-buffer
1499                                'url-http-response-status buffer 0) 100)))
1500       ;; Only parse the options if we got a 2xx response code!
1501       (with-current-buffer buffer
1502         (save-restriction
1503           (save-match-data
1504             (mail-narrow-to-head)
1505
1506             ;; Figure out what methods are supported.
1507             (when (setq header (mail-fetch-field "allow"))
1508               (setq options (plist-put
1509                              options 'methods
1510                              (mapcar 'intern (split-string header "[ ,]+")))))
1511
1512             ;; Check for DAV
1513             (when (setq header (mail-fetch-field "dav"))
1514               (setq options (plist-put
1515                              options 'dav
1516                              (delq 0
1517                                    (mapcar 'string-to-number
1518                                            (split-string header "[, ]+"))))))
1519
1520             ;; Now for DASL
1521             (when (setq header (mail-fetch-field "dasl"))
1522               (setq options (plist-put
1523                              options 'dasl
1524                              (split-string header "[, ]+"))))
1525
1526             ;; P3P - should get more detailed here.  FIXME
1527             (when (setq header (mail-fetch-field "p3p"))
1528               (setq options (plist-put options 'p3p header)))
1529
1530             ;; Check for whether they accept byte-range requests.
1531             (when (setq header (mail-fetch-field "accept-ranges"))
1532               (setq options (plist-put
1533                              options 'ranges
1534                              (delq 'none
1535                                    (mapcar 'intern
1536                                            (split-string header "[, ]+"))))))
1537             ))))
1538     (if buffer (kill-buffer buffer))
1539     options))
1540
1541 ;; HTTPS.  This used to be in url-https.el, but that file collides
1542 ;; with url-http.el on systems with 8-character file names.
1543 (require 'tls)
1544
1545 (defconst url-https-asynchronous-p t "HTTPS retrievals are asynchronous.")
1546
1547 ;; FIXME what is the point of this alias being an autoload?
1548 ;; Trying to use it will not cause url-http to be loaded,
1549 ;; since the full alias just gets dumped into loaddefs.el.
1550
1551 ;;;###autoload (autoload 'url-default-expander "url-expand")
1552 ;;;###autoload
1553 (defalias 'url-https-expand-file-name 'url-default-expander)
1554
1555 (defmacro url-https-create-secure-wrapper (method args)
1556   `(defun ,(intern (format (if method "url-https-%s" "url-https") method)) ,args
1557     ,(format "HTTPS wrapper around `%s' call." (or method "url-http"))
1558     (,(intern (format (if method "url-http-%s" "url-http") method))
1559      ,@(remove '&rest (remove '&optional (append args (if method nil '(nil 'tls))))))))
1560
1561 ;;;###autoload (autoload 'url-https "url-http")
1562 (url-https-create-secure-wrapper nil (url callback cbargs))
1563 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1564 (url-https-create-secure-wrapper file-exists-p (url))
1565 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1566 (url-https-create-secure-wrapper file-readable-p (url))
1567 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1568 (url-https-create-secure-wrapper file-attributes (url &optional id-format))
1569
1570 (provide 'url-http)
1571
1572 ;;; url-http.el ends here