bug-wget
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Bug-wget] gethttp cleanup


From: Giuseppe Scrivano
Subject: [Bug-wget] gethttp cleanup
Date: Mon, 16 Mar 2015 02:18:09 +0100

Hi,

spring is coming and it is time for some cleanup.

I have refactored a bit gethttp and moved some code to other functions.
It is mostly mechanical code movements, and I hope to do some more work
on this (if other people want to jump in, you are welcome!) and simplify
gethttp even more.  It is still a huge beast, but should be weaker after
the series below.

I will wait for comments before pushing these patches.

Miquel, I know you were working on gethttp to increase the header size
to 8kb, if you already have done something, I promise I will rebase your
patch on top of my series :-)

If some GSoC students want to have something more or less easy to warm
up, here is an idea: move gethttp to use a single cleanup label at the
end of the function.  Avoid early exits from gethttp and replace them
with "goto cleanup", all the cleanups must be done after this point
(this means local variables must be properly initialized as they could
be cleaned-up before they were supposed to be).

Cheers,
Giuseppe

>From f61530279b5d59d0ae3a954420ac6c64288b98ff Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Sun, 15 Mar 2015 22:59:29 +0100
Subject: [PATCH 1/6] Factor out some initialization code for gethttp

* src/http.c (gethttp): Move some initialization code in...
(initialize_request): ... a new function.
---
 src/http.c | 241 +++++++++++++++++++++++++++++++++----------------------------
 1 file changed, 130 insertions(+), 111 deletions(-)

diff --git a/src/http.c b/src/http.c
index b7020ef..1da7871 100644
--- a/src/http.c
+++ b/src/http.c
@@ -1646,108 +1646,13 @@ read_response_body (struct http_stat *hs, int sock, 
FILE *fp, wgint contlen,
 } while (0)
 #endif /* def __VMS [else] */
 
-/* Retrieve a document through HTTP protocol.  It recognizes status
-   code, and correctly handles redirections.  It closes the network
-   socket.  If it receives an error from the functions below it, it
-   will print it if there is enough information to do so (almost
-   always), returning the error to the caller (i.e. http_loop).
-
-   Various HTTP parameters are stored to hs.
-
-   If PROXY is non-NULL, the connection will be made to the proxy
-   server, and u->url will be requested.  */
-static uerr_t
-gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
-         struct iri *iri, int count)
+static struct request *
+initialize_request (struct url *u, struct http_stat *hs, int *dt, struct url 
*proxy,
+                    bool inhibit_keep_alive, bool *basic_auth_finished,
+                    wgint *body_data_size, char **user, char **passwd, uerr_t 
*ret)
 {
-  struct request *req;
-
-  char *type;
-  char *user, *passwd;
-  char *proxyauth;
-  int statcode;
-  int write_error;
-  wgint contlen, contrange;
-  struct url *conn;
-  FILE *fp;
-  int err;
-
-  int sock = -1;
-
-  /* Set to 1 when the authorization has already been sent and should
-     not be tried again. */
-  bool auth_finished = false;
-
-  /* Set to 1 when just globally-set Basic authorization has been sent;
-   * should prevent further Basic negotiations, but not other
-   * mechanisms. */
-  bool basic_auth_finished = false;
-
-  /* Whether NTLM authentication is used for this request. */
-  bool ntlm_seen = false;
-
-  /* Whether our connection to the remote host is through SSL.  */
-  bool using_ssl = false;
-
-  /* Whether a HEAD request will be issued (as opposed to GET or
-     POST). */
   bool head_only = !!(*dt & HEAD_ONLY);
-
-  char *head;
-  struct response *resp;
-  char hdrval[512];
-  char *message;
-
-  /* Declare WARC variables. */
-  bool warc_enabled = (opt.warc_filename != NULL);
-  FILE *warc_tmp = NULL;
-  char warc_timestamp_str [21];
-  char warc_request_uuid [48];
-  ip_address *warc_ip = NULL;
-  off_t warc_payload_offset = -1;
-
-  /* Whether this connection will be kept alive after the HTTP request
-     is done. */
-  bool keep_alive;
-
-  /* Is the server using the chunked transfer encoding?  */
-  bool chunked_transfer_encoding = false;
-
-  /* Whether keep-alive should be inhibited.  */
-  bool inhibit_keep_alive =
-    !opt.http_keep_alive || opt.ignore_length;
-
-  /* Headers sent when using POST. */
-  wgint body_data_size = 0;
-
-  bool host_lookup_failed = false;
-
-#ifdef HAVE_SSL
-  if (u->scheme == SCHEME_HTTPS)
-    {
-      /* Initialize the SSL context.  After this has once been done,
-         it becomes a no-op.  */
-      if (!ssl_init ())
-        {
-          scheme_disable (SCHEME_HTTPS);
-          logprintf (LOG_NOTQUIET,
-                     _("Disabling SSL due to encountered errors.\n"));
-          return SSLINITFAILED;
-        }
-    }
-#endif /* HAVE_SSL */
-
-  /* Initialize certain elements of struct http_stat.  */
-  hs->len = 0;
-  hs->contlen = -1;
-  hs->res = -1;
-  hs->rderrmsg = NULL;
-  hs->newloc = NULL;
-  xfree(hs->remote_time);
-  hs->error = NULL;
-  hs->message = NULL;
-
-  conn = u;
+  struct request *req;
 
   /* Prepare the request to send. */
   {
@@ -1793,20 +1698,20 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, 
struct url *proxy,
   request_set_header (req, "Accept-Encoding", "identity", rel_none);
 
   /* Find the username and password for authentication. */
-  user = u->user;
-  passwd = u->passwd;
+  *user = u->user;
+  *passwd = u->passwd;
   search_netrc (u->host, (const char **)&user, (const char **)&passwd, 0);
-  user = user ? user : (opt.http_user ? opt.http_user : opt.user);
-  passwd = passwd ? passwd : (opt.http_passwd ? opt.http_passwd : opt.passwd);
+  *user = *user ? *user : (opt.http_user ? opt.http_user : opt.user);
+  *passwd = *passwd ? *passwd : (opt.http_passwd ? opt.http_passwd : 
opt.passwd);
 
   /* We only do "site-wide" authentication with "global" user/password
    * values unless --auth-no-challange has been requested; URL user/password
    * info overrides. */
-  if (user && passwd && (!u->user || opt.auth_without_challenge))
+  if (user && *passwd && (!u->user || opt.auth_without_challenge))
     {
       /* If this is a host for which we've already received a Basic
        * challenge, we'll go ahead and send Basic authentication creds. */
-      basic_auth_finished = maybe_send_basic_creds(u->host, user, passwd, req);
+      *basic_auth_finished = maybe_send_basic_creds(u->host, *user, *passwd, 
req);
     }
 
   /* Generate the Host header, HOST:PORT.  Take into account that:
@@ -1848,20 +1753,21 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, 
struct url *proxy,
                               "application/x-www-form-urlencoded", rel_none);
 
           if (opt.body_data)
-            body_data_size = strlen (opt.body_data);
+            *body_data_size = strlen (opt.body_data);
           else
             {
-              body_data_size = file_size (opt.body_file);
-              if (body_data_size == -1)
+              *body_data_size = file_size (opt.body_file);
+              if (*body_data_size == -1)
                 {
                   logprintf (LOG_NOTQUIET, _("BODY data file %s missing: 
%s\n"),
                              quote (opt.body_file), strerror (errno));
                   request_free (req);
-                  return FILEBADFILE;
+                  *ret = FILEBADFILE;
+                  return NULL;
                 }
             }
           request_set_header (req, "Content-Length",
-                              xstrdup (number_to_static_string 
(body_data_size)),
+                              xstrdup (number_to_static_string 
(*body_data_size)),
                               rel_value);
         }
       else if (c_strcasecmp (opt.method, "post") == 0
@@ -1869,7 +1775,120 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, 
struct url *proxy,
                || c_strcasecmp (opt.method, "patch") == 0)
         request_set_header (req, "Content-Length", "0", rel_none);
     }
+  return req;
+}
 
+/* Retrieve a document through HTTP protocol.  It recognizes status
+   code, and correctly handles redirections.  It closes the network
+   socket.  If it receives an error from the functions below it, it
+   will print it if there is enough information to do so (almost
+   always), returning the error to the caller (i.e. http_loop).
+
+   Various HTTP parameters are stored to hs.
+
+   If PROXY is non-NULL, the connection will be made to the proxy
+   server, and u->url will be requested.  */
+static uerr_t
+gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
+         struct iri *iri, int count)
+{
+  struct request *req;
+
+  char *type;
+  char *user, *passwd;
+  char *proxyauth;
+  int statcode;
+  int write_error;
+  wgint contlen, contrange;
+  struct url *conn;
+  FILE *fp;
+  int err;
+
+  int sock = -1;
+
+  /* Set to 1 when the authorization has already been sent and should
+     not be tried again. */
+  bool auth_finished = false;
+
+  /* Set to 1 when just globally-set Basic authorization has been sent;
+   * should prevent further Basic negotiations, but not other
+   * mechanisms. */
+  bool basic_auth_finished = false;
+
+  /* Whether NTLM authentication is used for this request. */
+  bool ntlm_seen = false;
+
+  /* Whether our connection to the remote host is through SSL.  */
+  bool using_ssl = false;
+
+  /* Whether a HEAD request will be issued (as opposed to GET or
+     POST). */
+  bool head_only = !!(*dt & HEAD_ONLY);
+
+  char *head;
+  struct response *resp;
+  char hdrval[512];
+  char *message;
+
+  /* Declare WARC variables. */
+  bool warc_enabled = (opt.warc_filename != NULL);
+  FILE *warc_tmp = NULL;
+  char warc_timestamp_str [21];
+  char warc_request_uuid [48];
+  ip_address *warc_ip = NULL;
+  off_t warc_payload_offset = -1;
+
+  /* Whether this connection will be kept alive after the HTTP request
+     is done. */
+  bool keep_alive;
+
+  /* Is the server using the chunked transfer encoding?  */
+  bool chunked_transfer_encoding = false;
+
+  /* Whether keep-alive should be inhibited.  */
+  bool inhibit_keep_alive =
+    !opt.http_keep_alive || opt.ignore_length;
+
+  /* Headers sent when using POST. */
+  wgint body_data_size = 0;
+
+  bool host_lookup_failed = false;
+
+#ifdef HAVE_SSL
+  if (u->scheme == SCHEME_HTTPS)
+    {
+      /* Initialize the SSL context.  After this has once been done,
+         it becomes a no-op.  */
+      if (!ssl_init ())
+        {
+          scheme_disable (SCHEME_HTTPS);
+          logprintf (LOG_NOTQUIET,
+                     _("Disabling SSL due to encountered errors.\n"));
+          return SSLINITFAILED;
+        }
+    }
+#endif /* HAVE_SSL */
+
+  /* Initialize certain elements of struct http_stat.  */
+  hs->len = 0;
+  hs->contlen = -1;
+  hs->res = -1;
+  hs->rderrmsg = NULL;
+  hs->newloc = NULL;
+  xfree(hs->remote_time);
+  hs->error = NULL;
+  hs->message = NULL;
+
+  conn = u;
+
+  {
+    uerr_t ret;
+    req = initialize_request (u, hs, dt, proxy, inhibit_keep_alive,
+                              &basic_auth_finished, &body_data_size,
+                              &user, &passwd, &ret);
+    if (req == NULL)
+      return ret;
+  }
  retry_with_auth:
   /* We need to come back here when the initial attempt to retrieve
      without authorization header fails.  (Expected to happen at least
-- 
2.1.0

>From 0c4a400201123c6359b54273e96d5bb2d860da57 Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Sun, 15 Mar 2015 23:03:44 +0100
Subject: [PATCH 2/6] Factor out some proxy initialization code for gethttp

* src/http.c (gethttp): Move some initialization code in...
(initialize_proxy_configuration): ... a new function.
---
 src/http.c | 63 ++++++++++++++++++++++++++++++++------------------------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/src/http.c b/src/http.c
index 1da7871..decab66 100644
--- a/src/http.c
+++ b/src/http.c
@@ -1778,6 +1778,38 @@ initialize_request (struct url *u, struct http_stat *hs, 
int *dt, struct url *pr
   return req;
 }
 
+static void
+initialize_proxy_configuration (struct url *u, struct request *req,
+                                struct url *proxy, char **proxyauth)
+{
+  char *proxy_user, *proxy_passwd;
+  /* For normal username and password, URL components override
+     command-line/wgetrc parameters.  With proxy
+     authentication, it's the reverse, because proxy URLs are
+     normally the "permanent" ones, so command-line args
+     should take precedence.  */
+  if (opt.proxy_user && opt.proxy_passwd)
+    {
+      proxy_user = opt.proxy_user;
+      proxy_passwd = opt.proxy_passwd;
+    }
+  else
+    {
+      proxy_user = proxy->user;
+      proxy_passwd = proxy->passwd;
+    }
+  /* #### This does not appear right.  Can't the proxy request,
+     say, `Digest' authentication?  */
+  if (proxy_user && proxy_passwd)
+    *proxyauth = basic_authentication_encode (proxy_user, proxy_passwd);
+
+  /* Proxy authorization over SSL is handled below. */
+#ifdef HAVE_SSL
+  if (u->scheme != SCHEME_HTTPS)
+#endif
+    request_set_header (req, "Proxy-Authorization", *proxyauth, rel_value);
+}
+
 /* Retrieve a document through HTTP protocol.  It recognizes status
    code, and correctly handles redirections.  It closes the network
    socket.  If it receives an error from the functions below it, it
@@ -1917,38 +1949,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, 
struct url *proxy,
   proxyauth = NULL;
   if (proxy)
     {
-      char *proxy_user, *proxy_passwd;
-      /* For normal username and password, URL components override
-         command-line/wgetrc parameters.  With proxy
-         authentication, it's the reverse, because proxy URLs are
-         normally the "permanent" ones, so command-line args
-         should take precedence.  */
-      if (opt.proxy_user && opt.proxy_passwd)
-        {
-          proxy_user = opt.proxy_user;
-          proxy_passwd = opt.proxy_passwd;
-        }
-      else
-        {
-          proxy_user = proxy->user;
-          proxy_passwd = proxy->passwd;
-        }
-      /* #### This does not appear right.  Can't the proxy request,
-         say, `Digest' authentication?  */
-      if (proxy_user && proxy_passwd)
-        proxyauth = basic_authentication_encode (proxy_user, proxy_passwd);
-
-      /* If we're using a proxy, we will be connecting to the proxy
-         server.  */
       conn = proxy;
-
-      /* Proxy authorization over SSL is handled below. */
-#ifdef HAVE_SSL
-      if (u->scheme != SCHEME_HTTPS)
-#endif
-        request_set_header (req, "Proxy-Authorization", proxyauth, rel_value);
+      initialize_proxy_configuration (u, req, proxy, &proxyauth);
     }
-
   keep_alive = true;
 
   /* Establish the connection.  */
-- 
2.1.0

>From ad1df69a9874e15d549ea6d3de9ce238ea5e2bed Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Sun, 15 Mar 2015 23:34:49 +0100
Subject: [PATCH 3/6] Factor out some connection initialization code for
 gethttp

* src/http.c (gethttp): Move some initialization code in...
(establish_connection): ... a new function.
---
 src/http.c | 327 +++++++++++++++++++++++++++++++++----------------------------
 1 file changed, 178 insertions(+), 149 deletions(-)

diff --git a/src/http.c b/src/http.c
index decab66..2daa64c 100644
--- a/src/http.c
+++ b/src/http.c
@@ -1810,155 +1810,23 @@ initialize_proxy_configuration (struct url *u, struct 
request *req,
     request_set_header (req, "Proxy-Authorization", *proxyauth, rel_value);
 }
 
-/* Retrieve a document through HTTP protocol.  It recognizes status
-   code, and correctly handles redirections.  It closes the network
-   socket.  If it receives an error from the functions below it, it
-   will print it if there is enough information to do so (almost
-   always), returning the error to the caller (i.e. http_loop).
-
-   Various HTTP parameters are stored to hs.
-
-   If PROXY is non-NULL, the connection will be made to the proxy
-   server, and u->url will be requested.  */
 static uerr_t
-gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
-         struct iri *iri, int count)
+establish_connection (struct url *u, struct url **conn_ref,
+                      struct http_stat *hs, struct url *proxy,
+                      char **proxyauth,
+                      struct request **req_ref, bool *using_ssl,
+                      bool inhibit_keep_alive,
+                      int *sock_ref)
 {
-  struct request *req;
-
-  char *type;
-  char *user, *passwd;
-  char *proxyauth;
-  int statcode;
-  int write_error;
-  wgint contlen, contrange;
-  struct url *conn;
-  FILE *fp;
-  int err;
-
-  int sock = -1;
-
-  /* Set to 1 when the authorization has already been sent and should
-     not be tried again. */
-  bool auth_finished = false;
-
-  /* Set to 1 when just globally-set Basic authorization has been sent;
-   * should prevent further Basic negotiations, but not other
-   * mechanisms. */
-  bool basic_auth_finished = false;
-
-  /* Whether NTLM authentication is used for this request. */
-  bool ntlm_seen = false;
-
-  /* Whether our connection to the remote host is through SSL.  */
-  bool using_ssl = false;
-
-  /* Whether a HEAD request will be issued (as opposed to GET or
-     POST). */
-  bool head_only = !!(*dt & HEAD_ONLY);
-
-  char *head;
-  struct response *resp;
-  char hdrval[512];
-  char *message;
-
-  /* Declare WARC variables. */
-  bool warc_enabled = (opt.warc_filename != NULL);
-  FILE *warc_tmp = NULL;
-  char warc_timestamp_str [21];
-  char warc_request_uuid [48];
-  ip_address *warc_ip = NULL;
-  off_t warc_payload_offset = -1;
-
-  /* Whether this connection will be kept alive after the HTTP request
-     is done. */
-  bool keep_alive;
-
-  /* Is the server using the chunked transfer encoding?  */
-  bool chunked_transfer_encoding = false;
-
-  /* Whether keep-alive should be inhibited.  */
-  bool inhibit_keep_alive =
-    !opt.http_keep_alive || opt.ignore_length;
-
-  /* Headers sent when using POST. */
-  wgint body_data_size = 0;
-
   bool host_lookup_failed = false;
+  int sock = *sock_ref;
+  struct request *req = *req_ref;
+  struct url *conn = *conn_ref;
+  struct response *resp;
+  int write_error;
+  int statcode;
 
-#ifdef HAVE_SSL
-  if (u->scheme == SCHEME_HTTPS)
-    {
-      /* Initialize the SSL context.  After this has once been done,
-         it becomes a no-op.  */
-      if (!ssl_init ())
-        {
-          scheme_disable (SCHEME_HTTPS);
-          logprintf (LOG_NOTQUIET,
-                     _("Disabling SSL due to encountered errors.\n"));
-          return SSLINITFAILED;
-        }
-    }
-#endif /* HAVE_SSL */
-
-  /* Initialize certain elements of struct http_stat.  */
-  hs->len = 0;
-  hs->contlen = -1;
-  hs->res = -1;
-  hs->rderrmsg = NULL;
-  hs->newloc = NULL;
-  xfree(hs->remote_time);
-  hs->error = NULL;
-  hs->message = NULL;
-
-  conn = u;
-
-  {
-    uerr_t ret;
-    req = initialize_request (u, hs, dt, proxy, inhibit_keep_alive,
-                              &basic_auth_finished, &body_data_size,
-                              &user, &passwd, &ret);
-    if (req == NULL)
-      return ret;
-  }
- retry_with_auth:
-  /* We need to come back here when the initial attempt to retrieve
-     without authorization header fails.  (Expected to happen at least
-     for the Digest authorization scheme.)  */
-
-  if (opt.cookies)
-    request_set_header (req, "Cookie",
-                        cookie_header (wget_cookie_jar,
-                                       u->host, u->port, u->path,
-#ifdef HAVE_SSL
-                                       u->scheme == SCHEME_HTTPS
-#else
-                                       0
-#endif
-                                       ),
-                        rel_value);
-
-  /* Add the user headers. */
-  if (opt.user_headers)
-    {
-      int i;
-      for (i = 0; opt.user_headers[i]; i++)
-        request_set_user_header (req, opt.user_headers[i]);
-    }
-
-  proxyauth = NULL;
-  if (proxy)
-    {
-      conn = proxy;
-      initialize_proxy_configuration (u, req, proxy, &proxyauth);
-    }
-  keep_alive = true;
-
-  /* Establish the connection.  */
-
-  if (inhibit_keep_alive)
-    keep_alive = false;
-  else
+  if (! inhibit_keep_alive)
     {
       /* Look for a persistent connection to target host, unless a
          proxy is used.  The exception is when SSL is in use, in which
@@ -1980,7 +1848,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, 
struct url *proxy,
         {
           int family = socket_family (pconn.socket, ENDPOINT_PEER);
           sock = pconn.socket;
-          using_ssl = pconn.ssl;
+          *using_ssl = pconn.ssl;
 #if ENABLE_IPV6
           if (family == AF_INET6)
              logprintf (LOG_VERBOSE, _("Reusing existing connection to 
[%s]:%d.\n"),
@@ -2030,6 +1898,8 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, 
struct url *proxy,
 #ifdef HAVE_SSL
       if (proxy && u->scheme == SCHEME_HTTPS)
         {
+          char *head;
+          char *message;
           /* When requesting SSL URLs through proxies, use the
              CONNECT method to request passthrough.  */
           struct request *connreq = request_new ("CONNECT",
@@ -2038,11 +1908,11 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, 
struct url *proxy,
           if (proxyauth)
             {
               request_set_header (connreq, "Proxy-Authorization",
-                                  proxyauth, rel_value);
+                                  *proxyauth, rel_value);
               /* Now that PROXYAUTH is part of the CONNECT request,
                  zero it out so we don't send proxy authorization with
                  the regular request below.  */
-              proxyauth = NULL;
+              *proxyauth = NULL;
             }
           request_set_header (connreq, "Host",
                               aprintf ("%s:%d", u->host, u->port),
@@ -2122,10 +1992,169 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, 
struct url *proxy,
               request_free (req);
               return VERIFCERTERR;
             }
-          using_ssl = true;
+          *using_ssl = true;
         }
 #endif /* HAVE_SSL */
     }
+  *conn_ref = conn;
+  *req_ref = req;
+  *sock_ref = sock;
+  return RETROK;
+}
+
+/* Retrieve a document through HTTP protocol.  It recognizes status
+   code, and correctly handles redirections.  It closes the network
+   socket.  If it receives an error from the functions below it, it
+   will print it if there is enough information to do so (almost
+   always), returning the error to the caller (i.e. http_loop).
+
+   Various HTTP parameters are stored to hs.
+
+   If PROXY is non-NULL, the connection will be made to the proxy
+   server, and u->url will be requested.  */
+static uerr_t
+gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
+         struct iri *iri, int count)
+{
+  struct request *req;
+
+  char *type;
+  char *user, *passwd;
+  char *proxyauth;
+  int statcode;
+  int write_error;
+  wgint contlen, contrange;
+  struct url *conn;
+  FILE *fp;
+  int err;
+
+  int sock = -1;
+
+  /* Set to 1 when the authorization has already been sent and should
+     not be tried again. */
+  bool auth_finished = false;
+
+  /* Set to 1 when just globally-set Basic authorization has been sent;
+   * should prevent further Basic negotiations, but not other
+   * mechanisms. */
+  bool basic_auth_finished = false;
+
+  /* Whether NTLM authentication is used for this request. */
+  bool ntlm_seen = false;
+
+  /* Whether our connection to the remote host is through SSL.  */
+  bool using_ssl = false;
+
+  /* Whether a HEAD request will be issued (as opposed to GET or
+     POST). */
+  bool head_only = !!(*dt & HEAD_ONLY);
+
+  char *head;
+  struct response *resp;
+  char hdrval[512];
+  char *message;
+
+  /* Declare WARC variables. */
+  bool warc_enabled = (opt.warc_filename != NULL);
+  FILE *warc_tmp = NULL;
+  char warc_timestamp_str [21];
+  char warc_request_uuid [48];
+  ip_address *warc_ip = NULL;
+  off_t warc_payload_offset = -1;
+
+  /* Whether this connection will be kept alive after the HTTP request
+     is done. */
+  bool keep_alive;
+
+  /* Is the server using the chunked transfer encoding?  */
+  bool chunked_transfer_encoding = false;
+
+  /* Whether keep-alive should be inhibited.  */
+  bool inhibit_keep_alive =
+    !opt.http_keep_alive || opt.ignore_length;
+
+  /* Headers sent when using POST. */
+  wgint body_data_size = 0;
+
+#ifdef HAVE_SSL
+  if (u->scheme == SCHEME_HTTPS)
+    {
+      /* Initialize the SSL context.  After this has once been done,
+         it becomes a no-op.  */
+      if (!ssl_init ())
+        {
+          scheme_disable (SCHEME_HTTPS);
+          logprintf (LOG_NOTQUIET,
+                     _("Disabling SSL due to encountered errors.\n"));
+          return SSLINITFAILED;
+        }
+    }
+#endif /* HAVE_SSL */
+
+  /* Initialize certain elements of struct http_stat.  */
+  hs->len = 0;
+  hs->contlen = -1;
+  hs->res = -1;
+  hs->rderrmsg = NULL;
+  hs->newloc = NULL;
+  xfree(hs->remote_time);
+  hs->error = NULL;
+  hs->message = NULL;
+
+  conn = u;
+
+  {
+    uerr_t ret;
+    req = initialize_request (u, hs, dt, proxy, inhibit_keep_alive,
+                              &basic_auth_finished, &body_data_size,
+                              &user, &passwd, &ret);
+    if (req == NULL)
+      return ret;
+  }
+ retry_with_auth:
+  /* We need to come back here when the initial attempt to retrieve
+     without authorization header fails.  (Expected to happen at least
+     for the Digest authorization scheme.)  */
+
+  if (opt.cookies)
+    request_set_header (req, "Cookie",
+                        cookie_header (wget_cookie_jar,
+                                       u->host, u->port, u->path,
+#ifdef HAVE_SSL
+                                       u->scheme == SCHEME_HTTPS
+#else
+                                       0
+#endif
+                                       ),
+                        rel_value);
+
+  /* Add the user headers. */
+  if (opt.user_headers)
+    {
+      int i;
+      for (i = 0; opt.user_headers[i]; i++)
+        request_set_user_header (req, opt.user_headers[i]);
+    }
+
+  proxyauth = NULL;
+  if (proxy)
+    {
+      conn = proxy;
+      initialize_proxy_configuration (u, req, proxy, &proxyauth);
+    }
+  keep_alive = true;
+
+  /* Establish the connection.  */
+  if (inhibit_keep_alive)
+    keep_alive = false;
+
+  {
+    uerr_t err = establish_connection (u, &conn, hs, proxy, &proxyauth, &req,
+                                       &using_ssl, inhibit_keep_alive, &sock);
+    if (err != RETROK)
+      return err;
+  }
+
 
   /* Open the temporary file where we will write the request. */
   if (warc_enabled)
-- 
2.1.0

>From 1fa662a61e9c856e8802b2c804f41390d73c98b1 Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Mon, 16 Mar 2015 01:13:11 +0100
Subject: [PATCH 4/6] Factor out some gethttp code

* src/http.c (gethttp): Move some code in...
(check_file_output): ... a new function.
---
 src/http.c | 235 ++++++++++++++++++++++++++++++++-----------------------------
 1 file changed, 123 insertions(+), 112 deletions(-)

diff --git a/src/http.c b/src/http.c
index 2daa64c..13ca2a8 100644
--- a/src/http.c
+++ b/src/http.c
@@ -2002,6 +2002,118 @@ establish_connection (struct url *u, struct url 
**conn_ref,
   return RETROK;
 }
 
+static uerr_t
+check_file_output (struct url *u, struct http_stat *hs,
+                   struct response *resp, char *hdrval, size_t hdrsize)
+{
+  /* Determine the local filename if needed. Notice that if -O is used
+   * hstat.local_file is set by http_loop to the argument of -O. */
+  if (!hs->local_file)
+    {
+      char *local_file = NULL;
+
+      /* Honor Content-Disposition whether possible. */
+      if (!opt.content_disposition
+          || !resp_header_copy (resp, "Content-Disposition",
+                                hdrval, hdrsize)
+          || !parse_content_disposition (hdrval, &local_file))
+        {
+          /* The Content-Disposition header is missing or broken.
+           * Choose unique file name according to given URL. */
+          hs->local_file = url_file_name (u, NULL);
+        }
+      else
+        {
+          DEBUGP (("Parsed filename from Content-Disposition: %s\n",
+                  local_file));
+          hs->local_file = url_file_name (u, local_file);
+        }
+
+      xfree (local_file);
+    }
+
+  /* TODO: perform this check only once. */
+  if (!hs->existence_checked && file_exists_p (hs->local_file))
+    {
+      if (opt.noclobber && !opt.output_document)
+        {
+          /* If opt.noclobber is turned on and file already exists, do not
+             retrieve the file. But if the output_document was given, then this
+             test was already done and the file didn't exist. Hence the 
!opt.output_document */
+          return RETRUNNEEDED;
+        }
+      else if (!ALLOW_CLOBBER)
+        {
+          char *unique = unique_name (hs->local_file, true);
+          if (unique != hs->local_file)
+            xfree (hs->local_file);
+          hs->local_file = unique;
+        }
+    }
+  hs->existence_checked = true;
+
+  /* Support timestamping */
+  if (opt.timestamping && !hs->timestamp_checked)
+    {
+      size_t filename_len = strlen (hs->local_file);
+      char *filename_plus_orig_suffix = alloca (filename_len + sizeof 
(ORIG_SFX));
+      bool local_dot_orig_file_exists = false;
+      char *local_filename = NULL;
+      struct_stat st;
+
+      if (opt.backup_converted)
+        /* If -K is specified, we'll act on the assumption that it was 
specified
+           last time these files were downloaded as well, and instead of just
+           comparing local file X against server file X, we'll compare local
+           file X.orig (if extant, else X) against server file X.  If -K
+           _wasn't_ specified last time, or the server contains files called
+           *.orig, -N will be back to not operating correctly with -k. */
+        {
+          /* Would a single s[n]printf() call be faster?  --dan
+
+             Definitely not.  sprintf() is horribly slow.  It's a
+             different question whether the difference between the two
+             affects a program.  Usually I'd say "no", but at one
+             point I profiled Wget, and found that a measurable and
+             non-negligible amount of time was lost calling sprintf()
+             in url.c.  Replacing sprintf with inline calls to
+             strcpy() and number_to_string() made a difference.
+             --hniksic */
+          memcpy (filename_plus_orig_suffix, hs->local_file, filename_len);
+          memcpy (filename_plus_orig_suffix + filename_len,
+                  ORIG_SFX, sizeof (ORIG_SFX));
+
+          /* Try to stat() the .orig file. */
+          if (stat (filename_plus_orig_suffix, &st) == 0)
+            {
+              local_dot_orig_file_exists = true;
+              local_filename = filename_plus_orig_suffix;
+            }
+        }
+
+      if (!local_dot_orig_file_exists)
+        /* Couldn't stat() <file>.orig, so try to stat() <file>. */
+        if (stat (hs->local_file, &st) == 0)
+          local_filename = hs->local_file;
+
+      if (local_filename != NULL)
+        /* There was a local file, so we'll check later to see if the version
+           the server has is the same version we already have, allowing us to
+           skip a download. */
+        {
+          hs->orig_file_name = xstrdup (local_filename);
+          hs->orig_file_size = st.st_size;
+          hs->orig_file_tstamp = st.st_mtime;
+#ifdef WINDOWS
+          /* Modification time granularity is 2 seconds for Windows, so
+             increase local time by 1 second for later comparison. */
+          ++hs->orig_file_tstamp;
+#endif
+        }
+    }
+  return RETROK;
+}
+
 /* Retrieve a document through HTTP protocol.  It recognizes status
    code, and correctly handles redirections.  It closes the network
    socket.  If it receives an error from the functions below it, it
@@ -2566,118 +2678,17 @@ read_header:
     }
 
 
-  /* Determine the local filename if needed. Notice that if -O is used
-   * hstat.local_file is set by http_loop to the argument of -O. */
-  if (!hs->local_file)
-    {
-      char *local_file = NULL;
-
-      /* Honor Content-Disposition whether possible. */
-      if (!opt.content_disposition
-          || !resp_header_copy (resp, "Content-Disposition",
-                                hdrval, sizeof (hdrval))
-          || !parse_content_disposition (hdrval, &local_file))
-        {
-          /* The Content-Disposition header is missing or broken.
-           * Choose unique file name according to given URL. */
-          hs->local_file = url_file_name (u, NULL);
-        }
-      else
-        {
-          DEBUGP (("Parsed filename from Content-Disposition: %s\n",
-                  local_file));
-          hs->local_file = url_file_name (u, local_file);
-        }
-
-      xfree (local_file);
-    }
-
-  /* TODO: perform this check only once. */
-  if (!hs->existence_checked && file_exists_p (hs->local_file))
-    {
-      if (opt.noclobber && !opt.output_document)
-        {
-          /* If opt.noclobber is turned on and file already exists, do not
-             retrieve the file. But if the output_document was given, then this
-             test was already done and the file didn't exist. Hence the 
!opt.output_document */
-          get_file_flags (hs->local_file, dt);
-          request_free (req);
-          resp_free (resp);
-          xfree (head);
-          xfree (message);
-          return RETRUNNEEDED;
-        }
-      else if (!ALLOW_CLOBBER)
-        {
-          char *unique = unique_name (hs->local_file, true);
-          if (unique != hs->local_file)
-            xfree (hs->local_file);
-          hs->local_file = unique;
-        }
-    }
-  hs->existence_checked = true;
-
-  /* Support timestamping */
-  /* TODO: move this code out of gethttp. */
-  if (opt.timestamping && !hs->timestamp_checked)
-    {
-      size_t filename_len = strlen (hs->local_file);
-      char *filename_plus_orig_suffix = alloca (filename_len + sizeof 
(ORIG_SFX));
-      bool local_dot_orig_file_exists = false;
-      char *local_filename = NULL;
-      struct_stat st;
-
-      if (opt.backup_converted)
-        /* If -K is specified, we'll act on the assumption that it was 
specified
-           last time these files were downloaded as well, and instead of just
-           comparing local file X against server file X, we'll compare local
-           file X.orig (if extant, else X) against server file X.  If -K
-           _wasn't_ specified last time, or the server contains files called
-           *.orig, -N will be back to not operating correctly with -k. */
-        {
-          /* Would a single s[n]printf() call be faster?  --dan
-
-             Definitely not.  sprintf() is horribly slow.  It's a
-             different question whether the difference between the two
-             affects a program.  Usually I'd say "no", but at one
-             point I profiled Wget, and found that a measurable and
-             non-negligible amount of time was lost calling sprintf()
-             in url.c.  Replacing sprintf with inline calls to
-             strcpy() and number_to_string() made a difference.
-             --hniksic */
-          memcpy (filename_plus_orig_suffix, hs->local_file, filename_len);
-          memcpy (filename_plus_orig_suffix + filename_len,
-                  ORIG_SFX, sizeof (ORIG_SFX));
-
-          /* Try to stat() the .orig file. */
-          if (stat (filename_plus_orig_suffix, &st) == 0)
-            {
-              local_dot_orig_file_exists = true;
-              local_filename = filename_plus_orig_suffix;
-            }
-        }
-
-      if (!local_dot_orig_file_exists)
-        /* Couldn't stat() <file>.orig, so try to stat() <file>. */
-        if (stat (hs->local_file, &st) == 0)
-          local_filename = hs->local_file;
-
-      if (local_filename != NULL)
-        /* There was a local file, so we'll check later to see if the version
-           the server has is the same version we already have, allowing us to
-           skip a download. */
-        {
-          hs->orig_file_name = xstrdup (local_filename);
-          hs->orig_file_size = st.st_size;
-          hs->orig_file_tstamp = st.st_mtime;
-#ifdef WINDOWS
-          /* Modification time granularity is 2 seconds for Windows, so
-             increase local time by 1 second for later comparison. */
-          ++hs->orig_file_tstamp;
-#endif
-        }
-    }
-
+  {
+    uerr_t ret = check_file_output (u, hs, resp, hdrval, sizeof hdrval);
+    if (ret != RETROK)
+      {
+        request_free (req);
+        resp_free (resp);
+        xfree (head);
+        xfree (message);
+        return ret;
+      }
+  }
   request_free (req);
 
   hs->statcode = statcode;
-- 
2.1.0

>From 237e0a4145bff39ea03b907d5e3fe473b076727d Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Mon, 16 Mar 2015 01:41:20 +0100
Subject: [PATCH 5/6] Factor out some auth gethttp code

* src/http.c (gethttp): Move some code in...
(check_auth): ... a new function.
---
 src/http.c | 270 ++++++++++++++++++++++++++++++++++---------------------------
 1 file changed, 150 insertions(+), 120 deletions(-)

diff --git a/src/http.c b/src/http.c
index 13ca2a8..02e0283 100644
--- a/src/http.c
+++ b/src/http.c
@@ -2114,6 +2114,141 @@ check_file_output (struct url *u, struct http_stat *hs,
   return RETROK;
 }
 
+static uerr_t
+check_auth (struct url *u, char *user, char *passwd, struct response *resp,
+            struct request *req, bool *ntlm_seen_ref, bool *retry,
+            bool *basic_auth_finished_ref, bool *auth_finished_ref)
+{
+  uerr_t auth_err = RETROK;
+  bool basic_auth_finished = *basic_auth_finished_ref;
+  bool auth_finished = *auth_finished_ref;
+  bool ntlm_seen = *ntlm_seen_ref;
+  *retry = false;
+  if (!auth_finished && (user && passwd))
+    {
+      /* IIS sends multiple copies of WWW-Authenticate, one with
+         the value "negotiate", and other(s) with data.  Loop over
+         all the occurrences and pick the one we recognize.  */
+      int wapos;
+      char *buf;
+      const char *www_authenticate = NULL;
+      const char *wabeg, *waend;
+      const char *digest = NULL, *basic = NULL, *ntlm = NULL;
+      for (wapos = 0; !ntlm
+             && (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
+                                             &wabeg, &waend)) != -1;
+           ++wapos)
+        {
+          param_token name, value;
+
+          BOUNDED_TO_ALLOCA (wabeg, waend, buf);
+          www_authenticate = buf;
+
+          for (;!ntlm;)
+            {
+              /* extract the auth-scheme */
+              while (c_isspace (*www_authenticate)) www_authenticate++;
+              name.e = name.b = www_authenticate;
+              while (*name.e && !c_isspace (*name.e)) name.e++;
+
+              if (name.b == name.e)
+                break;
+
+              DEBUGP (("Auth scheme found '%.*s'\n", (int) (name.e - name.b), 
name.b));
+
+              if (known_authentication_scheme_p (name.b, name.e))
+                {
+                  if (BEGINS_WITH (name.b, "NTLM"))
+                    {
+                      ntlm = name.b;
+                      break; /* this is the most secure challenge, stop here */
+                    }
+                  else if (!digest && BEGINS_WITH (name.b, "Digest"))
+                    digest = name.b;
+                  else if (!basic && BEGINS_WITH (name.b, "Basic"))
+                    basic = name.b;
+                }
+
+              /* now advance over the auth-params */
+              www_authenticate = name.e;
+              DEBUGP (("Auth param list '%s'\n", www_authenticate));
+              while (extract_param (&www_authenticate, &name, &value, ',', 
NULL) && name.b && value.b)
+                {
+                  DEBUGP (("Auth param %.*s=%.*s\n",
+                           (int) (name.e - name.b), name.b, (int) (value.e - 
value.b), value.b));
+                }
+            }
+        }
+
+      if (!basic && !digest && !ntlm)
+        {
+          /* If the authentication header is missing or
+             unrecognized, there's no sense in retrying.  */
+          logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
+        }
+      else if (!basic_auth_finished
+               || !basic)
+        {
+          char *pth = url_full_path (u);
+          const char *value;
+          uerr_t *auth_stat;
+          auth_stat = xmalloc (sizeof (uerr_t));
+          *auth_stat = RETROK;
+
+          if (ntlm)
+            www_authenticate = ntlm;
+          else if (digest)
+            www_authenticate = digest;
+          else
+            www_authenticate = basic;
+
+          logprintf (LOG_NOTQUIET, _("Authentication selected: %s\n"), 
www_authenticate);
+
+          value =  create_authorization_line (www_authenticate,
+                                              user, passwd,
+                                              request_method (req),
+                                              pth,
+                                              &auth_finished,
+                                              auth_stat);
+
+          auth_err = *auth_stat;
+          if (auth_err == RETROK)
+            {
+              request_set_header (req, "Authorization", value, rel_value);
+
+              if (BEGINS_WITH (www_authenticate, "NTLM"))
+                ntlm_seen = true;
+              else if (!u->user && BEGINS_WITH (www_authenticate, "Basic"))
+                {
+                  /* Need to register this host as using basic auth,
+                   * so we automatically send creds next time. */
+                  register_basic_auth_host (u->host);
+                }
+
+              xfree (pth);
+              xfree (auth_stat);
+              *retry = true;
+              goto cleanup;
+            }
+          else
+            {
+              /* Creating the Authorization header went wrong */
+            }
+        }
+      else
+        {
+          /* We already did Basic auth, and it failed. Gotta
+           * give up. */
+        }
+    }
+
+ cleanup:
+  *ntlm_seen_ref = ntlm_seen;
+  *basic_auth_finished_ref = basic_auth_finished;
+  *auth_finished_ref = auth_finished;
+  return auth_err;
+}
+
 /* Retrieve a document through HTTP protocol.  It recognizes status
    code, and correctly handles redirections.  It closes the network
    socket.  If it receives an error from the functions below it, it
@@ -2488,7 +2623,7 @@ read_header:
     {
       /* Authorization is required.  */
       uerr_t auth_err = RETROK;
-
+      bool retry;
       /* Normally we are not interested in the response body.
          But if we are writing a WARC file we are: we like to keep everyting.  
*/
       if (warc_enabled)
@@ -2525,126 +2660,21 @@ read_header:
         }
 
       pconn.authorized = false;
-      if (!auth_finished && (user && passwd))
-        {
-          /* IIS sends multiple copies of WWW-Authenticate, one with
-             the value "negotiate", and other(s) with data.  Loop over
-             all the occurrences and pick the one we recognize.  */
-          int wapos;
-          char *buf;
-          const char *www_authenticate = NULL;
-          const char *wabeg, *waend;
-          const char *digest = NULL, *basic = NULL, *ntlm = NULL;
-          for (wapos = 0; !ntlm
-               && (wapos = resp_header_locate (resp, "WWW-Authenticate", wapos,
-                                            &wabeg, &waend)) != -1;
-               ++wapos)
-            {
-              param_token name, value;
-
-              BOUNDED_TO_ALLOCA (wabeg, waend, buf);
-              www_authenticate = buf;
-
-              for (;!ntlm;)
-                {
-                  /* extract the auth-scheme */
-                  while (c_isspace (*www_authenticate)) www_authenticate++;
-                  name.e = name.b = www_authenticate;
-                  while (*name.e && !c_isspace (*name.e)) name.e++;
-
-                  if (name.b == name.e)
-                    break;
-
-                  DEBUGP (("Auth scheme found '%.*s'\n", (int) (name.e - 
name.b), name.b));
-
-                  if (known_authentication_scheme_p (name.b, name.e))
-                    {
-                      if (BEGINS_WITH (name.b, "NTLM"))
-                        {
-                          ntlm = name.b;
-                          break; /* this is the most secure challenge, stop 
here */
-                        }
-                      else if (!digest && BEGINS_WITH (name.b, "Digest"))
-                        digest = name.b;
-                      else if (!basic && BEGINS_WITH (name.b, "Basic"))
-                        basic = name.b;
-                    }
-
-                  /* now advance over the auth-params */
-                  www_authenticate = name.e;
-                  DEBUGP (("Auth param list '%s'\n", www_authenticate));
-                  while (extract_param (&www_authenticate, &name, &value, ',', 
NULL) && name.b && value.b)
-                    {
-                      DEBUGP (("Auth param %.*s=%.*s\n",
-                               (int) (name.e - name.b), name.b, (int) (value.e 
- value.b), value.b));
-                    }
-                }
-            }
-
-          if (!basic && !digest && !ntlm)
-            {
-              /* If the authentication header is missing or
-                 unrecognized, there's no sense in retrying.  */
-              logputs (LOG_NOTQUIET, _("Unknown authentication scheme.\n"));
-            }
-          else if (!basic_auth_finished
-                   || !basic)
-            {
-              char *pth = url_full_path (u);
-              const char *value;
-              uerr_t *auth_stat;
-              auth_stat = xmalloc (sizeof (uerr_t));
-              *auth_stat = RETROK;
-
-              if (ntlm)
-                www_authenticate = ntlm;
-              else if (digest)
-                www_authenticate = digest;
-              else
-                www_authenticate = basic;
-
-              logprintf (LOG_NOTQUIET, _("Authentication selected: %s\n"), 
www_authenticate);
-
-              value =  create_authorization_line (www_authenticate,
-                                                  user, passwd,
-                                                  request_method (req),
-                                                  pth,
-                                                  &auth_finished,
-                                                  auth_stat);
 
-              auth_err = *auth_stat;
-              if (auth_err == RETROK)
-                {
-                  request_set_header (req, "Authorization", value, rel_value);
-
-                  if (BEGINS_WITH (www_authenticate, "NTLM"))
-                    ntlm_seen = true;
-                  else if (!u->user && BEGINS_WITH (www_authenticate, "Basic"))
-                    {
-                      /* Need to register this host as using basic auth,
-                       * so we automatically send creds next time. */
-                      register_basic_auth_host (u->host);
-                    }
-
-                  xfree (pth);
-                  xfree (message);
-                  resp_free (resp);
-                  xfree (head);
-                  xfree (auth_stat);
-                  xfree (hs->message);
-                  goto retry_with_auth;
-                }
-              else
-                {
-                  /* Creating the Authorization header went wrong */
-                }
-            }
-          else
-            {
-              /* We already did Basic auth, and it failed. Gotta
-               * give up. */
-            }
-        }
+      {
+        auth_err = check_auth (u, user, passwd, resp, req,
+                               &ntlm_seen, &retry,
+                               &basic_auth_finished,
+                               &auth_finished);
+        if (auth_err == RETROK && retry)
+          {
+            xfree (hs->message);
+            resp_free (resp);
+            xfree (message);
+            xfree (head);
+            goto retry_with_auth;
+          }
+      }
       request_free (req);
       xfree (message);
       resp_free (resp);
-- 
2.1.0

>From be34f9e944dd07b7b548e6534159e066f9b8ef79 Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Mon, 16 Mar 2015 02:06:37 +0100
Subject: [PATCH 6/6] Factor out some gethttp code

* src/http.c (gethttp): Move some code in...
(open_output_stream): ... a new function.
---
 src/http.c | 170 +++++++++++++++++++++++++++++++------------------------------
 1 file changed, 87 insertions(+), 83 deletions(-)

diff --git a/src/http.c b/src/http.c
index 02e0283..750b69d 100644
--- a/src/http.c
+++ b/src/http.c
@@ -2249,6 +2249,87 @@ check_auth (struct url *u, char *user, char *passwd, 
struct response *resp,
   return auth_err;
 }
 
+static uerr_t
+open_output_stream (struct http_stat *hs, int count, FILE **fp)
+{
+/* 2005-06-17 SMS.
+   For VMS, define common fopen() optional arguments.
+*/
+#ifdef __VMS
+# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
+# define FOPEN_BIN_FLAG 3
+#else /* def __VMS */
+# define FOPEN_BIN_FLAG true
+#endif /* def __VMS [else] */
+
+  /* Open the local file.  */
+  if (!output_stream)
+    {
+      mkalldirs (hs->local_file);
+      if (opt.backups)
+        rotate_backups (hs->local_file);
+      if (hs->restval)
+        {
+#ifdef __VMS
+          int open_id;
+
+          open_id = 21;
+          *fp = fopen (hs->local_file, "ab", FOPEN_OPT_ARGS);
+#else /* def __VMS */
+          *fp = fopen (hs->local_file, "ab");
+#endif /* def __VMS [else] */
+        }
+      else if (ALLOW_CLOBBER || count > 0)
+        {
+          if (opt.unlink && file_exists_p (hs->local_file))
+            {
+              if (unlink (hs->local_file) < 0)
+                {
+                  logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
+                             strerror (errno));
+                  return UNLINKERR;
+                }
+            }
+
+#ifdef __VMS
+          int open_id;
+
+          open_id = 22;
+          *fp = fopen (hs->local_file, "wb", FOPEN_OPT_ARGS);
+#else /* def __VMS */
+          *fp = fopen (hs->local_file, "wb");
+#endif /* def __VMS [else] */
+        }
+      else
+        {
+          *fp = fopen_excl (hs->local_file, FOPEN_BIN_FLAG);
+          if (!*fp && errno == EEXIST)
+            {
+              /* We cannot just invent a new name and use it (which is
+                 what functions like unique_create typically do)
+                 because we told the user we'd use this name.
+                 Instead, return and retry the download.  */
+              logprintf (LOG_NOTQUIET,
+                         _("%s has sprung into existence.\n"),
+                         hs->local_file);
+              return FOPEN_EXCL_ERR;
+            }
+        }
+      if (!*fp)
+        {
+          logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror 
(errno));
+          return FOPENERR;
+        }
+    }
+  else
+    *fp = output_stream;
+
+  /* Print fetch message, if opt.verbose.  */
+      logprintf (LOG_VERBOSE, _("Saving to: %s\n"),
+                 HYPHENP (hs->local_file) ? quote ("STDOUT") : quote 
(hs->local_file));
+      return RETROK;
+}
+
 /* Retrieve a document through HTTP protocol.  It recognizes status
    code, and correctly handles redirections.  It closes the network
    socket.  If it receives an error from the functions below it, it
@@ -3033,91 +3114,14 @@ read_header:
       return RETRFINISHED;
     }
 
-/* 2005-06-17 SMS.
-   For VMS, define common fopen() optional arguments.
-*/
-#ifdef __VMS
-# define FOPEN_OPT_ARGS "fop=sqo", "acc", acc_cb, &open_id
-# define FOPEN_BIN_FLAG 3
-#else /* def __VMS */
-# define FOPEN_BIN_FLAG true
-#endif /* def __VMS [else] */
-
-  /* Open the local file.  */
-  if (!output_stream)
+  err = open_output_stream (hs, count, &fp);
+  if (err != RETROK)
     {
-      mkalldirs (hs->local_file);
-      if (opt.backups)
-        rotate_backups (hs->local_file);
-      if (hs->restval)
-        {
-#ifdef __VMS
-          int open_id;
-
-          open_id = 21;
-          fp = fopen (hs->local_file, "ab", FOPEN_OPT_ARGS);
-#else /* def __VMS */
-          fp = fopen (hs->local_file, "ab");
-#endif /* def __VMS [else] */
-        }
-      else if (ALLOW_CLOBBER || count > 0)
-        {
-          if (opt.unlink && file_exists_p (hs->local_file))
-            {
-              if (unlink (hs->local_file) < 0)
-                {
-                  logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file,
-                             strerror (errno));
-                  CLOSE_INVALIDATE (sock);
-                  xfree (head);
-                  xfree (type);
-                  return UNLINKERR;
-                }
-            }
-
-#ifdef __VMS
-          int open_id;
-
-          open_id = 22;
-          fp = fopen (hs->local_file, "wb", FOPEN_OPT_ARGS);
-#else /* def __VMS */
-          fp = fopen (hs->local_file, "wb");
-#endif /* def __VMS [else] */
-        }
-      else
-        {
-          fp = fopen_excl (hs->local_file, FOPEN_BIN_FLAG);
-          if (!fp && errno == EEXIST)
-            {
-              /* We cannot just invent a new name and use it (which is
-                 what functions like unique_create typically do)
-                 because we told the user we'd use this name.
-                 Instead, return and retry the download.  */
-              logprintf (LOG_NOTQUIET,
-                         _("%s has sprung into existence.\n"),
-                         hs->local_file);
-              CLOSE_INVALIDATE (sock);
-              xfree (head);
-              xfree (type);
-              return FOPEN_EXCL_ERR;
-            }
-        }
-      if (!fp)
-        {
-          logprintf (LOG_NOTQUIET, "%s: %s\n", hs->local_file, strerror 
(errno));
-          CLOSE_INVALIDATE (sock);
-          xfree (head);
-          xfree (type);
-          return FOPENERR;
-        }
+      xfree (type);
+      xfree (head);
+      CLOSE_INVALIDATE (sock);
+      return err;
     }
-  else
-    fp = output_stream;
-
-  /* Print fetch message, if opt.verbose.  */
-      logprintf (LOG_VERBOSE, _("Saving to: %s\n"),
-                 HYPHENP (hs->local_file) ? quote ("STDOUT") : quote 
(hs->local_file));
-
 
   err = read_response_body (hs, sock, fp, contlen, contrange,
                             chunked_transfer_encoding,
-- 
2.1.0


reply via email to

[Prev in Thread] Current Thread [Next in Thread]