[GNUnet-SVN] [gnurl] 70/205: url: add option CURLOPT_SUPPRESS_CONNECT_HEADERS

[gnunet]

This is an automated email from the git hooks/post-receive script.

ng0 pushed a commit to annotated tag gnurl-7.54.0
in repository gnurl.

commit d2bcf1e3e247d116dc96bd3ea32056e3f089449c
Author: Desmond O. Chang <address@hidden>
AuthorDate: Thu Apr 28 17:33:25 2016 +0800

    url: add option CURLOPT_SUPPRESS_CONNECT_HEADERS
    
    - Add new option CURLOPT_SUPPRESS_CONNECT_HEADERS to allow suppressing
      proxy CONNECT response headers from the user callback functions
      CURLOPT_HEADERFUNCTION and CURLOPT_WRITEFUNCTION.
    
    - Add new tool option --suppress-connect-headers to expose
      CURLOPT_SUPPRESS_CONNECT_HEADERS and allow suppressing proxy CONNECT
      response headers from --dump-header and --include.
    
    Assisted-by: Jay Satiro
    Assisted-by: address@hidden
    Closes https://github.com/curl/curl/pull/783
---
 docs/cmdline-opts/Makefile.am                      |  3 +-
 docs/cmdline-opts/proxytunnel.d                    |  3 +
 docs/cmdline-opts/suppress-connect-headers.d       |  8 ++
 docs/libcurl/curl_easy_setopt.3                    |  2 +
 docs/libcurl/opts/CURLINFO_HEADER_SIZE.3           |  3 +
 docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3        |  3 +
 .../opts/CURLOPT_SUPPRESS_CONNECT_HEADERS.3        | 95 +++++++++++++++++++++
 docs/libcurl/opts/Makefile.am                      |  1 +
 docs/libcurl/symbols-in-versions                   |  1 +
 include/curl/curl.h                                |  3 +
 lib/http_proxy.c                                   | 19 ++---
 lib/url.c                                          |  3 +
 lib/urldata.h                                      |  2 +
 packages/OS400/curl.inc.in                         |  2 +
 src/tool_cfgable.h                                 |  2 +
 src/tool_getparam.c                                |  4 +
 src/tool_help.c                                    |  1 +
 src/tool_operate.c                                 |  5 +-
 tests/data/Makefile.inc                            |  1 +
 tests/data/test1288                                | 96 ++++++++++++++++++++++
 20 files changed, 245 insertions(+), 12 deletions(-)

diff --git a/docs/cmdline-opts/Makefile.am b/docs/cmdline-opts/Makefile.am
index f425ddd08..5cfd67605 100644
--- a/docs/cmdline-opts/Makefile.am
+++ b/docs/cmdline-opts/Makefile.am
@@ -59,7 +59,8 @@ DPAGES = abstract-unix-socket.d anyauth.d append.d basic.d 
cacert.d capath.d cer
   service-name.d show-error.d silent.d socks4a.d socks4.d socks5.d     \
   socks5-gssapi-nec.d socks5-gssapi-service.d socks5-hostname.d                
\
   speed-limit.d speed-time.d ssl-allow-beast.d ssl.d ssl-no-revoke.d   \
-  ssl-reqd.d sslv2.d sslv3.d stderr.d tcp-fastopen.d tcp-nodelay.d     \
+  ssl-reqd.d sslv2.d sslv3.d stderr.d suppress-connect-headers.d       \
+  tcp-fastopen.d tcp-nodelay.d                                         \
   telnet-option.d tftp-blksize.d tftp-no-options.d time-cond.d         \
   tls-max.d \
   tlsauthtype.d tlspassword.d tlsuser.d tlsv1.0.d tlsv1.1.d tlsv1.2.d  \
diff --git a/docs/cmdline-opts/proxytunnel.d b/docs/cmdline-opts/proxytunnel.d
index 09855ed34..f2e8b802d 100644
--- a/docs/cmdline-opts/proxytunnel.d
+++ b/docs/cmdline-opts/proxytunnel.d
@@ -8,3 +8,6 @@ to attempt to tunnel through the proxy instead of merely using 
it to do
 HTTP-like operations. The tunnel approach is made with the HTTP proxy CONNECT
 request and requires that the proxy allows direct connect to the remote port
 number curl wants to tunnel through to.
+
+To suppress proxy CONNECT response headers when curl is set to output headers
+use --suppress-connect-headers.
diff --git a/docs/cmdline-opts/suppress-connect-headers.d 
b/docs/cmdline-opts/suppress-connect-headers.d
new file mode 100644
index 000000000..d208b8917
--- /dev/null
+++ b/docs/cmdline-opts/suppress-connect-headers.d
@@ -0,0 +1,8 @@
+Long: suppress-connect-headers
+Help: Suppress proxy CONNECT response headers
+See-also: dump-header include proxytunnel
+---
+When --proxytunnel is used and a CONNECT request is made don't output proxy
+CONNECT response headers. This option is meant to be used with --dump-header or
+--include which are used to show protocol headers in the output. It has no
+effect on debug options such as --verbose or --trace, or any statistics.
diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 422cb8569..cb6884766 100644
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -137,6 +137,8 @@ Data pointer to pass to the chunk callbacks. See 
\fICURLOPT_CHUNK_DATA(3)\fP
 Callback for wildcard matching. See \fICURLOPT_FNMATCH_FUNCTION(3)\fP
 .IP CURLOPT_FNMATCH_DATA
 Data pointer to pass to the wildcard matching callback. See 
\fICURLOPT_FNMATCH_DATA(3)\fP
+.IP CURLOPT_SUPPRESS_CONNECT_HEADERS
+Suppress proxy CONNECT response headers from user callbacks. See 
\fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP
 .SH ERROR OPTIONS
 .IP CURLOPT_ERRORBUFFER
 Error message buffer. See \fICURLOPT_ERRORBUFFER(3)\fP
diff --git a/docs/libcurl/opts/CURLINFO_HEADER_SIZE.3 
b/docs/libcurl/opts/CURLINFO_HEADER_SIZE.3
index b4b9b578f..b687c80b9 100644
--- a/docs/libcurl/opts/CURLINFO_HEADER_SIZE.3
+++ b/docs/libcurl/opts/CURLINFO_HEADER_SIZE.3
@@ -30,6 +30,9 @@ CURLcode curl_easy_getinfo(CURL *handle, 
CURLINFO_HEADER_SIZE, long *sizep);
 .SH DESCRIPTION
 Pass a pointer to a long to receive the total size of all the headers
 received. Measured in number of bytes.
+
+The total includes the size of any received headers suppressed by
+\fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP.
 .SH PROTOCOLS
 All
 .SH EXAMPLE
diff --git a/docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3 
b/docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3
index 6b48a0527..bdd04dfff 100644
--- a/docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3
+++ b/docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3
@@ -40,6 +40,9 @@ it allows CONNECT requests to and often only port 80 and 443 
are allowed.
 
 When using this, it only makes sense to use \fICURLOPT_PROXYTYPE(3)\fP set to
 a HTTP proxy.
+
+To suppress proxy CONNECT response headers from user callbacks use
+\fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP.
 .SH DEFAULT
 0
 .SH PROTOCOLS
diff --git a/docs/libcurl/opts/CURLOPT_SUPPRESS_CONNECT_HEADERS.3 
b/docs/libcurl/opts/CURLOPT_SUPPRESS_CONNECT_HEADERS.3
new file mode 100644
index 000000000..8cfec7a8f
--- /dev/null
+++ b/docs/libcurl/opts/CURLOPT_SUPPRESS_CONNECT_HEADERS.3
@@ -0,0 +1,95 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <address@hidden>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_SUPPRESS_CONNECT_HEADERS 3 "13 February 2017" "libcurl 7.54.0" 
"curl_easy_setopt options"
+.SH NAME
+CURLOPT_SUPPRESS_CONNECT_HEADERS \- Suppress proxy CONNECT response headers 
from user callbacks
+.SH SYNOPSIS
+.nf
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, long 
onoff);
+.fi
+.SH DESCRIPTION
+When \fICURLOPT_HTTPPROXYTUNNEL(3)\fP is used and a CONNECT request is made,
+suppress proxy CONNECT response headers from the user callback functions
+\fICURLOPT_HEADERFUNCTION(3)\fP and \fICURLOPT_WRITEFUNCTION(3)\fP.
+
+Proxy CONNECT response headers can complicate header processing since it's
+essentially a separate set of headers. You can enable this option to suppress
+those headers.
+
+For example let's assume an HTTPS URL is to be retrieved via CONNECT. On
+success there would normally be two sets of headers, and each header line sent
+to the header function and/or the write function. The data given to the
+callbacks would look like this:
+
+.nf
+HTTP/1.1 200 Connection established
+{headers}...
+
+HTTP/1.1 200 OK
+Content-Type: application/json
+{headers}...
+
+{body}...
+.fi
+
+However by enabling this option the CONNECT response headers are suppressed, so
+the data given to the callbacks would look like this:
+
+.nf
+HTTP/1.1 200 OK
+Content-Type: application/json
+{headers}...
+
+{body}...
+.fi
+
+.SH DEFAULT
+0
+.SH PROTOCOLS
+All
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  curl_easy_setopt(curl, CURLOPT_URL, "https://example.com";);
+
+  curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
+  curl_easy_setopt(curl, CURLOPT_PROXY, "http://foo:3128";);
+  curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, 1L);
+  curl_easy_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L);
+
+  curl_easy_perform(curl);
+
+  /* always cleanup */
+  curl_easy_cleanup(curl);
+}
+.fi
+.SH AVAILABILITY
+Added in 7.54.0
+.SH RETURN VALUE
+CURLE_OK or an error such as CURLE_UNKNOWN_OPTION.
+.SH "SEE ALSO"
+.BR CURLOPT_HEADER "(3), " CURLOPT_PROXY "(3), "
+.BR CURLOPT_HTTPPROXYTUNNEL "(3), "
diff --git a/docs/libcurl/opts/Makefile.am b/docs/libcurl/opts/Makefile.am
index 5829a144f..26bb2eeb9 100644
--- a/docs/libcurl/opts/Makefile.am
+++ b/docs/libcurl/opts/Makefile.am
@@ -300,6 +300,7 @@ man_MANS =                                      \
  CURLOPT_STREAM_DEPENDS.3                       \
  CURLOPT_STREAM_DEPENDS_E.3                     \
  CURLOPT_STREAM_WEIGHT.3                        \
+ CURLOPT_SUPPRESS_CONNECT_HEADERS.3             \
  CURLOPT_TCP_FASTOPEN.3                         \
  CURLOPT_TCP_KEEPALIVE.3                        \
  CURLOPT_TCP_KEEPIDLE.3                         \
diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index f4cd9805f..6ac8869ca 100644
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -559,6 +559,7 @@ CURLOPT_STDERR                  7.1
 CURLOPT_STREAM_DEPENDS          7.46.0
 CURLOPT_STREAM_DEPENDS_E        7.46.0
 CURLOPT_STREAM_WEIGHT           7.46.0
+CURLOPT_SUPPRESS_CONNECT_HEADERS 7.54.0
 CURLOPT_TCP_KEEPALIVE           7.25.0
 CURLOPT_TCP_KEEPIDLE            7.25.0
 CURLOPT_TCP_KEEPINTVL           7.25.0
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 8b221ef8f..7c7d28d6a 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -1778,6 +1778,9 @@ typedef enum {
   /* Path to an abstract Unix domain socket */
   CINIT(ABSTRACT_UNIX_SOCKET, STRINGPOINT, 264),
 
+  /* Suppress proxy CONNECT response headers from user callbacks */
+  CINIT(SUPPRESS_CONNECT_HEADERS, LONG, 265),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 
diff --git a/lib/http_proxy.c b/lib/http_proxy.c
index a67328647..f09304a26 100644
--- a/lib/http_proxy.c
+++ b/lib/http_proxy.c
@@ -316,8 +316,6 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
       perline = 0;
 
       while(nread < BUFSIZE && keepon && !error) {
-        int writetype;
-
         if(Curl_pgrsUpdate(conn))
           return CURLE_ABORTED_BY_CALLBACK;
 
@@ -419,19 +417,20 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
           Curl_debug(data, CURLINFO_HEADER_IN,
                      line_start, (size_t)perline, conn);
 
-        /* send the header to the callback */
-        writetype = CLIENTWRITE_HEADER;
-        if(data->set.include_header)
-          writetype |= CLIENTWRITE_BODY;
+        if(!data->set.suppress_connect_headers) {
+          /* send the header to the callback */
+          int writetype = CLIENTWRITE_HEADER;
+          if(data->set.include_header)
+            writetype |= CLIENTWRITE_BODY;
 
-        result = Curl_client_write(conn, writetype, line_start, perline);
+          result = Curl_client_write(conn, writetype, line_start, perline);
+          if(result)
+            return result;
+        }
 
         data->info.header_size += (long)perline;
         data->req.headerbytecount += (long)perline;
 
-        if(result)
-          return result;
-
         /* Newlines are CRLF, so the CR is ignored as the line isn't
            really terminated until the LF comes. Treat a following CR
            as end-of-headers as well.*/
diff --git a/lib/url.c b/lib/url.c
index f9c324269..c306b5776 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -2894,6 +2894,9 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption 
option,
   case CURLOPT_CONNECT_TO:
     data->set.connect_to = va_arg(param, struct curl_slist *);
     break;
+  case CURLOPT_SUPPRESS_CONNECT_HEADERS:
+    data->set.suppress_connect_headers = (0 != va_arg(param, long))?TRUE:FALSE;
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_UNKNOWN_OPTION;
diff --git a/lib/urldata.h b/lib/urldata.h
index 7402332d1..2dd7938e0 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1756,6 +1756,8 @@ struct UserDefined {
   bool pipewait;        /* wait for pipe/multiplex status before starting a
                            new connection */
   long expect_100_timeout; /* in milliseconds */
+  bool suppress_connect_headers;  /* suppress proxy CONNECT response headers
+                                     from user callbacks */
 
   struct Curl_easy *stream_depends_on;
   bool stream_depends_e; /* set or don't set the Exclusive bit */
diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in
index b9b0808c7..e4d2d30ca 100644
--- a/packages/OS400/curl.inc.in
+++ b/packages/OS400/curl.inc.in
@@ -1310,6 +1310,8 @@
      d                 c                   10263
      d  CURLOPT_ABSTRACT_UNIX_SOCKET...
      d                 c                   10264
+     d  CURLOPT_SUPPRESS_CONNECT_HEADERS...
+     d                 c                   00265
       *
       /if not defined(CURL_NO_OLDIES)
      d  CURLOPT_FILE   c                   10001
diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
index b05c440e5..b7fe1d340 100644
--- a/src/tool_cfgable.h
+++ b/src/tool_cfgable.h
@@ -235,6 +235,8 @@ struct OperationConfig {
   bool falsestart;
   bool path_as_is;
   double expect100timeout;
+  bool suppress_connect_headers;  /* suppress proxy CONNECT response headers
+                                     from user callbacks */
   struct GlobalConfig *global;
   struct OperationConfig *prev;
   struct OperationConfig *next;   /* Always last in the struct */
diff --git a/src/tool_getparam.c b/src/tool_getparam.c
index 85d75ae6e..e7ac35476 100644
--- a/src/tool_getparam.c
+++ b/src/tool_getparam.c
@@ -185,6 +185,7 @@ static const struct LongShort aliases[]= {
   {"$U", "connect-to",               TRUE},
   {"$W", "abstract-unix-socket",     TRUE},
   {"$X", "tls-max",                  TRUE},
+  {"$Y", "suppress-connect-headers", FALSE},
   {"0",   "http1.0",                 FALSE},
   {"01",  "http1.1",                 FALSE},
   {"02",  "http2",                   FALSE},
@@ -1066,6 +1067,9 @@ ParameterError getparameter(char *flag,    /* f or 
-long-flag */
         if(err)
           return err;
         break;
+      case 'Y': /* --suppress-connect-headers */
+        config->suppress_connect_headers = toggle;
+        break;
       }
       break;
     case '#': /* --progress-bar */
diff --git a/src/tool_help.c b/src/tool_help.c
index f6fe3527b..86d35899a 100644
--- a/src/tool_help.c
+++ b/src/tool_help.c
@@ -249,6 +249,7 @@ static const char *const helptext[] = {
   "     --ssl-allow-beast  Allow security flaw to improve interop (SSL)",
   "     --ssl-no-revoke    Disable cert revocation checks (WinSSL)",
   "     --stderr FILE   Where to redirect stderr (use \"-\" for stdout)",
+  "     --suppress-connect-headers  Suppress proxy CONNECT response headers",
   "     --tcp-nodelay   Use the TCP_NODELAY option",
   "     --tcp-fastopen  Use TCP Fast Open",
   " -t, --telnet-option OPT=VAL  Set telnet option",
diff --git a/src/tool_operate.c b/src/tool_operate.c
index a489b8dbd..8f767152a 100644
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -886,8 +886,11 @@ static CURLcode operate_do(struct GlobalConfig *global,
 
           /* new in libcurl 7.19.4 */
           my_setopt_str(curl, CURLOPT_NOPROXY, config->noproxy);
+
+          my_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS,
+                    config->suppress_connect_headers?1L:0L);
         }
-#endif
+#endif /* !CURL_DISABLE_PROXY */
 
         my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror?1L:0L);
         my_setopt(curl, CURLOPT_UPLOAD, uploadfile?1L:0L);
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index a115ab69d..7742bcfec 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -132,6 +132,7 @@ test1252 test1253 test1254 test1255 test1256 test1257 
test1258 test1259 \
 test1260 \
 \
 test1280 test1281 test1282 test1283 test1284 test1285 test1286 test1287 \
+test1288 \
 \
 test1300 test1301 test1302 test1303 test1304 test1305 test1306 test1307 \
 test1308 test1309 test1310 test1311 test1312 test1313 test1314 test1315 \
diff --git a/tests/data/test1288 b/tests/data/test1288
new file mode 100644
index 000000000..543aa3d6e
--- /dev/null
+++ b/tests/data/test1288
@@ -0,0 +1,96 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+HTTP CONNECT
+HTTP proxy
+proxytunnel
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<connect>
+HTTP/1.1 200 Mighty fine indeed
+Server: test tunnel 2000
+
+</connect>
+
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Content-Type: text/html
+Funny-head: yesyes
+Content-Length: 9
+Connection: keep-alive
+
+contents
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+http-proxy
+</server>
+<name>
+Suppress proxy CONNECT response headers
+</name>
+<command>
+--proxytunnel --suppress-connect-headers --dump-header - --include --write-out 
"\nCONNECT CODE: %{http_connect}\nRECEIVED HEADER BYTE TOTAL: %{size_header}\n" 
--proxy %HOSTIP:%PROXYPORT http://%HOSTIP.1288:%HTTPPORT/we/want/that/page/1288
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<proxy>
+CONNECT %HOSTIP.1288:%HTTPPORT HTTP/1.1
+Host: %HOSTIP.1288:%HTTPPORT
+Proxy-Connection: Keep-Alive
+
+</proxy>
+<protocol>
+GET /we/want/that/page/1288 HTTP/1.1
+Host: %HOSTIP.1288:%HTTPPORT
+Accept: */*
+
+</protocol>
+
+# This test is structured to test all the expectations of
+# --suppress-connect-headers, which are:
+# Must suppress in --include and --dump-header
+# Must not suppress in --verbose and --trace
+# Must not suppress in statistics (eg received header byte total)
+<stdout>
+HTTP/1.1 200 OK
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Server: test-server/fake
+Content-Type: text/html
+Content-Type: text/html
+Funny-head: yesyes
+Funny-head: yesyes
+Content-Length: 9
+Content-Length: 9
+Connection: keep-alive
+Connection: keep-alive
+
+
+contents
+
+CONNECT CODE: 200
+RECEIVED HEADER BYTE TOTAL: 231
+</stdout>
+</verify>
+</testcase>

-- 
To stop receiving notification emails like this one, please contact
address@hidden