[mod_proxy] stream request using HTTP/1.1 chunked (fixes #3006)

stream request body using HTTP/1.1 Transfer-Encoding: chunked

(Note: if backend proxy target does not support HTTP/1.1,
 then do not use server.stream-request-body = 1 or 2)

If not streaming to backend, collect request body
(now supporting Transfer-Encoding: chunked from client
 and then sending with Content-Length to backend)

x-ref:
  "Lighty returns HTTP 411 Length Required with proxy and streaming requests/reponses body"
  https://redmine.lighttpd.net/issues/3006

src/gw_backend.c

@@ -2052,43 +2052,40 @@ handler_t gw_handle_subrequest(request_st * const r, void *p_d) {
         }
         else {
             handler_t rc = connection_handle_read_post_state(r);
-            chunkqueue *req_cq = r->reqbody_queue;
-          #if 0 /*(not reached since we send 411 Length Required below)*/
+
+            /* XXX: create configurable flag */
+            /* CGI environment requires that Content-Length be set.
+             * Send 411 Length Required if Content-Length missing.
+             * (occurs here if client sends Transfer-Encoding: chunked
+             *  and module is flagged to stream request body to backend) */
+            if (-1 == r->reqbody_length && hctx->opts.backend != BACKEND_PROXY){
+                return (r->conf.stream_request_body & FDEVENT_STREAM_REQUEST)
+                  ? connection_handle_read_post_error(r, 411)
+                  : HANDLER_WAIT_FOR_EVENT;
+            }
+
             if (hctx->wb_reqlen < -1 && r->reqbody_length >= 0) {
                 /* (completed receiving Transfer-Encoding: chunked) */
-                hctx->wb_reqlen = -hctx->wb_reqlen + r->reqbody_length;
+                hctx->wb_reqlen = -hctx->wb_reqlen;
                 if (hctx->stdin_append) {
                     handler_t rca = hctx->stdin_append(hctx);
                     if (HANDLER_GO_ON != rca) return rca;
                 }
             }
-          #endif
+
             if ((0 != hctx->wb->bytes_in || -1 == hctx->wb_reqlen)
-                && !chunkqueue_is_empty(req_cq)) {
+                && !chunkqueue_is_empty(r->reqbody_queue)) {
                 if (hctx->stdin_append) {
                     handler_t rca = hctx->stdin_append(hctx);
                     if (HANDLER_GO_ON != rca) return rca;
                 }
                 else
-                    chunkqueue_append_chunkqueue(hctx->wb, req_cq);
+                    chunkqueue_append_chunkqueue(hctx->wb, r->reqbody_queue);
                 if (fdevent_fdnode_interest(hctx->fdn) & FDEVENT_OUT) {
                     return (rc == HANDLER_GO_ON) ? HANDLER_WAIT_FOR_EVENT : rc;
                 }
             }
             if (rc != HANDLER_GO_ON) return rc;
-
-
-            /* XXX: create configurable flag */
-            /* CGI environment requires that Content-Length be set.
-             * Send 411 Length Required if Content-Length missing.
-             * (occurs here if client sends Transfer-Encoding: chunked
-             *  and module is flagged to stream request body to backend) */
-            /* proxy currently sends HTTP/1.0 request and ideally should send
-             * Content-Length with request if request body is present, so
-             * send 411 Length Required if Content-Length missing. */
-            if (-1 == r->reqbody_length) {
-                return connection_handle_read_post_error(r, 411);
-            }
         }
     }
 

src/http-header-glue.c

@@ -1036,14 +1036,6 @@ static int http_response_process_headers(request_st * const r, http_response_opt
             r->content_length = strtoul(value, NULL, 10);
             break;
           case HTTP_HEADER_TRANSFER_ENCODING:
-            if (opts->backend == BACKEND_PROXY) {
-                log_error(r->conf.errh, __FILE__, __LINE__,
-                  "proxy backend sent invalid response header "
-                  "(Transfer-Encoding) to HTTP/1.0 request");
-                r->http_status = 502; /* Bad Gateway */
-                r->handler_module = NULL;
-                return -1;
-            }
             break;
           default:
             break;

src/mod_cgi.c

@@ -982,7 +982,9 @@ SUBREQUEST_FUNC(mod_cgi_handle_subrequest) {
 			 * (occurs here if client sends Transfer-Encoding: chunked
 			 *  and module is flagged to stream request body to backend) */
 			if (-1 == r->reqbody_length) {
-				return connection_handle_read_post_error(r, 411);
+				return (r->conf.stream_request_body & FDEVENT_STREAM_REQUEST)
+				  ? connection_handle_read_post_error(r, 411)
+				  : HANDLER_WAIT_FOR_EVENT;
 			}
 		}
 	}

src/mod_proxy.c

@@ -813,6 +813,42 @@ static void proxy_set_Forwarded(connection * const con, request_st * const r, co
 }
 
 
+static handler_t proxy_stdin_append(gw_handler_ctx *hctx) {
+    /*handler_ctx *hctx = (handler_ctx *)gwhctx;*/
+    chunkqueue * const req_cq = hctx->r->reqbody_queue;
+    const off_t req_cqlen = req_cq->bytes_in - req_cq->bytes_out;
+    if (req_cqlen) {
+        /* XXX: future: use http_chunk_len_append() */
+        buffer * const tb = hctx->r->tmp_buf;
+        buffer_clear(tb);
+        buffer_append_uint_hex_lc(tb, (uintmax_t)req_cqlen);
+        buffer_append_string_len(tb, CONST_STR_LEN("\r\n"));
+
+        const off_t len = (off_t)buffer_string_length(tb)
+                        + 2 /*(+2 end chunk "\r\n")*/
+                        + req_cqlen;
+        if (-1 != hctx->wb_reqlen)
+            hctx->wb_reqlen += (hctx->wb_reqlen >= 0) ? len : -len;
+
+        (chunkqueue_is_empty(hctx->wb) || hctx->wb->first->type == MEM_CHUNK)
+                                          /* else FILE_CHUNK for temp file */
+          ? chunkqueue_append_mem(hctx->wb, CONST_BUF_LEN(tb))
+          : chunkqueue_append_mem_min(hctx->wb, CONST_BUF_LEN(tb));
+        chunkqueue_steal(hctx->wb, req_cq, req_cqlen);
+
+        chunkqueue_append_mem_min(hctx->wb, CONST_STR_LEN("\r\n"));
+    }
+
+    if (hctx->wb->bytes_in == hctx->wb_reqlen) {/*hctx->r->reqbody_length >= 0*/
+        /* terminate STDIN */
+        chunkqueue_append_mem(hctx->wb, CONST_STR_LEN("0\r\n\r\n"));
+        hctx->wb_reqlen += (int)sizeof("0\r\n\r\n");
+    }
+
+    return HANDLER_GO_ON;
+}
+
+
 static handler_t proxy_create_env(gw_handler_ctx *gwhctx) {
 	handler_ctx *hctx = (handler_ctx *)gwhctx;
 	request_st * const r = hctx->gw.r;
@@ -831,7 +867,13 @@ static handler_t proxy_create_env(gw_handler_ctx *gwhctx) {
 	buffer_append_string_buffer(b, &r->target);
 	if (remap_headers)
 		http_header_remap_uri(b, buffer_string_length(b) - buffer_string_length(&r->target), &hctx->conf.header, 1);
-	if (!upgrade)
+
+	int stream_chunked = 0;
+	if (-1 == r->reqbody_length && r->conf.stream_request_body) {
+		stream_chunked = 1;
+		hctx->gw.stdin_append = proxy_stdin_append;
+	}
+	if (!stream_chunked && !upgrade)
 		buffer_append_string_len(b, CONST_STR_LEN(" HTTP/1.0\r\n"));
 	else
 		buffer_append_string_len(b, CONST_STR_LEN(" HTTP/1.1\r\n"));
@@ -869,6 +911,8 @@ static handler_t proxy_create_env(gw_handler_ctx *gwhctx) {
 			                        buf, li_itostrn(buf, sizeof(buf), r->reqbody_length));
 		}
 	}
+	else if (stream_chunked)
+		buffer_append_string_len(b, CONST_STR_LEN("Transfer-Encoding: chunked\r\n"));
 
 	/* request header */
 	for (size_t i = 0, used = r->rqst_headers.used; i < used; ++i) {