Fix #1508, http-client support read chunked response. 3.0.76

README.md

@@ -145,7 +145,8 @@ For previous versions, please read:
 
 ## V3 changes
 
-* v3.0, 2019-12-20, For [#1508][bug #1508], Refactor srs_is_digital, support all zeros.
+* v3.0, 2019-12-20, Fix [#1508][bug #1508], http-client support read chunked response. 3.0.76
+* v3.0, 2019-12-20, For [#1508][bug #1508], refactor srs_is_digital, support all zeros.
 * <strong>v3.0, 2019-12-19, [3.0 alpha5(3.0.75)][r3.0a5] released. 115362 lines.</strong>
 * v3.0, 2019-12-19, Refine the RTMP iovs cache increasing to much faster.
 * v3.0, 2019-12-19, Fix [#1524][bug #1524], memory leak for amf0 strict array. 3.0.75

trunk/src/core/srs_core.hpp

@@ -27,7 +27,7 @@
 // The version config.
 #define VERSION_MAJOR       3
 #define VERSION_MINOR       0
-#define VERSION_REVISION    75
+#define VERSION_REVISION    76
 
 // The macros generated by configure script.
 #include <srs_auto_headers.hpp>

trunk/src/service/srs_service_http_conn.cpp

@@ -40,6 +40,8 @@ SrsHttpParser::SrsHttpParser()
 {
     buffer = new SrsFastStream();
     header = NULL;
+
+    p_body_start = p_header_tail = NULL;
 }
 
 SrsHttpParser::~SrsHttpParser()
@@ -80,7 +82,7 @@ srs_error_t SrsHttpParser::parse_message(ISrsReader* reader, ISrsHttpMessage** p
     state = SrsHttpParseStateInit;
     hp_header = http_parser();
     // The body that we have read from cache.
-    pbody = NULL;
+    p_body_start = p_header_tail = NULL;
     // We must reset the field name and value, because we may get a partial value in on_header_value.
     field_name = field_value = "";
     // The header of the request.
@@ -115,19 +117,26 @@ srs_error_t SrsHttpParser::parse_message_imp(ISrsReader* reader)
     
     while (true) {
         if (buffer->size() > 0) {
-            ssize_t nparsed = http_parser_execute(&parser, &settings, buffer->bytes(), buffer->size());
+            ssize_t consumed = http_parser_execute(&parser, &settings, buffer->bytes(), buffer->size());
 
             // The error is set in http_errno.
             enum http_errno code;
 	        if ((code = HTTP_PARSER_ERRNO(&parser)) != HPE_OK) {
 	            return srs_error_new(ERROR_HTTP_PARSE_HEADER, "parse %dB, nparsed=%d, err=%d/%s %s",
-	                buffer->size(), nparsed, http_errno_name(code), http_errno_description(code));
+	                buffer->size(), consumed, http_errno_name(code), http_errno_description(code));
 	        }
 
-			// The consumed size, does not include the body.
+            // When buffer consumed these bytes, it's dropped so the new ptr is actually the HTTP body. But http-parser
-	        ssize_t consumed = nparsed;
+            // doesn't indicate the specific sizeof header, so we must finger it out.
-	        if (pbody && buffer->bytes() < pbody) {
+            // @remark We shouldn't use on_body, because it only works for normal case, and losts the chunk header and length.
-	           consumed = pbody - buffer->bytes();
+            // @see https://github.com/ossrs/srs/issues/1508
+	        if (p_header_tail && buffer->bytes() < p_body_start) {
+	            for (const char* p = p_header_tail; p <= p_body_start - 4; p++) {
+	                if (p[0] == SRS_CONSTS_CR && p[1] == SRS_CONSTS_LF && p[2] == SRS_CONSTS_CR && p[3] == SRS_CONSTS_LF) {
+	                    consumed = p + 4 - buffer->bytes();
+	                    break;
+	                }
+	            }
 	        }
             srs_info("size=%d, nparsed=%d, consumed=%d", buffer->size(), (int)nparsed, consumed);
 
@@ -203,6 +212,11 @@ int SrsHttpParser::on_url(http_parser* parser, const char* at, size_t length)
     if (length > 0) {
         obj->url = string(at, (int)length);
     }
+
+    // When header parsed, we must save the position of start for body,
+    // because we have to consume the header in buffer.
+    // @see https://github.com/ossrs/srs/issues/1508
+    obj->p_header_tail = at;
     
     srs_info("Method: %d, Url: %.*s", parser->method, (int)length, at);
     
@@ -222,6 +236,11 @@ int SrsHttpParser::on_header_field(http_parser* parser, const char* at, size_t l
     if (length > 0) {
         obj->field_name.append(at, (int)length);
     }
+
+    // When header parsed, we must save the position of start for body,
+    // because we have to consume the header in buffer.
+    // @see https://github.com/ossrs/srs/issues/1508
+    obj->p_header_tail = at;
     
     srs_info("Header field(%d bytes): %.*s", (int)length, (int)length, at);
     return 0;
@@ -235,6 +254,11 @@ int SrsHttpParser::on_header_value(http_parser* parser, const char* at, size_t l
     if (length > 0) {
         obj->field_value.append(at, (int)length);
     }
+
+    // When header parsed, we must save the position of start for body,
+    // because we have to consume the header in buffer.
+    // @see https://github.com/ossrs/srs/issues/1508
+    obj->p_header_tail = at;
     
     srs_info("Header value(%d bytes): %.*s", (int)length, (int)length, at);
     return 0;
@@ -248,9 +272,10 @@ int SrsHttpParser::on_body(http_parser* parser, const char* at, size_t length)
     // save the parser when body parsed.
     obj->state = SrsHttpParseStateBody;
 
-    // Save the body position.
+    // Used to discover the header length.
-    obj->pbody = at;
+    // @see https://github.com/ossrs/srs/issues/1508
-    
+    obj->p_body_start = at;
+
     srs_info("Body: %.*s", (int)length, at);
     
     return 0;

trunk/src/service/srs_service_http_conn.hpp

@@ -55,7 +55,11 @@ private:
     http_parser hp_header;
     std::string url;
     SrsHttpHeader* header;
-    const char* pbody;
+private:
+    // Point to the start of body.
+    const char* p_body_start;
+    // To discover the length of header, point to the last few bytes in header.
+    const char* p_header_tail;
 public:
     SrsHttpParser();
     virtual ~SrsHttpParser();