request: be more liberal about transfer-encoding value

For instance, the following header may be considered as valid:
Transfer-Encoding: chunked, chunked

Introduces a new helper function to recognize header value tokens

Ticket: #6415

htp/htp_private.h

@@ -248,6 +248,8 @@ htp_status_t htp_tx_urldecode_params_inplace(htp_tx_t *tx, bstr *input);
 
 void htp_connp_destroy_decompressors(htp_connp_t *connp);
 
+htp_status_t htp_header_has_token(const unsigned char *hvp, size_t hvlen, const unsigned char *value);
+
 #ifndef HAVE_STRLCAT
 size_t strlcat(char *dst, const char *src, size_t size);
 #endif

htp/htp_transaction.c

@@ -405,7 +405,7 @@ static htp_status_t htp_tx_process_request_headers(htp_tx_t *tx) {
         //      (2.2.22 on Ubuntu 12.04 LTS) instead errors out with "Unknown Transfer-Encoding: identity".
         //      And it behaves strangely, too, sending a 501 and proceeding to process the request
         //      (e.g., PHP is run), but without the body. It then closes the connection.
-        if (bstr_cmp_c_nocase(te->value, "chunked") != 0) {
+        if (htp_header_has_token(bstr_ptr(te->value), bstr_len(te->value), (unsigned char*) "chunked") != HTP_OK) {
             // Invalid T-E header value.
             tx->request_transfer_coding = HTP_CODING_INVALID;
             tx->flags |= HTP_REQUEST_INVALID_T_E;

htp/htp_util.c

@@ -2543,3 +2543,60 @@ htp_uri_t *htp_uri_alloc(void) {
 char *htp_get_version(void) {
     return HTP_VERSION_STRING_FULL;
 }
+
+/**
+ * Tells if a header value (haystack) contains a token (needle)
+ * This is done with a caseless comparison
+ *
+ * @param[in] hvp header value pointer
+ * @param[in] hvlen length of header value buffer
+ * @param[in] value token to look for (null-terminated string), should be a lowercase constant
+ * @return HTP_OK if the header has the token; HTP_ERROR if it has not.
+ */
+htp_status_t htp_header_has_token(const unsigned char *hvp, size_t hvlen, const unsigned char *value) {
+    int state = 0;
+    // offset to compare in value
+    size_t v_off = 0;
+    // The header value is a list of comma-separated tokens (with additional spaces)
+    for (size_t i = 0; i < hvlen; i++) {
+        switch (state) {
+            case 0:
+                if (v_off == 0 && htp_is_space(hvp[i])) {
+                    // skip leading space
+                    continue;
+                }
+                if (tolower(hvp[i]) == value[v_off]) {
+                    v_off++;
+                    if (value[v_off] == 0) {
+                        // finish validation if end of token
+                        state = 2;
+                    }
+                    continue;
+                } else {
+                    // wait for a new token
+                    v_off = 0;
+                    state = 1;
+                }
+                // fallthrough
+            case 1:
+                if (hvp[i] == ',') {
+                    // start of next token
+                    state = 0;
+                }
+                break;
+            case 2:
+                if (hvp[i] == ',') {
+                    return HTP_OK;
+                }
+                if (!htp_is_space(hvp[i])) {
+                    // trailing junk in token, wait for a next one
+                    v_off = 0;
+                    state = 1;
+                }
+        }
+    }
+    if (state == 2) {
+        return HTP_OK;
+    }
+    return HTP_ERROR;
+}

test/test_utils.cpp

@@ -1801,3 +1801,39 @@ TEST_F(UrlencodedParser, UrlDecode1) {
     ASSERT_EQ(0, bstr_cmp_c(s, "/one/two/three/%3"));
     bstr_free(s);
 }
+
+TEST(UtilTest, HeaderHasToken) {
+    char data[100];
+
+    // Basic
+    strcpy(data, "chunked");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+
+    // Negative
+    strcpy(data, "notchunked");
+    EXPECT_EQ(HTP_ERROR, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, "chunkednot");
+    EXPECT_EQ(HTP_ERROR, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, "chunk,ed");
+    EXPECT_EQ(HTP_ERROR, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, "chunk ed");
+    EXPECT_EQ(HTP_ERROR, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+
+    // Positive
+    strcpy(data, " notchunked , chunked , yetanother");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, "chunked,yetanother");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, "not,chunked");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, "chunk,chunked");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, " chunked");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, "chunked ");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, "chunked,");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+    strcpy(data, ",chunked");
+    EXPECT_EQ(HTP_OK, htp_header_has_token((unsigned char*) data, strlen(data), (unsigned char *)"chunked"));
+}