diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/ChangeLog libhtp-0.5.32.201912141440~ubuntu20.04.1/ChangeLog --- libhtp-0.5.31.201910241325~ubuntu20.04.1/ChangeLog 2019-10-24 13:25:55.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/ChangeLog 2019-12-14 14:40:11.000000000 +0000 @@ -1,3 +1,8 @@ +0.5.32 (13 December 2019) +-------------------------- + +- bug fixes around pipelining + 0.5.31 (24 September 2019) -------------------------- diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/debian/bzr-builder.manifest libhtp-0.5.32.201912141440~ubuntu20.04.1/debian/bzr-builder.manifest --- libhtp-0.5.31.201910241325~ubuntu20.04.1/debian/bzr-builder.manifest 2019-10-24 13:26:08.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/debian/bzr-builder.manifest 2019-12-14 14:40:19.000000000 +0000 @@ -1,3 +1,3 @@ -# bzr-builder format 0.3 deb-version 1:0.5.31.201910241325 -lp:~oisf/suricata-daily-git-libhtp-0.5.x/libhtp revid:git-v1:f0119bbdb7627b33a2ff1a18cf3c439b96606a14 -nest-part packaging lp:~oisf/suricata-daily-git-test/suricata-packaging-git libhtp/debian debian revid:petermanev@gmail.com-20191013155530-et3nyx6oj423imvv +# bzr-builder format 0.3 deb-version 1:0.5.32.201912141440 +lp:~oisf/suricata-daily-git-libhtp-0.5.x/libhtp revid:git-v1:aff5fb54b5eaca4b82a097a6efc4c421c0962a15 +nest-part packaging lp:~oisf/suricata-daily-git-test/suricata-packaging-git libhtp/debian debian revid:petermanev@gmail.com-20191210204031-aom8wre7sfjh39zq diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/debian/changelog libhtp-0.5.32.201912141440~ubuntu20.04.1/debian/changelog --- libhtp-0.5.31.201910241325~ubuntu20.04.1/debian/changelog 2019-10-24 13:26:08.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/debian/changelog 2019-12-14 14:40:19.000000000 +0000 @@ -1,8 +1,8 @@ -libhtp (1:0.5.31.201910241325~ubuntu20.04.1) focal; urgency=low +libhtp (1:0.5.32.201912141440~ubuntu20.04.1) focal; urgency=low * Auto build. - -- Peter Manev Thu, 24 Oct 2019 13:26:08 +0000 + -- Peter Manev Sat, 14 Dec 2019 14:40:19 +0000 htp (0.2.13-2ubuntu2) precise; urgency=low diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/htp/htp_connection_parser.c libhtp-0.5.32.201912141440~ubuntu20.04.1/htp/htp_connection_parser.c --- libhtp-0.5.31.201910241325~ubuntu20.04.1/htp/htp_connection_parser.c 2019-10-24 13:25:55.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/htp/htp_connection_parser.c 2019-12-14 14:40:11.000000000 +0000 @@ -44,6 +44,18 @@ connp->last_error = NULL; } +void htp_connp_req_close(htp_connp_t *connp, const htp_time_t *timestamp) { + if (connp == NULL) return; + + // Update internal flags + if (connp->in_status != HTP_STREAM_ERROR) + connp->in_status = HTP_STREAM_CLOSED; + + // Call the parsers one last time, which will allow them + // to process the events that depend on stream closure + htp_connp_req_data(connp, timestamp, NULL, 0); +} + void htp_connp_close(htp_connp_t *connp, const htp_time_t *timestamp) { if (connp == NULL) return; diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/htp/htp_connection_parser.h libhtp-0.5.32.201912141440~ubuntu20.04.1/htp/htp_connection_parser.h --- libhtp-0.5.31.201910241325~ubuntu20.04.1/htp/htp_connection_parser.h 2019-10-24 13:25:55.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/htp/htp_connection_parser.h 2019-12-14 14:40:11.000000000 +0000 @@ -57,6 +57,7 @@ * @param[in] timestamp Optional. */ void htp_connp_close(htp_connp_t *connp, const htp_time_t *timestamp); +void htp_connp_req_close(htp_connp_t *connp, const htp_time_t *timestamp); /** * Creates a new connection parser using the provided configuration. Because diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/htp/htp_request.c libhtp-0.5.32.201912141440~ubuntu20.04.1/htp/htp_request.c --- libhtp-0.5.31.201910241325~ubuntu20.04.1/htp/htp_request.c 2019-10-24 13:25:55.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/htp/htp_request.c 2019-12-14 14:40:11.000000000 +0000 @@ -828,40 +828,73 @@ } htp_status_t htp_connp_REQ_FINALIZE(htp_connp_t *connp) { - size_t bytes_left = connp->in_current_len - connp->in_current_read_offset; - - if (bytes_left > 0) { - // If we have more bytes - // Either it is request pipelining - // Or we interpret it as body data - int64_t pos = connp->in_current_read_offset; - int64_t mstart = 0; - // skip past leading whitespace. IIS allows this - while ((pos < connp->in_current_len) && htp_is_space(connp->in_current_data[pos])) - pos++; - if (pos < connp->in_current_len) { - mstart = pos; - // The request method starts at the beginning of the - // line and ends with the first whitespace character. - while ((pos < connp->in_current_len) && (!htp_is_space(connp->in_current_data[pos]))) - pos++; - - int methodi = HTP_M_UNKNOWN; - bstr *method = bstr_dup_mem(connp->in_current_data + mstart, pos - mstart); - if (method) { - methodi = htp_convert_method_to_number(method); - bstr_free(method); - } - if (methodi == HTP_M_UNKNOWN) { - // Interpret remaining bytes as body data - htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected request body"); - connp->in_tx->request_progress = HTP_REQUEST_BODY; - connp->in_state = htp_connp_REQ_BODY_IDENTITY; - connp->in_body_data_left = bytes_left; - return HTP_OK; + if (connp->in_status != HTP_STREAM_CLOSED) { + IN_PEEK_NEXT(connp); + if (connp->in_next_byte == -1) { + return htp_tx_state_request_complete(connp->in_tx); + } + if (connp->in_next_byte != LF || connp->in_current_consume_offset >= connp->in_current_read_offset) { + for (;;) {//;i < max_read; i++) { + IN_COPY_BYTE_OR_RETURN(connp); + // Have we reached the end of the line? For some reason + // we can't test after IN_COPY_BYTE_OR_RETURN */ + if (connp->in_next_byte == LF) + break; } } } + + unsigned char *data; + size_t len; + if (htp_connp_req_consolidate_data(connp, &data, &len) != HTP_OK) { + return HTP_ERROR; + } +#ifdef HTP_DEBUG + fprint_raw_data(stderr, "PROBING request finalize", data, len); +#endif + if (len == 0) { + //closing + return htp_tx_state_request_complete(connp->in_tx); + } + + size_t pos = 0; + size_t mstart = 0; + // skip past leading whitespace. IIS allows this + while ((pos < len) && htp_is_space(data[pos])) + pos++; + if (pos) + mstart = pos; + // The request method starts at the beginning of the + // line and ends with the first whitespace character. + while ((pos < len) && (!htp_is_space(data[pos]))) + pos++; + + if (pos > mstart) { + int methodi = HTP_M_UNKNOWN; + bstr *method = bstr_dup_mem(data + mstart, pos - mstart); + if (method) { + methodi = htp_convert_method_to_number(method); + bstr_free(method); + } + if (methodi == HTP_M_UNKNOWN) { + // Interpret remaining bytes as body data + htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected request body"); + htp_status_t rc = htp_tx_req_process_body_data_ex(connp->in_tx, data, len); + htp_connp_req_clear_buffer(connp); + return rc; + } + } + //else + //unread last end of line so that REQ_LINE works + if (connp->in_current_read_offset < (int64_t)len) { + connp->in_current_read_offset=0; + } else { + connp->in_current_read_offset-=len; + } + if (connp->in_current_read_offset < connp->in_current_consume_offset) { + connp->in_current_consume_offset=connp->in_current_read_offset; + } + return htp_tx_state_request_complete(connp->in_tx); } diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/htp/htp_response.c libhtp-0.5.32.201912141440~ubuntu20.04.1/htp/htp_response.c --- libhtp-0.5.31.201910241325~ubuntu20.04.1/htp/htp_response.c 2019-10-24 13:25:55.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/htp/htp_response.c 2019-12-14 14:40:11.000000000 +0000 @@ -1070,21 +1070,52 @@ } htp_status_t htp_connp_RES_FINALIZE(htp_connp_t *connp) { - int bytes_left = connp->out_current_len - connp->out_current_read_offset; - unsigned char * data = connp->out_current_data + connp->out_current_read_offset; + if (connp->out_status != HTP_STREAM_CLOSED) { + OUT_PEEK_NEXT(connp); + if (connp->out_next_byte == -1) { + return htp_tx_state_response_complete_ex(connp->out_tx, 0); + } + if (connp->out_next_byte != LF || connp->out_current_consume_offset >= connp->out_current_read_offset) { + for (;;) {//;i < max_read; i++) { + OUT_COPY_BYTE_OR_RETURN(connp); + // Have we reached the end of the line? For some reason + // we can't test after IN_COPY_BYTE_OR_RETURN */ + if (connp->out_next_byte == LF) + break; + } + } + } + size_t bytes_left; + unsigned char * data; + + if (htp_connp_res_consolidate_data(connp, &data, &bytes_left) != HTP_OK) { + return HTP_ERROR; + } +#ifdef HTP_DEBUG + fprint_raw_data(stderr, "PROBING response finalize", data, bytes_left); +#endif + if (bytes_left == 0) { + //closing + return htp_tx_state_response_complete_ex(connp->out_tx, 0); + } - if (bytes_left > 0 && - htp_treat_response_line_as_body(data, bytes_left)) { + if (htp_treat_response_line_as_body(data, bytes_left)) { // Interpret remaining bytes as body data htp_log(connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "Unexpected response body"); - connp->out_current_read_offset += bytes_left; - connp->out_current_consume_offset += bytes_left; - connp->out_stream_offset += bytes_left; - connp->out_body_data_left -= bytes_left; htp_status_t rc = htp_tx_res_process_body_data_ex(connp->out_tx, data, bytes_left); + htp_connp_res_clear_buffer(connp); return rc; } + //unread last end of line so that RES_LINE works + if (connp->out_current_read_offset < (int64_t)bytes_left) { + connp->out_current_read_offset=0; + } else { + connp->out_current_read_offset-=bytes_left; + } + if (connp->out_current_read_offset < connp->out_current_consume_offset) { + connp->out_current_consume_offset=connp->out_current_read_offset; + } return htp_tx_state_response_complete_ex(connp->out_tx, 0 /* not hybrid mode */); } @@ -1112,6 +1143,10 @@ connp->out_tx = htp_list_get(connp->conn->transactions, connp->out_next_tx_index); if (connp->out_tx == NULL) { htp_log(connp, HTP_LOG_MARK, HTP_LOG_ERROR, 0, "Unable to match response to request"); + // finalize dangling request waiting for next request or body + if (connp->in_state == htp_connp_REQ_FINALIZE) { + htp_tx_state_request_complete(connp->in_tx); + } connp->out_tx = htp_connp_tx_create(connp); if (connp->out_tx == NULL) { return HTP_ERROR; diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/test/files/91-request-unexpected-body.t libhtp-0.5.32.201912141440~ubuntu20.04.1/test/files/91-request-unexpected-body.t --- libhtp-0.5.31.201910241325~ubuntu20.04.1/test/files/91-request-unexpected-body.t 2019-10-24 13:25:55.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/test/files/91-request-unexpected-body.t 2019-12-14 14:40:11.000000000 +0000 @@ -4,6 +4,7 @@ Content-Type: application/x-www-form-urlencoded login=foo&password=bar + <<< HTTP/1.1 200 OK Content-Length: 0 diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/test/files/97-requests-cut.t libhtp-0.5.32.201912141440~ubuntu20.04.1/test/files/97-requests-cut.t --- libhtp-0.5.31.201910241325~ubuntu20.04.1/test/files/97-requests-cut.t 1970-01-01 00:00:00.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/test/files/97-requests-cut.t 2019-12-14 14:40:11.000000000 +0000 @@ -0,0 +1,9 @@ +>>> +GET /?p=%20 HTTP/1.1 +User-Agent: Mozilla + +G +>>> +ET /?p=%21 HTTP/1.1 +User-Agent: Mozilla + diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/test/files/98-responses-cut.t libhtp-0.5.32.201912141440~ubuntu20.04.1/test/files/98-responses-cut.t --- libhtp-0.5.31.201910241325~ubuntu20.04.1/test/files/98-responses-cut.t 1970-01-01 00:00:00.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/test/files/98-responses-cut.t 2019-12-14 14:40:11.000000000 +0000 @@ -0,0 +1,26 @@ +>>> +GET /?p=%20 HTTP/1.1 +User-Agent: Mozilla + +GET /?p=%21 HTTP/1.1 +User-Agent: Mozilla + +<<< +HTTP/1.0 200 OK +Date: Mon, 31 Aug 2009 20:25:50 GMT +Server: Apache +Connection: close +Content-Type: text/html +Content-Length: 14 + +Hello World! +H +<<< +TTP/1.0 200 OK +Date: Mon, 31 Aug 2009 20:25:50 GMT +Server: Apache +Connection: close +Content-Type: text/html +Content-Length: 13 + +Hello People! \ No newline at end of file diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/test/test_main.cpp libhtp-0.5.32.201912141440~ubuntu20.04.1/test/test_main.cpp --- libhtp-0.5.31.201910241325~ubuntu20.04.1/test/test_main.cpp 2019-10-24 13:25:55.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/test/test_main.cpp 2019-12-14 14:40:11.000000000 +0000 @@ -1483,7 +1483,7 @@ htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1); ASSERT_TRUE(tx != NULL); - ASSERT_EQ(1, tx->request_ignored_lines); + /*part of previous request body ASSERT_EQ(1, tx->request_ignored_lines);*/ } TEST_F(ConnectionParsing, PostNoBody) { @@ -1594,7 +1594,7 @@ htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0); ASSERT_TRUE(tx != NULL); - ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + //error first ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); ASSERT_EQ(HTP_RESPONSE_HEADERS, tx->response_progress); } @@ -2032,3 +2032,39 @@ ASSERT_EQ(68, tx->response_entity_len); } #endif + +TEST_F(ConnectionParsing, RequestsCut) { + int rc = test_run(home, "97-requests-cut.t", cfg, &connp); + ASSERT_GE(rc, 0); + + ASSERT_EQ(2, htp_list_size(connp->conn->transactions)); + htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + + tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); +} + +TEST_F(ConnectionParsing, ResponsesCut) { + int rc = test_run(home, "98-responses-cut.t", cfg, &connp); + ASSERT_GE(rc, 0); + + ASSERT_EQ(2, htp_list_size(connp->conn->transactions)); + htp_tx_t *tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 0); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + ASSERT_EQ(200, tx->response_status_number); + ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress); + + tx = (htp_tx_t *) htp_list_get(connp->conn->transactions, 1); + ASSERT_TRUE(tx != NULL); + ASSERT_EQ(0, bstr_cmp_c(tx->request_method, "GET")); + ASSERT_EQ(HTP_REQUEST_COMPLETE, tx->request_progress); + ASSERT_EQ(200, tx->response_status_number); + ASSERT_EQ(HTP_RESPONSE_COMPLETE, tx->response_progress); +} diff -Nru libhtp-0.5.31.201910241325~ubuntu20.04.1/VERSION libhtp-0.5.32.201912141440~ubuntu20.04.1/VERSION --- libhtp-0.5.31.201910241325~ubuntu20.04.1/VERSION 2019-10-24 13:25:55.000000000 +0000 +++ libhtp-0.5.32.201912141440~ubuntu20.04.1/VERSION 2019-12-14 14:40:11.000000000 +0000 @@ -1,2 +1,2 @@ # This file is intended to be sourced by sh -PKG_VERSION=0.5.31 +PKG_VERSION=0.5.32