Skip to content

v2: Allow partial processing of XML, JSON, and multipart request body #3483

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hnakamur wants to merge 25 commits into
base: v2/master
Choose a base branch
from
+2,716 −152
Open

v2: Allow partial processing of XML, JSON, and multipart request body #3483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
a682f13
Allow partial processing of multipart, JSON, and XML request body
hnakamur Dec 25, 2025
55270e5
Adjust test since ProcessPartial no more causes "no final boundary mi…
hnakamur Dec 30, 2025
586e1ef
Fix json test match_log
hnakamur Dec 30, 2025
e9dca3c
Fix expected error message in regression test
hnakamur Dec 24, 2025
0e4728f
Accept partial epilogue in body larger than limit for ProcessPartial
hnakamur Dec 31, 2025
24508d6
Fix indent in apache2/msc_multipart.c
hnakamur Dec 31, 2025
bea943b
Fix indent in apache2/msc_json.c
hnakamur Dec 31, 2025
9b2a5fe
Add tests for url-encoded, JSON, and XML with ProcessPartial
hnakamur Jan 1, 2026
0bfb828
Refine tests for multipart with ProcessPartial
hnakamur Jan 1, 2026
43f95fd
Modify url-encoded reqbody tests for ProcesPartial
hnakamur Jan 2, 2026
3d73c02
Support partial processing of url-encoded reqbody
hnakamur Jan 2, 2026
d914f23
Add tests for MULTIPART_PART_HEADERS with ProcessPartial
hnakamur Jan 3, 2026
55a1828
Reject invalid final boundary for multipart with ProcessPartial
hnakamur Jan 3, 2026
0b599ad
Fix indent in apache2/msc_multipart.c
hnakamur Jan 3, 2026
cb95a24
Modify tests for multipart with ProcessPartial
hnakamur Jan 3, 2026
61d4e45
Modify multipart_complete for incomplete final boundary
hnakamur Jan 3, 2026
be5feee
Process an incomplete boundary as a non-final boundary
hnakamur Jan 3, 2026
50de8bb
Update tests/regression/rule/15-json.t
hnakamur Jan 23, 2026
a64b544
Update tests/regression/rule/15-json.t
hnakamur Jan 23, 2026
a83c26c
Fix spelling of reqbody_partial_processing_enabled
hnakamur Jan 23, 2026
da8b174
Fix indentations in test files
hnakamur Jan 23, 2026
f679e11
Add rules to check multipart error in tests
hnakamur Jan 23, 2026
9277589
Fix indentations in tests/regression/rule/10-xml.t
hnakamur Jan 23, 2026
96a8326
Make handling of SecRequestBodyNoFilesLimit consistent
hnakamur Jan 23, 2026
2681c70
Add tests for long body
hnakamur Jan 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 16 additions & 26 deletions apache2/apache2_io.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,38 +299,25 @@ apr_status_t read_request_body(modsec_rec *msr, char **error_msg) {
#endif
}

if (msr->reqbody_length + buflen > (apr_size_t)msr->txcfg->reqbody_limit && msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL) {
buflen = (apr_size_t)msr->txcfg->reqbody_limit - msr->reqbody_length;
finished_reading = 1;
modsecurity_request_body_enable_partial_processing(msr);
}

msr->reqbody_length += buflen;

if (buflen != 0) {
int rcbs = modsecurity_request_body_store(msr, buf, buflen, error_msg);

if (msr->reqbody_length > (apr_size_t)msr->txcfg->reqbody_limit && msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL) {
finished_reading = 1;
}

if (rcbs < 0) {
if (rcbs == -5) {
if((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
*error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
"configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
return HTTP_REQUEST_ENTITY_TOO_LARGE;
} else if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL)) {
*error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
"configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
} else if ((msr->txcfg->is_enabled == MODSEC_DETECTION_ONLY) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_PARTIAL)) {
*error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
"configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
} else {
*error_msg = apr_psprintf(msr->mp, "Request body no files data length is larger than the "
"configured limit (%ld).", msr->txcfg->reqbody_no_files_limit);
return HTTP_REQUEST_ENTITY_TOO_LARGE;
}
return HTTP_REQUEST_ENTITY_TOO_LARGE;
}

if((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT))
if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
return HTTP_INTERNAL_SERVER_ERROR;
}
}

}

if (APR_BUCKET_IS_EOS(bucket)) {
Expand All @@ -351,11 +338,14 @@ apr_status_t read_request_body(modsec_rec *msr, char **error_msg) {

msr->if_status = IF_STATUS_WANTS_TO_RUN;

if (rcbe == -5) {
return HTTP_REQUEST_ENTITY_TOO_LARGE;
}
if (rcbe < 0) {
return HTTP_INTERNAL_SERVER_ERROR;
if (rcbe == -5) {
return HTTP_REQUEST_ENTITY_TOO_LARGE;
}

if ((msr->txcfg->is_enabled == MODSEC_ENABLED) && (msr->txcfg->if_limit_action == REQUEST_BODY_LIMIT_ACTION_REJECT)) {
return HTTP_INTERNAL_SERVER_ERROR;
}
}
return APR_SUCCESS;
}
Expand Down
3 changes: 3 additions & 0 deletions apache2/modsecurity.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ struct modsec_rec {
unsigned int if_started_forwarding;

apr_size_t reqbody_length;
unsigned int reqbody_partial_processing_enabled;

apr_bucket_brigade *of_brigade;
unsigned int of_status;
Expand Down Expand Up @@ -736,6 +737,8 @@ apr_status_t DSOLOCAL modsecurity_request_body_start(modsec_rec *msr, char **err
apr_status_t DSOLOCAL modsecurity_request_body_store(modsec_rec *msr,
const char *data, apr_size_t length, char **error_msg);

void DSOLOCAL modsecurity_request_body_enable_partial_processing(modsec_rec *msr);

apr_status_t DSOLOCAL modsecurity_request_body_end(modsec_rec *msr, char **error_msg);

apr_status_t DSOLOCAL modsecurity_request_body_to_stream(modsec_rec *msr, const char *buffer, int buflen, char **error_msg);
Expand Down
40 changes: 22 additions & 18 deletions apache2/msc_json.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ static int yajl_start_array(void *ctx) {
msr->json->current_depth++;
if (msr->json->current_depth > msr->txcfg->reqbody_json_depth_limit) {
msr->json->depth_limit_exceeded = 1;
return 0;
return 0;
}

if (msr->txcfg->debuglog_level >= 9) {
Expand Down Expand Up @@ -262,7 +262,7 @@ static int yajl_start_map(void *ctx)
msr->json->current_depth++;
if (msr->json->current_depth > msr->txcfg->reqbody_json_depth_limit) {
msr->json->depth_limit_exceeded = 1;
return 0;
return 0;
}

if (msr->txcfg->debuglog_level >= 9) {
Expand Down Expand Up @@ -367,6 +367,10 @@ int json_init(modsec_rec *msr, char **error_msg) {
return 1;
}

void json_allow_partial_values(modsec_rec *msr) {
(void)yajl_config(msr->json->handle, yajl_allow_partial_values, 1);
}

/**
* Feed one chunk of data to the JSON parser.
*/
Expand All @@ -380,16 +384,16 @@ int json_process_chunk(modsec_rec *msr, const char *buf, unsigned int size, char
/* Feed our parser and catch any errors */
msr->json->status = yajl_parse(msr->json->handle, buf, size);
if (msr->json->status != yajl_status_ok) {
if (msr->json->depth_limit_exceeded) {
*error_msg = "JSON depth limit exceeded";
} else {
if (msr->json->yajl_error) *error_msg = msr->json->yajl_error;
else {
char* yajl_err = yajl_get_error(msr->json->handle, 0, buf, size);
*error_msg = apr_pstrdup(msr->mp, yajl_err);
yajl_free_error(msr->json->handle, yajl_err);
if (msr->json->depth_limit_exceeded) {
*error_msg = "JSON depth limit exceeded";
} else {
if (msr->json->yajl_error) *error_msg = msr->json->yajl_error;
else {
char* yajl_err = yajl_get_error(msr->json->handle, 0, buf, size);
*error_msg = apr_pstrdup(msr->mp, yajl_err);
yajl_free_error(msr->json->handle, yajl_err);
}
}
}
return -1;
}

Expand All @@ -409,13 +413,13 @@ int json_complete(modsec_rec *msr, char **error_msg) {
/* Wrap up the parsing process */
msr->json->status = yajl_complete_parse(msr->json->handle);
if (msr->json->status != yajl_status_ok) {
if (msr->json->depth_limit_exceeded) {
*error_msg = "JSON depth limit exceeded";
} else {
char *yajl_err = yajl_get_error(msr->json->handle, 0, NULL, 0);
*error_msg = apr_pstrdup(msr->mp, yajl_err);
yajl_free_error(msr->json->handle, yajl_err);
}
if (msr->json->depth_limit_exceeded) {
*error_msg = "JSON depth limit exceeded";
} else {
char *yajl_err = yajl_get_error(msr->json->handle, 0, NULL, 0);
*error_msg = apr_pstrdup(msr->mp, yajl_err);
yajl_free_error(msr->json->handle, yajl_err);
}

return -1;
}
Expand Down
2 changes: 2 additions & 0 deletions apache2/msc_json.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ struct json_data {

int DSOLOCAL json_init(modsec_rec *msr, char **error_msg);

void DSOLOCAL json_allow_partial_values(modsec_rec *msr);

int DSOLOCAL json_process(modsec_rec *msr, const char *buf,
unsigned int size, char **error_msg);

Expand Down
110 changes: 83 additions & 27 deletions apache2/msc_multipart.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ static int multipart_process_part_header(modsec_rec *msr, char **error_msg) {
if (data == msr->mpd->buf) {
*error_msg = apr_psprintf(msr->mp, "Multipart: Invalid part header (header name missing).");

return -1;
return -1;
}

/* check if multipart header contains any invalid characters */
Expand Down Expand Up @@ -1023,38 +1023,94 @@ int multipart_complete(modsec_rec *msr, char **error_msg) {
* processed yet) in the buffer.
*/
if (msr->mpd->buf_contains_line) {
if ( ((unsigned int)(MULTIPART_BUF_SIZE - msr->mpd->bufleft) == (4 + strlen(msr->mpd->boundary)))
/*
* Note that the buffer may end with the final boundary followed by only CR,
* coming from the [CRLF epilogue], when allow_process_partial == 1 (which is
* set when SecRequestBodyLimitAction is ProcessPartial and the request body
* length exceeds SecRequestBodyLimit).
*
* The following definitions are copied from RFC 2046:
*
* dash-boundary := "--" boundary
*
* delimiter := CRLF dash-boundary
*
* close-delimiter := delimiter "--"
*
* multipart-body := [preamble CRLF]
* dash-boundary transport-padding CRLF
* body-part *encapsulation
* close-delimiter transport-padding
* [CRLF epilogue]
*/
unsigned int buf_data_len = (unsigned int)(MULTIPART_BUF_SIZE - msr->mpd->bufleft);
size_t boundary_len = strlen(msr->mpd->boundary);
if ( (buf_data_len >= 2 + boundary_len)
&& (*(msr->mpd->buf) == '-')
&& (*(msr->mpd->buf + 1) == '-')
&& (strncmp(msr->mpd->buf + 2, msr->mpd->boundary, strlen(msr->mpd->boundary)) == 0)
&& (*(msr->mpd->buf + 2 + strlen(msr->mpd->boundary)) == '-')
&& (*(msr->mpd->buf + 2 + strlen(msr->mpd->boundary) + 1) == '-') )
&& (strncmp(msr->mpd->buf + 2, msr->mpd->boundary, boundary_len) == 0) )
{
if ((msr->mpd->crlf_state_buf_end == 2) && (msr->mpd->flag_lf_line != 1)) {
msr->mpd->flag_lf_line = 1;
if (msr->mpd->flag_crlf_line) {
msr_log(msr, 4, "Multipart: Warning: mixed line endings used (CRLF/LF).");
} else {
msr_log(msr, 4, "Multipart: Warning: incorrect line endings used (LF).");
if ( (buf_data_len >= 2 + boundary_len + 2)
&& (*(msr->mpd->buf + 2 + boundary_len) == '-')
&& (*(msr->mpd->buf + 2 + boundary_len + 1) == '-') )
{
/* If body fits in limit and ends with final boundary plus just CR, reject it. */
if ( (msr->mpd->allow_process_partial == 0)
&& (buf_data_len == 2 + boundary_len + 2 + 1)
&& (*(msr->mpd->buf + 2 + boundary_len + 2) == '\r') )
{
*error_msg = apr_psprintf(msr->mp, "Multipart: Invalid epilogue after final boundary.");
return -1;
}

if ((msr->mpd->crlf_state_buf_end == 2) && (msr->mpd->flag_lf_line != 1)) {
msr->mpd->flag_lf_line = 1;
if (msr->mpd->flag_crlf_line) {
msr_log(msr, 4, "Multipart: Warning: mixed line endings used (CRLF/LF).");
} else {
msr_log(msr, 4, "Multipart: Warning: incorrect line endings used (LF).");
}
}
if (msr->mpd->mpp_substate_part_data_read == 0) {
/* it looks like the final boundary, but it's where part data should begin */
msr->mpd->flag_invalid_part = 1;
msr_log(msr, 4, "Multipart: Warning: Invalid part (data contains final boundary)");
}
/* Looks like the final boundary - process it. */
if (multipart_process_boundary(msr, 1 /* final */, error_msg) < 0) {
msr->mpd->flag_error = 1;
return -1;
}

/* The payload is complete after all. */
msr->mpd->is_complete = 1;
}
if (msr->mpd->mpp_substate_part_data_read == 0) {
/* it looks like the final boundary, but it's where part data should begin */
msr->mpd->flag_invalid_part = 1;
msr_log(msr, 4, "Multipart: Warning: Invalid part (data contains final boundary)");
}
/* Looks like the final boundary - process it. */
if (multipart_process_boundary(msr, 1 /* final */, error_msg) < 0) {
msr->mpd->flag_error = 1;
return -1;
else if (msr->mpd->allow_process_partial == 1) {
if (buf_data_len >= 2 + boundary_len + 1) {
if (*(msr->mpd->buf + 2 + boundary_len) == '-') {
if ( (buf_data_len >= 2 + boundary_len + 2)
&& (*(msr->mpd->buf + 2 + boundary_len + 1) != '-') ) {
*error_msg = apr_psprintf(msr->mp, "Multipart: Invalid final boundary.");
return -1;
}
}
else if ( (*(msr->mpd->buf + 2 + boundary_len) != '\r')
|| ((buf_data_len >= 2 + boundary_len + 2)
&& (*(msr->mpd->buf + 2 + boundary_len + 1) != '\n')) ) {
*error_msg = apr_psprintf(msr->mp, "Multipart: Invalid boundary.");
return -1;
}
}
/* process it as a non-final boundary to avoid building a new part. */
if (multipart_process_boundary(msr, 0, error_msg) < 0) {
msr->mpd->flag_error = 1;
return -1;
}
}

/* The payload is complete after all. */
msr->mpd->is_complete = 1;
}
}

if (msr->mpd->is_complete == 0) {
if (msr->mpd->is_complete == 0 && msr->mpd->allow_process_partial == 0) {
*error_msg = apr_psprintf(msr->mp, "Multipart: Final boundary missing.");
return -1;
}
Expand Down Expand Up @@ -1296,10 +1352,10 @@ int multipart_process_chunk(modsec_rec *msr, const char *buf,
if (c == 0x0a) {
if (msr->mpd->crlf_state == 1) {
msr->mpd->crlf_state = 3;
} else {
} else {
msr->mpd->crlf_state = 2;
}
}
}
}
msr->mpd->crlf_state_buf_end = msr->mpd->crlf_state;
}

Expand Down
1 change: 1 addition & 0 deletions apache2/msc_multipart.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ struct multipart_data {

int seen_data;
int is_complete;
int allow_process_partial;

int flag_error;
int flag_data_before;
Expand Down
4 changes: 2 additions & 2 deletions apache2/msc_parsers.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ int parse_arguments(modsec_rec *msr, const char *s, apr_size_t inputlength,
value = &buf[j];
}
}
else {
else if (i < inputlength || msr->reqbody_partial_processing_enabled == 0) {
arg->value_len = urldecode_nonstrict_inplace_ex((unsigned char *)value, arg->value_origin_len, invalid_count, &changed);
arg->value = apr_pstrmemdup(msr->mp, value, arg->value_len);

Expand All @@ -330,7 +330,7 @@ int parse_arguments(modsec_rec *msr, const char *s, apr_size_t inputlength,
}

/* the last parameter was empty */
if (status == 1) {
if (status == 1 && msr->reqbody_partial_processing_enabled == 0) {
arg->value_len = 0;
arg->value = "";

Expand Down
Loading
Loading