Ubuntu
apache2 package

apache2 segfault using mod_deflate

Bug #409987 reported by Sylvain Filteau
14
This bug affects 2 people
Affects Status Importance Assigned to Milestone
apache2 (Ubuntu)
Fix Released
Medium
Marc Deslauriers

Bug Description

Binary package hint: apache2.2-common

On my production webserver, I started having segfault in my error log :

Jul 28 04:32:08 2009] [notice] child pid 9005 exit signal Segmentation fault (11)
Jul 28 05:30:53 2009] [notice] child pid 15156 exit signal Segmentation fault (11)
Jul 28 05:32:52 2009] [notice] child pid 15204 exit signal Segmentation fault (11)
Jul 28 05:39:18 2009] [notice] child pid 15013 exit signal Segmentation fault (11)
Jul 28 05:45:33 2009] [notice] child pid 15202 exit signal Segmentation fault (11)
[...]

Here is a gdb backtrace of a core dump :

(gdb) bt full
#0 0x00007f91e7e9bd37 in crc32 () from /usr/lib/libz.so.1
No symbol table info available.
#1 0x00007f91e5ffd204 in deflate_out_filter (f=0xaab9d0, bb=0xaa4978)
    at /build/buildd/apache2-2.2.8/modules/filters/mod_deflate.c:698
 data = 0x36b7f88 <Address 0x36b7f88 out of bounds>
 b = <value optimized out>
 len = 2523705
 e = (apr_bucket *) 0xa99f58
 r = (request_rec *) 0xaab598
 ctx = (deflate_ctx *) 0xaa4c70
 zRC = <value optimized out>
 c = (deflate_filter_config *) 0x6dfda8
#2 0x00007f91e5358bbb in ?? () from /usr/lib/apache2/modules/libphp5.so
No symbol table info available.
#3 0x0000000000437daa in ap_run_handler (r=0xaab598)
    at /build/buildd/apache2-2.2.8/server/config.c:158
 n = 3
 rv = 2523705
#4 0x000000000043b1cc in ap_invoke_handler (r=0xaab598)
    at /build/buildd/apache2-2.2.8/server/config.c:373
 handler = 0x7d19c8 "application/x-httpd-php"
 result = 0
 old_handler = 0x0
 ignore = <value optimized out>
#5 0x000000000044773a in ap_internal_redirect (new_uri=<value optimized out>,
    r=<value optimized out>)
    at /build/buildd/apache2-2.2.8/modules/http/http_request.c:477
 new = (request_rec *) 0xaab598
 access_status = 0
#6 0x00007f91e441f2d0 in handler_redirect (r=0xaa1ca8)
    at /build/buildd/apache2-2.2.8/modules/mappers/mod_rewrite.c:4762
No locals.
#7 0x0000000000437daa in ap_run_handler (r=0xaa1ca8)
    at /build/buildd/apache2-2.2.8/server/config.c:158
 n = 4
 rv = 2523705
#8 0x000000000043b1cc in ap_invoke_handler (r=0xaa1ca8)
    at /build/buildd/apache2-2.2.8/server/config.c:373
 handler = 0x200000000 <Address 0x200000000 out of bounds>
 result = 0
 old_handler = 0x7f91e4423aab "redirect-handler"
 ignore = <value optimized out>
#9 0x00000000004478ae in ap_process_request (r=0xaa1ca8)
    at /build/buildd/apache2-2.2.8/modules/http/http_request.c:258
 access_status = 0
#10 0x0000000000444ca8 in ap_process_http_connection (c=0xa95b58)
    at /build/buildd/apache2-2.2.8/modules/http/http_core.c:190
 r = (request_rec *) 0xaa1ca8
 csd = (apr_socket_t *) 0x0
#11 0x000000000043ef02 in ap_run_process_connection (c=0xa95b58)
    at /build/buildd/apache2-2.2.8/server/connection.c:43
 n = 0
 rv = 2523705
---Type <return> to continue, or q <return> to quit---
#12 0x000000000044b6a5 in child_main (child_num_arg=<value optimized out>)
    at /build/buildd/apache2-2.2.8/server/mpm/prefork/prefork.c:662
 current_conn = (conn_rec *) 0xa95b58
 csd = (void *) 0xa95968
 ptrans = (apr_pool_t *) 0xa958f8
 allocator = (apr_allocator_t *) 0xa937f0
 status = <value optimized out>
 i = <value optimized out>
 lr = <value optimized out>
 pollset = (apr_pollset_t *) 0xa939e8
 sbh = (ap_sb_handle_t *) 0xa939e0
 bucket_alloc = (apr_bucket_alloc_t *) 0xa99bf8
 last_poll_idx = 1
#13 0x000000000044b955 in make_child (s=0x674968, slot=7)
    at /build/buildd/apache2-2.2.8/server/mpm/prefork/prefork.c:759
 pid = 0
#14 0x000000000044c1e8 in ap_mpm_run (_pconf=<value optimized out>,
    plog=<value optimized out>, s=<value optimized out>)
    at /build/buildd/apache2-2.2.8/server/mpm/prefork/prefork.c:894
 status = 0
 pid = {pid = -1, in = 0x8485d0, out = 0x676180, err = 0x668040}
 child_slot = <value optimized out>
 exitwhy = APR_PROC_EXIT
 processed_status = <value optimized out>
 index = <value optimized out>
 remaining_children_to_start = 0
 rv = <value optimized out>
#15 0x0000000000425a44 in main (argc=3, argv=0x7ffff3a90848)
    at /build/buildd/apache2-2.2.8/server/main.c:732
 c = 0 '\0'
 configtestonly = 0
 confname = 0x44ddba "/etc/apache2/apache2.conf"
 def_server_root = 0x45296a ""
 temp_error_log = 0x0
 error = <value optimized out>
 process = (process_rec *) 0x66c238
 server_conf = (server_rec *) 0x674968
 pglobal = (apr_pool_t *) 0x66c158
 pconf = (apr_pool_t *) 0x66e168
 plog = (apr_pool_t *) 0x6a2308
 ptemp = (apr_pool_t *) 0x6761a8
 pcommands = (apr_pool_t *) 0x670178
 opt = (apr_getopt_t *) 0x670260
 rv = 0
 optarg = 0x7ffff3a90848 "8\017���\177"

--------------------------------------

A little search on google pop me out this page : http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=537665 (DSA-1834-2
) talking about a bug that looks like my issue. The reporter have done something with gdb that I copy and pasted and I thought maybe it could help with this bug report :

(gdb) select 1
(gdb) p *r
$1 = {pool = 0xaa1c38, connection = 0xa95b58, server = 0x7fb040, next = 0x0,
  prev = 0xaa1ca8, main = 0x0, the_request = 0xaa3238 "POST /siam/engin HTTP/1.1",
  assbackwards = 0, proxyreq = 0, header_only = 0, protocol = 0xaa32c0 "HTTP/1.1",
  proto_num = 1001, hostname = 0xaa3938 "[hidden-hostname]",
  request_time = 1249575009636661, status_line = 0x454fd3 "200 OK", status = 200,
  method = 0xaa3288 "POST", method_number = 2, allowed = 0, allowed_xmethods = 0x0,
  allowed_methods = 0xa9c898, sent_bodyct = 1, bytes_sent = 56682, mtime = 0,
  chunked = 1, range = 0x0, clength = 0, remaining = 0, read_length = 0,
  read_body = 0, read_chunked = 0, expecting_100 = 0, headers_in = 0xaa1f88,
  headers_out = 0xa9c130, err_headers_out = 0xaa28d0, subprocess_env = 0xa9c378,
  notes = 0xa9c6f8, content_type = 0xaa4bb0 "text/html",
  handler = 0x7d19c8 "application/x-httpd-php", content_encoding = 0x0,
  content_languages = 0x0, vlist_validator = 0xaab130 "\"44bd08b592a80\"",
  user = 0x0, ap_auth_type = 0x0, no_cache = 0, no_local_copy = 1,
  unparsed_uri = 0xaab878 "/index.php/srv/www/sygestran/production/htdocs/siam/engin", uri = 0xaab8b8 "/index.php/srv/www/sygestran/production/htdocs/siam/engin",
  filename = 0xa9cfa0 "/srv/www/sygestran/production/htdocs/index.php",
  canonical_filename = 0xa9cfa0 "/srv/www/sygestran/production/htdocs/index.php",
  path_info = 0xa9ce76 "/srv/www/sygestran/production/htdocs/siam/engin",
  args = 0x0, finfo = {pool = 0xaa1c38, valid = 7598448, protection = 1604,
    filetype = APR_REG, user = 1000, group = 1000, inode = 1426560, device = 2056,
    nlink = 1, size = 3199, csize = 8598318192, atime = 1213997387000000,
    mtime = 1213997387000000, ctime = 1213997387000000,
    fname = 0xa9cfa0 "/srv/www/sygestran/production/htdocs/index.php",
    name = 0x4384cd "I\211\004,H\213[ H\205�t5HcC\bI\213T�", filehand = 0xa9c970},
  parsed_uri = {scheme = 0x0, hostinfo = 0x0, user = 0x0, password = 0x0,
    hostname = 0x0, port_str = 0x0,
    path = 0xaab8b8 "/index.php/srv/www/sygestran/production/htdocs/siam/engin",
    query = 0x0, fragment = 0x0, hostent = 0x0, port = 0, is_initialized = 1,
    dns_looked_up = 0, dns_resolved = 0}, used_path_info = 0,
  per_dir_config = 0xa9d568, request_config = 0xa9bc08, htaccess = 0xa9e1f8,
  output_filters = 0xaa4c00, input_filters = 0xaa3958,
  proto_output_filters = 0xaa3180, proto_input_filters = 0xaa3958, eos_sent = 1}

--------------------------------------

$ lsb_release -rd
Description: Ubuntu 8.04.3 LTS
Release: 8.04

$ apt-cache policy apache2.2-common
apache2.2-common:
  Installed: 2.2.8-1ubuntu0.10
  Candidate: 2.2.8-1ubuntu0.10
  Version table:
 *** 2.2.8-1ubuntu0.10 0
        500 http://ca.archive.ubuntu.com hardy-updates/main Packages
        500 http://ca.archive.ubuntu.com hardy-security/main Packages
        100 /var/lib/dpkg/status
     2.2.8-1 0
        500 http://ca.archive.ubuntu.com hardy/main Packages

Revision history for this message
Chuck Short (zulcss) wrote :

Can you try the version in my ppa when its built (http://launchpad.net/~zulcss/+archive)?

Thanks
chuck

Changed in apache2 (Ubuntu):
importance: Undecided → Medium
Revision history for this message
Marc Deslauriers (mdeslaur) wrote :

Relevant thread: http://<email address hidden>/msg44655.html

Revision history for this message
Sylvain Filteau (cidsphere) wrote :

@chuck I installed the apache version of your ppa :
$ apache2 -v
Server version: Apache/2.2.8 (Ubuntu)
Server built: Aug 7 2009 13:02:54

Sadly, it didn't resolved my issue

-----

@marc I tried the php script but I can't trigger the segfault with it...

-----

Since this morning I work to reproduce the problem in my dev environment with success.

Now I try to build a script that reproduce my problem but this task is hard because my application is really big. I can say that the output is a json string of 2.5M with these http headers :

HTTP/1.1 100 Continue

HTTP/1.1 200 OK
Date: Fri, 07 Aug 2009 20:06:31 GMT
Server: Apache/2.2.8 (Ubuntu) PHP/5.2.4-2ubuntu5.6 with Suhosin-Patch
X-Powered-By: PHP/5.2.4-2ubuntu5.6
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
ADAFramework-identity: some_string_important_for_me
Vary: Accept-Encoding
Content-Encoding: gzip
Transfer-Encoding: chunked
Content-Type: text/html

-----

Have a nice weekend !

Revision history for this message
Sylvain Filteau (cidsphere) wrote :

I wrote a small php script that triggered my problem :

<?php
echo file_get_contents('big-shuffled.json');
?>

Put this as 'x.php' in your apache document root with the file in attachment and run this command :
$ curl -is -H 'Accept-Encoding: gzip' http://localhost/x.php

On my machine, it triggered the problem.

I tried to generate a 2.5M of random stuff but it didn't work. Only this file does the problem with mod_deflate.

Revision history for this message
Marc Deslauriers (mdeslaur) wrote :

Thanks for the reproducer Sylvain. I could now reproduce the segfault locally.

Could you please try the updated packages in my PPA:

https://launchpad.net/~mdeslaur/+archive/ppa

If they solve the segfaults for you, I'll push out some updates.

Thanks.

Changed in apache2 (Ubuntu):
status: New → Incomplete
assignee: nobody → Marc Deslauriers (mdeslaur)
Revision history for this message
Sylvain Filteau (cidsphere) wrote :

Looks good on my side !

Thank you very much for your help !

Revision history for this message
Launchpad Janitor (janitor) wrote :

This bug was fixed in the package apache2 - 2.2.12-1ubuntu2

---------------
apache2 (2.2.12-1ubuntu2) karmic; urgency=low

  * debian/patches/203_fix_legacy_ap_rputs_segfaults.dpatch:
    - Fix potential segfaults with the use of the legacy ap_rputs() etc
      interfaces, in cases where an output filter fails. This happens
      frequently after CVE-2009-1891 got fixed. (LP: #409987)

 -- Marc Deslauriers <email address hidden> Mon, 17 Aug 2009 15:38:47 -0400

Changed in apache2 (Ubuntu):
status: Incomplete → Fix Released
Revision history for this message
Marc Deslauriers (mdeslaur) wrote :

Updates for current releases were just published:

http://www.ubuntu.com/usn/USN-802-2

Revision history for this message
tiiibs (tiiibs) wrote :

Hi,

I've the same problem. I've patched the server but the error is still here!

[Tue Aug 25 11:25:02 2009] [notice] child pid 10025 exit signal Segmentation fault (11)
[Tue Aug 25 11:25:02 2009] [notice] child pid 10026 exit signal Segmentation fault (11)
[Tue Aug 25 11:25:02 2009] [notice] child pid 10027 exit signal Segmentation fault (11)

what are tests that I can do?

Revision history for this message
Marc Deslauriers (mdeslaur) wrote :

tiiibs: are you sure it's the same problem? What apache2 package version are you running? What release of Ubuntu?

Revision history for this message
Xeno (xeno22) wrote :

Also have this issue on some servers.

Running Ubuntu 14.04 LTS:

Server version: Apache/2.4.7 (Ubuntu)
Server built: May 9 2017 16:14:10

root@server:/var/www/obs# php -v
PHP 5.5.9-1ubuntu4.21 (cli) (built: Feb 9 2017 20:54:58)
Copyright (c) 1997-2014 The PHP Group
Zend Engine v2.5.0, Copyright (c) 1998-2014 Zend Technologies
    with SourceGuardian v11.1.0, Copyright (c) 2000-2017, by SourceGuardian Ltd.
    with Zend OPcache v7.0.3, Copyright (c) 1999-2014, by Zend Technologies

root@server:/var/www/obs# apt-cache policy zlib1g
zlib1g:
  Installed: 1:1.2.8.dfsg-1ubuntu1

root@server:/var/www/obs# apt-cache policy apache2
apache2:
  Installed: 2.4.7-1ubuntu4.15

root@server:/var/www/obs# apt-cache policy php5
php5:
  Installed: 5.5.9+dfsg-1ubuntu4.21

Here is the core dump:

Reading symbols from /usr/sbin/apache2...(no debugging symbols found)...done.
[New LWP 14926]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/sbin/apache2 -k start'.
Program terminated with signal SIGBUS, Bus error.
#0 0x00007f7d535a97b0 in crc32 () from /lib/x86_64-linux-gnu/libz.so.1
(gdb) bt
#0 0x00007f7d535a97b0 in crc32 () from /lib/x86_64-linux-gnu/libz.so.1
#1 0x00007f7d537c3344 in ?? () from /usr/lib/apache2/modules/mod_deflate.so
#2 0x00007f7d52f9d479 in ?? () from /usr/lib/apache2/modules/mod_filter.so
#3 0x00007f7d52f9d479 in ?? () from /usr/lib/apache2/modules/mod_filter.so
#4 0x00007f7d5654c71f in ?? ()
#5 0x00007f7d5655cf40 in ap_run_handler ()
#6 0x00007f7d5655d489 in ap_invoke_handler ()
#7 0x00007f7d5657251c in ap_internal_redirect ()
#8 0x00007f7d5056bcfc in ?? () from /usr/lib/apache2/modules/mod_rewrite.so
#9 0x00007f7d5655cf40 in ap_run_handler ()
#10 0x00007f7d5655d489 in ap_invoke_handler ()
#11 0x00007f7d56572a5a in ap_process_async_request ()
#12 0x00007f7d56572d34 in ap_process_request ()
#13 0x00007f7d5656f7d2 in ?? ()
#14 0x00007f7d565665b0 in ap_run_process_connection ()
#15 0x00007f7d52b91767 in ?? () from /usr/lib/apache2/modules/mod_mpm_prefork.so
#16 0x00007f7d52b919a6 in ?? () from /usr/lib/apache2/modules/mod_mpm_prefork.so
#17 0x00007f7d52b9260e in ?? () from /usr/lib/apache2/modules/mod_mpm_prefork.so
#18 0x00007f7d5654223e in ap_run_mpm ()
#19 0x00007f7d5653b276 in main ()

To post a comment you must log in.
This report contains Public information  
Everyone can see this information.
Subscribing...

Other bug subscribers

Remote bug watches

Bug watches keep track of this bug in other bug trackers.