Nginx Chroot set up how-to.
This is the way we compile from source.
There is another way in which we leverage an existing nginx rpm and build up a chroot env. out of it. we will visit this option in another post.

$ ./configure –help

–help                             print this message

–prefix=PATH                      set installation prefix
–sbin-path=PATH                   set nginx binary pathname
–conf-path=PATH                   set nginx.conf pathname
–error-log-path=PATH              set error log pathname
–pid-path=PATH                    set nginx.pid pathname
–lock-path=PATH                   set nginx.lock pathname

–user=USER                        set non-privileged user for
worker processes
–group=GROUP                      set non-privileged group for
worker processes

–builddir=DIR                     set build directory

–with-rtsig_module                enable rtsig module
–with-select_module               enable select module
–without-select_module            disable select module
–with-poll_module                 enable poll module
–without-poll_module              disable poll module

–with-file-aio                    enable file AIO support
–with-ipv6                        enable IPv6 support

–with-http_ssl_module             enable ngx_http_ssl_module
–with-http_realip_module          enable ngx_http_realip_module
–with-http_addition_module        enable ngx_http_addition_module
–with-http_xslt_module            enable ngx_http_xslt_module
–with-http_image_filter_module    enable ngx_http_image_filter_module
–with-http_geoip_module           enable ngx_http_geoip_module
–with-http_sub_module             enable ngx_http_sub_module
–with-http_dav_module             enable ngx_http_dav_module
–with-http_flv_module             enable ngx_http_flv_module
–with-http_mp4_module             enable ngx_http_mp4_module
–with-http_gzip_static_module     enable ngx_http_gzip_static_module
–with-http_random_index_module    enable ngx_http_random_index_module
–with-http_secure_link_module     enable ngx_http_secure_link_module
–with-http_degradation_module     enable ngx_http_degradation_module
–with-http_stub_status_module     enable ngx_http_stub_status_module

–without-http_charset_module      disable ngx_http_charset_module
–without-http_gzip_module         disable ngx_http_gzip_module
–without-http_ssi_module          disable ngx_http_ssi_module
–without-http_userid_module       disable ngx_http_userid_module
–without-http_access_module       disable ngx_http_access_module
–without-http_auth_basic_module   disable ngx_http_auth_basic_module
–without-http_autoindex_module    disable ngx_http_autoindex_module
–without-http_geo_module          disable ngx_http_geo_module
–without-http_map_module          disable ngx_http_map_module
–without-http_split_clients_module disable ngx_http_split_clients_module
–without-http_referer_module      disable ngx_http_referer_module
–without-http_rewrite_module      disable ngx_http_rewrite_module
–without-http_proxy_module        disable ngx_http_proxy_module
–without-http_fastcgi_module      disable ngx_http_fastcgi_module
–without-http_uwsgi_module        disable ngx_http_uwsgi_module
–without-http_scgi_module         disable ngx_http_scgi_module
–without-http_memcached_module    disable ngx_http_memcached_module
–without-http_limit_conn_module   disable ngx_http_limit_conn_module
–without-http_limit_req_module    disable ngx_http_limit_req_module
–without-http_empty_gif_module    disable ngx_http_empty_gif_module
–without-http_browser_module      disable ngx_http_browser_module
–without-http_upstream_ip_hash_module
disable ngx_http_upstream_ip_hash_module
–without-http_upstream_least_conn_module
disable ngx_http_upstream_least_conn_module
–without-http_upstream_keepalive_module
disable ngx_http_upstream_keepalive_module

–with-http_perl_module            enable ngx_http_perl_module
–with-perl_modules_path=PATH      set Perl modules path
–with-perl=PATH                   set perl binary pathname

–http-log-path=PATH               set http access log pathname
–http-client-body-temp-path=PATH  set path to store
http client request body temporary files
–http-proxy-temp-path=PATH        set path to store
http proxy temporary files
–http-fastcgi-temp-path=PATH      set path to store
http fastcgi temporary files
–http-uwsgi-temp-path=PATH        set path to store
http uwsgi temporary files
–http-scgi-temp-path=PATH         set path to store
http scgi temporary files

–without-http                     disable HTTP server
–without-http-cache               disable HTTP cache

–with-mail                        enable POP3/IMAP4/SMTP proxy module
–with-mail_ssl_module             enable ngx_mail_ssl_module
–without-mail_pop3_module         disable ngx_mail_pop3_module
–without-mail_imap_module         disable ngx_mail_imap_module
–without-mail_smtp_module         disable ngx_mail_smtp_module

–with-google_perftools_module     enable ngx_google_perftools_module
–with-cpp_test_module             enable ngx_cpp_test_module

–add-module=PATH                  enable an external module

–with-cc=PATH                     set C compiler pathname
–with-cpp=PATH                    set C preprocessor pathname
–with-cc-opt=OPTIONS              set additional C compiler options
–with-ld-opt=OPTIONS              set additional linker options
–with-cpu-opt=CPU                 build for the specified CPU, valid values:
pentium, pentiumpro, pentium3, pentium4,
athlon, opteron, sparc32, sparc64, ppc64

–without-pcre                     disable PCRE library usage
–with-pcre                        force PCRE library usage
–with-pcre=DIR                    set path to PCRE library sources
–with-pcre-opt=OPTIONS            set additional build options for PCRE
–with-pcre-jit                    build PCRE with JIT compilation support

–with-md5=DIR                     set path to md5 library sources
–with-md5-opt=OPTIONS             set additional build options for md5
–with-md5-asm                     use md5 assembler sources

–with-sha1=DIR                    set path to sha1 library sources
–with-sha1-opt=OPTIONS            set additional build options for sha1
–with-sha1-asm                    use sha1 assembler sources

–with-zlib=DIR                    set path to zlib library sources
–with-zlib-opt=OPTIONS            set additional build options for zlib
–with-zlib-asm=CPU                use zlib assembler sources optimized
for the specified CPU, valid values:
pentium, pentiumpro

–with-libatomic                   force libatomic_ops library usage
–with-libatomic=DIR               set path to libatomic_ops library sources

–with-openssl=DIR                 set path to OpenSSL library sources
–with-openssl-opt=OPTIONS         set additional build options for OpenSSL

–with-debug                       enable debug logging

# a sample compile options of our 1.0.15 nginx server

$ nginx -V
nginx version: nginx/1.0.15
built by gcc 4.1.2 20080704 (Red Hat 4.1.2-52)
TLS SNI support disabled
configure arguments: –prefix=/usr/share/nginx –sbin-path=/usr/sbin/nginx –conf-path=/etc/nginx/nginx.conf –error-log-path=/var/log/nginx/error.log –http-log-path=/var/log/nginx/access.log –http-client-body-temp-path=/var/lib/nginx/tmp/client_body –http-proxy-temp-path=/var/lib/nginx/tmp/proxy –http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi –http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi –http-scgi-temp-path=/var/lib/nginx/tmp/scgi –pid-path=/var/run/nginx.pid –lock-path=/var/lock/subsys/nginx –user=nginx –group=nginx –add-module=nginx-upload-progress-module –with-file-aio –with-ipv6 –with-http_ssl_module –with-http_realip_module –with-http_addition_module –with-http_xslt_module –with-http_image_filter_module –with-http_sub_module –with-http_dav_module –with-http_flv_module –with-http_mp4_module –with-http_gzip_static_module –with-http_random_index_module –with-http_secure_link_module –with-http_degradation_module –with-http_stub_status_module –with-http_perl_module –with-mail –with-mail_ssl_module –with-cc-opt=’-O2 -g -m64 -mtune=generic’ –with-ld-opt=-Wl,-E

# http://wiki.nginx.org/InstallOptions on compile time option details.
# http://wiki.nginx.org/Optimizations on rtsig/select/poll
# http://wiki.nginx.org/Modules for standard, optional and 3rd party modules.

# we decided to compile as below for our chroot nginx

# some basic compile options:
–prefix=/opt/nginx-1.2.6 \
–http-client-body-temp-path=/opt/nginx-1.2.6/nginx_temp/client_body \
–http-proxy-temp-path=/opt/nginx-1.2.6/nginx_temp/proxy \
–http-fastcgi-temp-path=/opt/nginx-1.2.6/nginx_temp/fastcgi \
–http-uwsgi-temp-path=/opt/nginx-1.2.6/nginx_temp/uwsgi \
–http-scgi-temp-path=/opt/nginx-1.2.6/nginx_temp/scgi \
–user=nginx \
–group=nginx \
–with-file-aio \
–with-ipv6 \

# optional modules
–with-http_addition_module \
–with-http_degradation_module \
–with-http_geoip_module \
–with-http_gzip_static_module \
–with-http_image_filter_module \
–with-http_mp4_module \
–with-http_random_index_module \
–with-http_realip_module \
–with-http_secure_link_module \
–with-http_ssl_module \
–with-http_stub_status_module \
–with-http_sub_module \
–with-http_dav_module \
–with-http_xslt_module \
–with-mail \
–with-mail_ssl_module

# 3rd party modules
–add-module=nginx-upload-progress-module

# we need to build the geoIP library for nginx first as centOS yum does not have it
# go to http://www.maxmind.com/download/geoip/api/c/
$ wget http://www.maxmind.com/download/geoip/api/c/GeoIP-1.4.8.tar.gz
$ tar xvzf GeoIP-1.4.8.tar.gz
$ cd GeoIP-1.4.8
$ ./configure

!error
$ make
CDPATH=”${ZSH_VERSION+.}:” && cd . && /bin/sh /root/setup/GeoIP-1.4.8/missing –run aclocal-1.11
cd . && /bin/sh /root/setup/GeoIP-1.4.8/missing –run automake-1.11 –gnu
libGeoIP/Makefile.am:1: Libtool library used but `LIBTOOL’ is undefined
libGeoIP/Makefile.am:1:   The usual way to define `LIBTOOL’ is to add `AC_PROG_LIBTOOL’
libGeoIP/Makefile.am:1:   to `configure.in’ and run `aclocal’ and `autoconf’ again.
libGeoIP/Makefile.am:1:   If `AC_PROG_LIBTOOL’ is in `configure.in’, make sure
libGeoIP/Makefile.am:1:   its definition is in aclocal’s search path.
make: *** [Makefile.in] Error 1
!end error

$ yum install libtool.x86_64
# make clean or just start afresh
$ make
$ make check
$ make install

$ ./configure –prefix=/opt/nginx-1.2.6 \
–http-client-body-temp-path=/opt/nginx-1.2.6/nginx_temp/client_body \
–http-proxy-temp-path=/opt/nginx-1.2.6/nginx_temp/proxy \
–http-fastcgi-temp-path=/opt/nginx-1.2.6/nginx_temp/fastcgi \
–http-uwsgi-temp-path=/opt/nginx-1.2.6/nginx_temp/uwsgi \
–http-scgi-temp-path=/opt/nginx-1.2.6/nginx_temp/scgi \
–user=nginx \
–group=nginx \
–with-file-aio \
–with-ipv6 \
–with-http_addition_module \
–with-http_degradation_module \
–with-http_geoip_module \
–with-http_gzip_static_module \
–with-http_image_filter_module \
–with-http_mp4_module \
–with-http_random_index_module \
–with-http_realip_module \
–with-http_secure_link_module \
–with-http_ssl_module \
–with-http_stub_status_module \
–with-http_sub_module \
–with-http_dav_module \
–with-http_xslt_module \
–with-mail \
–with-mail_ssl_module \
–add-module=nginx-upload-progress-module

$ make
$ make install

$ cd /opt/nginx-1.2.6/sbin
$ ldd nginx
linux-vdso.so.1 =>  (0x00007fff1fb10000)
libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f2144888000)
libcrypt.so.1 => /lib64/libcrypt.so.1 (0x00007f2144650000)
libpcre.so.0 => /lib64/libpcre.so.0 (0x00007f2144420000)
libssl.so.10 => /usr/lib64/libssl.so.10 (0x00007f21441c0000)
libcrypto.so.10 => /usr/lib64/libcrypto.so.10 (0x00007f2143e20000)
libdl.so.2 => /lib64/libdl.so.2 (0x00007f2143c18000)
libz.so.1 => /lib64/libz.so.1 (0x00007f2143a00000)
libxml2.so.2 => /usr/lib64/libxml2.so.2 (0x00007f21436a8000)
libxslt.so.1 => /usr/lib64/libxslt.so.1 (0x00007f2143468000)
libexslt.so.0 => /usr/lib64/libexslt.so.0 (0x00007f2143250000)
libgd.so.2 => /usr/lib64/libgd.so.2 (0x00007f2143008000)
libGeoIP.so.1 => not found
libc.so.6 => /lib64/libc.so.6 (0x00007f2142c70000)
/lib64/ld-linux-x86-64.so.2 (0x00007f2144ab8000)
libfreebl3.so => /lib64/libfreebl3.so (0x00007f2142a08000)
libgssapi_krb5.so.2 => /lib64/libgssapi_krb5.so.2 (0x00007f21427c0000)
libkrb5.so.3 => /lib64/libkrb5.so.3 (0x00007f21424e0000)
libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00007f21422d8000)
libk5crypto.so.3 => /lib64/libk5crypto.so.3 (0x00007f21420a8000)
libm.so.6 => /lib64/libm.so.6 (0x00007f2141e20000)
libgcrypt.so.11 => /lib64/libgcrypt.so.11 (0x00007f2141ba8000)
libgpg-error.so.0 => /lib64/libgpg-error.so.0 (0x00007f21419a0000)
libXpm.so.4 => /usr/lib64/libXpm.so.4 (0x00007f2141788000)
libX11.so.6 => /usr/lib64/libX11.so.6 (0x00007f2141448000)
libjpeg.so.62 => /usr/lib64/libjpeg.so.62 (0x00007f2141220000)
libfontconfig.so.1 => /usr/lib64/libfontconfig.so.1 (0x00007f2140fe8000)
libfreetype.so.6 => /usr/lib64/libfreetype.so.6 (0x00007f2140d48000)
libpng12.so.0 => /usr/lib64/libpng12.so.0 (0x00007f2140b20000)
libkrb5support.so.0 => /lib64/libkrb5support.so.0 (0x00007f2140910000)
libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007f2140708000)
libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f21404e8000)
libxcb.so.1 => /usr/lib64/libxcb.so.1 (0x00007f21402c8000)
libexpat.so.1 => /lib64/libexpat.so.1 (0x00007f21400a0000)
libselinux.so.1 => /lib64/libselinux.so.1 (0x00007f213fe80000)
libXau.so.6 => /usr/lib64/libXau.so.6 (0x00007f213fc78000)

#! libGeoIP.so.1 => not found? how come?
$ find / -iname ‘*libGeoIP.so.1*’
/usr/local/lib/libGeoIP.so.1
/usr/local/lib/libGeoIP.so.1.4.8
……

# check system library path

$ ldconfig -v

# check the output and seems /usr/local/lib is not included.
# to add this..

$ cd /etc/ld.so.conf.d/

# add a file called “usr-local-lib.conf” with ‘/usr/local/lib’ in it. then…

$ ldconfig
$ ldconfig -v

# now libGeoIP.so.1 is in the path

$ cd /opt/nginx-1.2.6/sbin
$ ldd nginx
……
libGeoIP.so.1 => /usr/local/lib/libGeoIP.so.1 (0x00007fed6f598000)
……

# the directory we configure nginx to store those temp, this needs to be manually created.
$ mkdir -p /opt/nginx-1.2.6/nginx_temp

$ ./nginx -t
nginx: the configuration file /opt/nginx-1.2.6/conf/nginx.conf syntax is ok
nginx: configuration file /opt/nginx-1.2.6/conf/nginx.conf test is successful

# run it and try accessing your website and see the nginx welcome screen
$ ./nginx

# Now start the chroot build
$ killall nginx

# chroot_dir.pl basically relies on ldd and “recursively” (meaning if a .so has dependencies, we drill down again and again)
# copy needed .so file to the chroot env.

$ perl chroot_dir.pl /opt/nginx-1.2.6 /opt/nginx-1.2.6
#####################################
# Residential files that need chroot:
/lib64/libexpat.so.1 => 1 time(s)
/lib64/libk5crypto.so.3 => 1 time(s)
/lib64/libcrypt.so.1 => 1 time(s)
/usr/lib64/libssl.so.10 => 1 time(s)
/usr/local/lib/libGeoIP.so.1 => 1 time(s)
/usr/lib64/libgd.so.2 => 1 time(s)
/lib64/libfreebl3.so => 1 time(s)
/lib64/libselinux.so.1 => 1 time(s)
/usr/lib64/libXau.so.6 => 1 time(s)
/lib64/libc.so.6 => 1 time(s)
/usr/lib64/libxcb.so.1 => 1 time(s)
/lib64/libpcre.so.0 => 1 time(s)
/usr/lib64/libjpeg.so.62 => 1 time(s)
/usr/lib64/libcrypto.so.10 => 1 time(s)
/opt/nginx-1.2.6/sbin/nginx => 1 time(s)
/lib64/libcom_err.so.2 => 1 time(s)
/lib64/libm.so.6 => 1 time(s)
/usr/lib64/libfontconfig.so.1 => 1 time(s)
/lib64/libkeyutils.so.1 => 1 time(s)
/lib64/libz.so.1 => 1 time(s)
/usr/lib64/libXpm.so.4 => 1 time(s)
/usr/lib64/libxml2.so.2 => 1 time(s)
/lib64/libgcrypt.so.11 => 1 time(s)
/usr/lib64/libpng12.so.0 => 1 time(s)
/lib64/libkrb5.so.3 => 1 time(s)
/lib64/libpthread.so.0 => 1 time(s)
/lib64/libgpg-error.so.0 => 1 time(s)
/usr/lib64/libfreetype.so.6 => 1 time(s)
/lib64/libkrb5support.so.0 => 1 time(s)
/usr/lib64/libX11.so.6 => 1 time(s)
/lib64/libgssapi_krb5.so.2 => 1 time(s)
/lib64/libdl.so.2 => 1 time(s)
/lib64/libresolv.so.2 => 1 time(s)
/usr/lib64/libexslt.so.0 => 1 time(s)
/usr/lib64/libxslt.so.1 => 1 time(s)

#############################
# Dependency libs got copied:
/opt/nginx-1.2.6/lib64/ld-linux-x86-64.so.2 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libxcb.so.1 => 1 time(s)
/opt/nginx-1.2.6/lib64/libk5crypto.so.3 => 1 time(s)
/opt/nginx-1.2.6/lib64/libdl.so.2 => 1 time(s)
/opt/nginx-1.2.6/lib64/libkeyutils.so.1 => 1 time(s)
/opt/nginx-1.2.6/lib64/libpcre.so.0 => 1 time(s)
/opt/nginx-1.2.6/lib64/libgpg-error.so.0 => 1 time(s)
/opt/nginx-1.2.6/lib64/libpthread.so.0 => 1 time(s)
/opt/nginx-1.2.6/lib64/libcom_err.so.2 => 1 time(s)
/opt/nginx-1.2.6/lib64/libgssapi_krb5.so.2 => 1 time(s)
/opt/nginx-1.2.6/lib64/libgcrypt.so.11 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libcrypto.so.10 => 1 time(s)
/opt/nginx-1.2.6/lib64/libfreebl3.so => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libfontconfig.so.1 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libfreetype.so.6 => 1 time(s)
/opt/nginx-1.2.6/lib64/libexpat.so.1 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libssl.so.10 => 1 time(s)
/opt/nginx-1.2.6/lib64/libm.so.6 => 1 time(s)
/opt/nginx-1.2.6/lib64/libkrb5support.so.0 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libpng12.so.0 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libxml2.so.2 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libXau.so.6 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libgd.so.2 => 1 time(s)
/opt/nginx-1.2.6/lib64/libresolv.so.2 => 1 time(s)
/opt/nginx-1.2.6/usr/local/lib/libGeoIP.so.1 => 1 time(s)
/opt/nginx-1.2.6/lib64/libkrb5.so.3 => 1 time(s)
/opt/nginx-1.2.6/lib64/libc.so.6 => 1 time(s)
/opt/nginx-1.2.6/lib64/libcrypt.so.1 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libX11.so.6 => 1 time(s)
/opt/nginx-1.2.6/lib64/libz.so.1 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libjpeg.so.62 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libXpm.so.4 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libexslt.so.0 => 1 time(s)
/opt/nginx-1.2.6/lib64/libselinux.so.1 => 1 time(s)
/opt/nginx-1.2.6/usr/lib64/libxslt.so.1 => 1 time(s)

##############################################################
#libs needed but already exists (exlcuding those copied over):

#########################
#Not matched LDD outputs:
linux-vdso.so.1 =>  (0x00007fff17658000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff13c00000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffc68d8000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff109b8000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff6ca00000) => 2 time(s)
linux-vdso.so.1 =>  (0x00007fff0e8c8000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff80a00000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffe5fd8000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff17a00000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff2b790000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff57200000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff8aa00000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff7dea8000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff69130000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff0e0e8000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff80738000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffa5d60000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff8cfa0000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffda800000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff281b0000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff0b9a0000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffd0e00000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff56a00000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff90400000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff94000000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffc6b10000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff28000000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffbf800000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffb9800000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffefe00000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffa0200000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffef388000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fff07200000) => 1 time(s)
linux-vdso.so.1 =>  (0x00007fffcd288000) => 1 time(s)

# test chroot
$ chroot /opt/nginx-1.2.6 /sbin/nginx -t
/sbin/nginx: error while loading shared libraries: libGeoIP.so.1: cannot open shared object file: No such file or directory

# why?

$ strace chroot /opt/nginx-1.2.6 /sbin/nginx -t

# shows that, when being chrooted. the dynamic lib loading path is
# open(“/lib64/libGeoIP.so.1”, O_RDONLY)  = -1 ENOENT (No such file or directory)
# open(“/usr/lib64/libGeoIP.so.1”, O_RDONLY) = -1 ENOENT (No such file or directory)

# so simple fix is either copy libGeoIP.so.1 to /lib/64 ? but it is not a 64bits lib. better not mix.
# try start with an env. variable of “LD_LIBRARY_PATH” and note, we need a startup script that export this env. also now

$ export LD_LIBRARY_PATH=/usr/local/lib
$ chroot /opt/nginx-1.2.6 /sbin/nginx -t

# the geoip lib loading error is gone, but we got a new error.
nginx: [alert] could not open error log file: open() “/opt/nginx-1.2.6/logs/error.log” failed (2: No such file or directory)
2013/01/13 04:37:44 [emerg] 683035#0: open() “/opt/nginx-1.2.6/conf/nginx.conf” failed (2: No such file or directory)
nginx: configuration file /opt/nginx-1.2.6/conf/nginx.conf test failed

# this is because we compile with –prefix=/opt/nginx-1.2.6 and log files are then /opt/nginx-1.2.6/logs
# in chroot of /opt/nginx-1.2.6 there is no /opt/nginx-1.2.6
# a simple trick to fix
$ mkdir /opt/nginx-1.2.6/opt
$ cd /opt/nginx-1.2.6/opt
$ ln -s .. nginx-1.2.6

$ chroot /opt/nginx-1.2.6 /sbin/nginx -t

# now the error is different…
nginx: the configuration file /opt/nginx-1.2.6/conf/nginx.conf syntax is ok
nginx: [emerg] getpwnam(“nginx”) failed (2: No such file or directory)
nginx: configuration file /opt/nginx-1.2.6/conf/nginx.conf test failed

# this looks like to be looking for nginx in /etc/passwd to launch the worker process.

$ mkdir /opt/nginx-1.2.6/etc
$ cd /opt/nginx-1.2.6/etc
$ cp /etc/passwd .
$ cp /etc/group .
$ cp /etc/shadow .
$ cd ..
$ cp /lib64/libnss* ./lib64

$ chroot /opt/nginx-1.2.6 /sbin/nginx -t
# Good!

# try start it…
$ chroot /opt/nginx-1.2.6 /sbin/nginx
nginx: [emerg] open(“/dev/null”) failed (2: No such file or directory)

# so we need various devices too

$ ls -l /dev/{null,random,urandom}
crw-rw-rw- 1 root root 1, 3 Jan  5 20:16 /dev/null
crw-rw-rw- 1 root root 1, 8 Jan  5 20:16 /dev/random
crw-rw-rw- 1 root root 1, 9 Jan  5 20:16 /dev/urandom

$ mkdir /opt/nginx-1.2.6/dev
$ mknod -m 0666 /opt/nginx-1.2.6/dev/null c 1 3
$ mknod -m 0666 /opt/nginx-1.2.6/dev/random c 1 8
$ mknod -m 0666 /opt/nginx-1.2.6/dev/urandom c 1 9

$ chroot /opt/nginx-1.2.6 /sbin/nginx
# Yes! it runs, now in the chroot

# however, we further strace the chroot start up process. it shows that we d better fix following too.
$ strace chroot /opt/nginx-1.2.6 /sbin/nginx > /tmp/nginx-chroot-start 2>&1

# missing libs ?
/lib64/tls/x86_64/libpthread.so.0
/lib64/tls/libpthread.so.0
/lib64/x86_64/libpthread.so.0
/lib64/libssl.so.10
/usr/lib64/tls/x86_64/libssl.so.10
/usr/lib64/tls/libssl.so.10
/usr/lib64/x86_64/libssl.so.10
/lib64/libcrypto.so.10
/lib64/libxml2.so.2
/lib64/libxslt.so.1
/lib64/libexslt.so.0
/lib64/libgd.so.2
/lib64/libXpm.so.4
/lib64/libX11.so.6
/lib64/libjpeg.so.62
/lib64/libfontconfig.so.1
/lib64/libfreetype.so.6
/lib64/libpng12.so.0
/lib64/libxcb.so.1
/lib64/libXau.so.6
/lib64/libnsl.so.1

# but only /lib64/libnsl.so.1 needs to be copied! all the rest we have a copy in different location in chroot.
# the errors are only due to shared lib searching

$ cp /lib64/libnsl.so.1 /opt/nginx-1.2.6/lib64

# missing file system
/etc/ld.so.cache
/etc/localtime
/etc/pki/tls/openssl.cnf
/etc/nsswitch.conf
/selinux
/proc/filesystems
/proc/sys/crypto/fips_enabled
/proc/stat
/proc/cpuinfo

# we tend to ignore the following at this point
/selinux
/proc/filesystems
/proc/sys/crypto/fips_enabled
/proc/stat
/proc/cpuinfo

# and copied the following:
/etc/ld.so.cache which is generated by $ ldconfig which reads config files in /etc/ld.so.conf.d/
/etc/localtime # if we change localtime later, we d better re-copy
/etc/pki/tls/openssl.cnf
/etc/nsswitch.conf

# now compare normal start and chroot start

$ strace chroot /opt/nginx-1.2.6 /sbin/nginx 2>&1 | grep -i ‘no such file’
access(“/etc/ld.so.preload”, R_OK)      = -1 ENOENT (No such file or directory)
access(“/etc/ld.so.preload”, R_OK)      = -1 ENOENT (No such file or directory)
statfs(“/selinux”, 0x7fff3ae8cde0)      = -1 ENOENT (No such file or directory)
open(“/proc/filesystems”, O_RDONLY)     = -1 ENOENT (No such file or directory)
open(“/proc/sys/crypto/fips_enabled”, O_RDONLY) = -1 ENOENT (No such file or directory)
open(“/proc/stat”, O_RDONLY|O_CLOEXEC)  = -1 ENOENT (No such file or directory)
open(“/proc/cpuinfo”, O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)

$ strace /opt/nginx-1.2.6/sbin/nginx 2>&1 | grep -i ‘no such file’
access(“/etc/ld.so.preload”, R_OK)      = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)

# only the /proc seems to be of some concern ?
# let us still try to fix it, though this does not seem to affect normal operation of nginx in chroot
# revisit later, will /proc mount in chroot be a security concern.

$ mkdir /opt/nginx-1.2.6/proc
$ mount -o bind /proc /opt/nginx-1.2.6/proc

$ strace chroot /opt/nginx-1.2.6 /sbin/nginx 2>&1 | grep -i ‘no such file’
access(“/etc/ld.so.preload”, R_OK)      = -1 ENOENT (No such file or directory)
access(“/etc/ld.so.preload”, R_OK)      = -1 ENOENT (No such file or directory)
statfs(“/selinux”, 0x7ffff5dc0ad0)      = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)
connect(4, {sa_family=AF_FILE, path=”/var/run/nscd/socket”}, 110) = -1 ENOENT (No such file or directory)

# now chroot nginx works same as the normal start now (/selinux is ignored)