Test mod_wsgi in Evergreen

behackett · behackett · commit 01f773c9f7e0 · 2017-04-27T12:14:02.000-07:00
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -327,6 +327,15 @@ functions:
           ${PREPARE_SHELL}
           sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh
 
+  "run mod_wsgi tests":
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+          PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh
+
   "run tests":
     - command: shell.exec
       type: test
@@ -622,6 +631,24 @@ tasks:
             TOPOLOGY: "server"
         - func: "run enterprise auth tests"
 
+    - name: "mod-wsgi-standalone"
+      tags: ["mod_wsgi"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            VERSION: "latest"
+            TOPOLOGY: "server"
+        - func: "run mod_wsgi tests"
+
+    - name: "mod-wsgi-replica-set"
+      tags: ["mod_wsgi"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            VERSION: "latest"
+            TOPOLOGY: "replica_set"
+        - func: "run mod_wsgi tests"
+
 # }}}
 
 
@@ -842,6 +869,21 @@ axes:
         display_name: "Jython 2.7"
         variables:
            PYTHON_BINARY: "/opt/python/jython2.7/bin/jython"
+  - id: mod-wsgi-version
+    display_name: "mod_wsgi version"
+    values:
+      - id: "2.8"
+        display_name: "mod_wsgi 2.8"
+        variables:
+           MOD_WSGI_VERSION: "2.8"
+      - id: "3.5"
+        display_name: "mod_wsgi 3.5"
+        variables:
+           MOD_WSGI_VERSION: "3.5"
+      - id: "4.5.15"
+        display_name: "mod_wsgi 4.5.15"
+        variables:
+           MOD_WSGI_VERSION: "4.5.15"
   - id: c-extensions
     display_name: "C Extensions"
     values:
@@ -1068,6 +1110,16 @@ buildvariants:
   tasks:
      - name: "test-enterprise-auth"
 
+- matrix_name: "tests-mod-wsgi"
+  matrix_spec: {"python-version": ["2.4", "2.6", "2.7"], "mod-wsgi-version": "*"}
+  exclude_spec:
+    python-version: ["2.4"]
+    mod-wsgi-version: ["3.5", "4.5.15"]
+  display_name: "${mod-wsgi-version} ${python-version}"
+  run_on: ubuntu1204-test
+  tasks:
+     - ".mod_wsgi"
+
       # Platform notes
       # i386 builds of OpenSSL or Cyrus SASL are not available
       # Ubuntu14.04 only supports 2.6+ with SSL
diff --git a/.evergreen/run-mod-wsgi-tests.sh b/.evergreen/run-mod-wsgi-tests.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+set -o xtrace
+set -o errexit
+
+APACHE=$(command -v apache2 || command -v /usr/lib/apache2/mpm-prefork/apache2) || true
+if [ -z "$APACHE" ]; then
+    echo "Could not find apache2 binary"
+    exit 1
+fi
+
+PYTHON_VERSION=$(${PYTHON_BINARY} -c "import sys; sys.stdout.write('.'.join(str(val) for val in sys.version_info[:2]))")
+
+if [ $MOD_WSGI_VERSION = "2.8" ] && [ $PYTHON_VERSION = "2.7" ]; then
+    # mod_wsgi 2.8 segfaults when built against the toolchain Python 2.7. Build
+    # against the system Python 2.7 instead.
+    git clone https://github.com/GrahamDumpleton/mod_wsgi.git
+    cd mod_wsgi
+    git checkout tags/2.8
+    ./configure
+    make
+    export MOD_WSGI_SO=$(pwd)/.libs/mod_wsgi.so
+    cd ..
+else
+    export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_version/$MOD_WSGI_VERSION/mod_wsgi.so
+    export PYTHONHOME=/opt/python/$PYTHON_VERSION
+fi
+
+cd ..
+$APACHE -k start -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/apache22ubuntu1204.conf
+trap "$APACHE -k stop -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/apache22ubuntu1204.conf" EXIT HUP
+
+set +e
+wget -t 1 -T 10 -O - "http://localhost:8080${PROJECT_DIRECTORY}"
+STATUS=$?
+set -e
+
+# Debug
+cat error_log
+
+if [ $STATUS != 0 ]; then
+    exit $STATUS
+fi
+
+${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel http://localhost:8080${PROJECT_DIRECTORY}
+
+${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial http://localhost:8080${PROJECT_DIRECTORY}
+
diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst
@@ -27,17 +27,12 @@ Compile Python
 We need a Python interpreter built as a shared library. Download the
 source tarball for each Python version tested, untar it, and run::
 
-    ./configure --prefix=/some/directory --enable-shared
+    ./configure --prefix=/some/directory --enable-shared LDFLAGS="-Wl,--rpath=/some/directory/lib"
     make
     make install
 
-This results in an executable named "python" and a shared
-library named something like "libpython2.7.so.1.0".
-It may be necessary to add /some/directory/lib to your system's
-LD_LIBRARY_PATH, or to make a symlink from your system's default library
-directory to the shared library. For example, on Ubuntu::
-
-    ln -s /some/directory/lib/libpython2.7.so.1.0 /usr/lib64/
+This results in an executable named "python" or "python3" and a shared
+library named something like "libpython2.7.so.1.0" or "libpython3.3m.so.1.0".
 
 Compile mod_wsgi
 ................
@@ -50,7 +45,7 @@ RedHat-like Linux::
     wget https://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz
     tar xzf mod_wsgi-3.4.tar.gz
     cd mod_wsgi-3.4
-    ./configure --with-python=/some/directory/python
+    ./configure --with-python=/some/directory/bin/python LDFLAGS="-Wl,--rpath=/some/directory/lib"
     make
     make install
 
@@ -66,8 +61,12 @@ listening on port 27017.
 Configure Apache
 ................
 
-Copy the appropriate version of ``mod_wsgi.so`` into Apache's modules
-directory. Start Apache with the ``mod_wsgi_test.conf`` in this directory.
+Set a MOD_WSGI_SO environment variable so our ``mod_wsgi_test.conf``
+can find and load mod_wsgi.so::
+
+    export MOD_WSGI_SO=/path/to/mod_wsgi.so
+
+Start Apache with one of the config files in this directory.
 
 Run the test
 ------------
@@ -90,7 +89,7 @@ The ``test_client.py`` script merely makes HTTP requests to Apache. Its
 exit code is non-zero if any of its requests fails, for example with an
 HTTP 500.
 
-The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi`.
+The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi``.
 This script inserts some documents into MongoDB at startup, then queries
 documents for each HTTP request.
 
@@ -100,7 +99,6 @@ the test will fail when PyMongo exhausts its file descriptors.
 Automation
 ----------
 
-At MongoDB, Inc. we use a Jenkins job that tests each combination in the
-matrix. The job copies the appropriate version of ``mod_wsgi.so`` into
-place, sets up Apache, starts a single server or replica set,
-and runs ``test_client.py`` with the proper arguments.
+At MongoDB, Inc. we use a continuous integration job that tests each
+combination in the matrix. The job starts up Apache, starts a single server
+or replica set, and runs ``test_client.py`` with the proper arguments.
diff --git a/test/mod_wsgi_test/apache22amazon.conf b/test/mod_wsgi_test/apache22amazon.conf
@@ -0,0 +1,31 @@
+# This is a minimal httpd.conf file written for Apache 2.2 on Amazon Linux
+ 
+# The modules directory is here by default.
+# ServerRoot "/etc/httpd"
+DocumentRoot ${PWD}
+PidFile ${PWD}/apache2.pid
+ 
+# Bind to localhost only, don't require sudo.
+Listen 127.0.0.1:8080
+ 
+# Required modules.
+LoadModule authz_host_module modules/mod_authz_host.so
+# Needed so we can set a custom log location.
+LoadModule log_config_module modules/mod_log_config.so
+ 
+ErrorLog ${PWD}/error_log
+CustomLog ${PWD}/access_log combined
+ 
+<Directory "/">
+    AllowOverride None
+    Order Deny,Allow
+    Deny from All
+</Directory>
+ 
+<Directory ${PWD}>
+    AllowOverride None
+    Order Allow,Deny
+    Allow from All
+</Directory>
+
+Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf
diff --git a/test/mod_wsgi_test/apache22ubuntu1204.conf b/test/mod_wsgi_test/apache22ubuntu1204.conf
@@ -0,0 +1,29 @@
+# This is a minimal httpd.conf file written for Apache 2.2 on Ubuntu 12.04
+ 
+# The modules directory is here on Ubuntu.
+ServerRoot "/usr/lib/apache2"
+DocumentRoot ${PWD}
+PidFile ${PWD}/apache2.pid
+ 
+# Bind to localhost only, don't require sudo.
+Listen 127.0.0.1:8080
+ 
+# Required modules.
+LoadModule authz_host_module modules/mod_authz_host.so
+ 
+ErrorLog ${PWD}/error_log
+CustomLog ${PWD}/access_log combined
+ 
+<Directory "/">
+    AllowOverride None
+    Order Deny,Allow
+    Deny from All
+</Directory>
+
+<Directory ${PWD}>
+    AllowOverride None
+    Order Allow,Deny
+    Allow from All
+</Directory>
+ 
+Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf
diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf
@@ -14,7 +14,7 @@
 
 # Minimal test of PyMongo in a WSGI application, see bug PYTHON-353
 
-LoadModule wsgi_module modules/mod_wsgi.so
+LoadModule wsgi_module ${MOD_WSGI_SO}
 
 # Avoid permissions issues
 WSGISocketPrefix /tmp/
diff --git a/test/mod_wsgi_test/mod_wsgi_test.wsgi b/test/mod_wsgi_test/mod_wsgi_test.wsgi
@@ -55,7 +55,7 @@ except:
 def application(environ, start_response):
     results = list(collection.find().batch_size(10))
     assert len(results) == ndocs
-    output = 'python %s, mod_wsgi %s, pymongo %s' % (
+    output = ' python %s, mod_wsgi %s, pymongo %s ' % (
         sys.version, mod_wsgi_version, pymongo.version)
     response_headers = [('Content-Length', str(len(output)))]
     start_response('200 OK', response_headers)
