benchmark: serve files via twistd

- remove benchmark.file_server
- move serve_files function to benchmark.download_sites module

Author: immerrr

splash/benchmark/benchmark.py

@@ -20,7 +20,7 @@
 import re
 
 import requests
-from splash.benchmark.file_server import serve_files
+from splash.benchmark.download_sites import serve_files
 from splash.tests.utils import SplashServer
 
 
@@ -68,8 +68,10 @@ def make_render_png_lua_req(splash, params):
 PORT = 8806
 #: Combinations of width & height to test.
 WIDTH_HEIGHT = [(None, None), (500, None), (None, 500), (500, 500)]
-#: Splash log filename.
+#: Splash log filename (set to None to put it to stdout).
 SPLASH_LOG = 'splash.log'
+#: Static file server log filename (set to None to put it to stdout).
+FILESERVER_LOG = 'fileserver.log'
 #: This script is used to collect maxrss & cpu time from splash process.
 GET_PERF_STATS_SCRIPT = """
 function main(splash)
@@ -172,7 +174,8 @@ def main():
                         '--disable-xvfb',
                         '--max-timeout=600'])
 
-    with splash, serve_files(PORT, args.sites_dir):
+    with splash, serve_files(port=PORT, directory=args.sites_dir,
+                             logfile=FILESERVER_LOG):
         start_time = time()
         results = parallel_map(invoke_request, generate_requests(splash, args),
                                args.thread_count)

splash/benchmark/download_sites.py

@@ -4,18 +4,20 @@
 Site downloader script for Splash benchmark suite.
 """
 
+from contextlib import contextmanager
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 import errno
 import json
 import os
 import re
 import subprocess
 from urlparse import urlsplit
+import time
 
+import requests
 from lxml import html
 
 import w3lib.html
-from splash.benchmark.file_server import serve_files
 from splash.tests.stress import lua_runonce
 
 SCRIPT_HTML = """
@@ -72,6 +74,31 @@ def preprocess_main_page(sites_dir, url):
     return filename
 
 
+@contextmanager
+def serve_files(port, directory, logfile=None):
+    """Serve files from specified directory statically in a subprocess."""
+    command = ['twistd',
+               '-n',    # don't daemonize
+               'web',   # start web component
+               '--port', str(int(port)),
+               '--path', os.path.abspath(directory), ]
+    if logfile is not None:
+        command += ['--logfile', logfile]
+    site_server = subprocess.Popen(command)
+    try:
+        # It might take some time to bring up the server, wait for up to 10s.
+        for i in xrange(100):
+            try:
+                requests.get('http://localhost:%d' % port)
+            except requests.ConnectionError:
+                time.sleep(0.1)
+            else:
+                break
+        yield
+    finally:
+        site_server.terminate()
+
+
 def download_sites(sites_dir, sites):
     local_files = [preprocess_main_page(sites_dir, s) for s in sites]