From c38bc4d53aa51694eae20d65423a4dd08b713b40 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 22 Aug 2017 09:30:38 +0200 Subject: [PATCH] Use a higher value for connecion_pool_maxsize. Fixes #26 --- configuration.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/configuration.py b/configuration.py index 3e211d11..924f6e01 100644 --- a/configuration.py +++ b/configuration.py @@ -20,6 +20,7 @@ from __future__ import absolute_import +import multiprocessing import logging import sys @@ -83,11 +84,12 @@ def __init__(self): # Set this to True/False to enable/disable SSL hostname verification. self.assert_hostname = None # urllib3 connection pool's maximum number of connections saved - # per pool. Increasing this is useful for cases when you are - # making a lot of possibly parallel requests to the same host, - # which is often the case here. - # When set to `None`, will default to whatever urllib3 uses - self.connection_pool_maxsize = None + # per pool. urllib3 uses 1 connection as default value, but this is + # not the best value when you are making a lot of possibly parallel + # requests to the same host, which is often the case here. + # cpu_count * 5 is used as default value to increase performance + # This is used because it's the default value for ThreadPoolExecutor + self.connection_pool_maxsize = multiprocessing.cpu_count() * 5 # http proxy setting self.http_proxy_url = None