[PATCH] make the upload block size configurable
by xning
diff -Naur koji.old/cli/koji koji/cli/koji
--- koji.old/cli/koji 2016-03-04 13:23:36.000000000 +0800
+++ koji/cli/koji 2016-03-07 14:43:32.000000000 +0800
@@ -156,6 +156,7 @@
parser.add_option("--weburl", help=_("url of the Koji web interface"))
parser.add_option("--topurl", help=_("url for Koji file access"))
parser.add_option("--pkgurl", help=optparse.SUPPRESS_HELP)
+ parser.add_option("--upload-blocksize", help=_("upload the koji hup how many bytes per request"))
parser.add_option("--help-commands", action="store_true", default=False, help=_("list commands"))
(options, args) = parser.parse_args()
@@ -208,6 +209,7 @@
'keepalive' : True,
'timeout' : None,
'use_fast_upload': False,
+ 'upload_blocksize': 1048576,
'poll_interval': 5,
'krbservice': 'host',
'cert': '~/.koji/client.crt',
@@ -252,7 +254,8 @@
if name in ('anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'):
defaults[name] = config.getboolean(options.profile, name)
elif name in ('max_retries', 'retry_interval',
- 'offline_retry_interval', 'poll_interval', 'timeout'):
+ 'offline_retry_interval', 'poll_interval', 'timeout',
+ 'upload_blocksize'):
try:
defaults[name] = int(value)
except ValueError:
@@ -6900,8 +6903,9 @@
session_opts = {}
for k in ('user', 'password', 'krbservice', 'debug_xmlrpc', 'debug', 'max_retries',
- 'retry_interval', 'offline_retry', 'offline_retry_interval',
- 'anon_retry', 'keepalive', 'timeout', 'use_fast_upload'):
+ 'retry_interval', 'offline_retry', 'offline_retry_interval',
+ 'anon_retry', 'keepalive', 'timeout', 'use_fast_upload',
+ 'upload_blocksize'):
value = getattr(options,k)
if value is not None:
session_opts[k] = value
diff -Naur koji.old/koji/__init__.py koji/koji/__init__.py
--- koji.old/koji/__init__.py 2016-03-04 13:23:36.000000000 +0800
+++ koji/koji/__init__.py 2016-03-07 10:14:32.000000000 +0800
@@ -2016,7 +2016,10 @@
# raise AttributeError, "no attribute %r" % name
return VirtualMethod(self._callMethod,name)
- def fastUpload(self, localfile, path, name=None, callback=None, blocksize=1048576, overwrite=False):
+ def fastUpload(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=False):
+ if blocksize is None:
+ blocksize = self.opts.get('upload_blocksize')
+
if not self.logged_in:
raise ActionNotAllowed, 'You must be logged in to upload files'
if name is None:
@@ -2090,8 +2093,11 @@
request = chunk
return handler, headers, request
- def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=1048576, overwrite=True):
+ def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=True):
"""upload a file in chunks using the uploadFile call"""
+ if blocksize is None:
+ blocksize = self.opts.get('upload_blocksize')
+
if self.opts.get('use_fast_upload'):
self.fastUpload(localfile, path, name, callback, blocksize, overwrite)
return
8 years, 1 month
upload blocksize
by Dennis Gregorovic
Hi,
In the koji/__init__.py:fastUpload method there is a blocksize parameter that is used to determine how many bytes are uploaded to the koji hub per request. By default it is set to 1MB and as far as I can tell that is the value actually used. Is there a way to change this value via koji configuration? A larger blocksize value could significantly improve the upload throughput.
-- Dennis
8 years, 1 month
