[PATCH 21/21] lookaside: Take over file uploads
Mathieu Bridon
bochecha at fedoraproject.org
Wed May 6 11:53:17 UTC 2015
From: Mathieu Bridon <bochecha at daitauha.fr>
This is the last piece of code related to the lookaside cache
interaction which needed to be moved.
In addition to just moving the code, we now better handle errors, we
exclusively use pycurl instead of calling the curl command, and this
whole thing is completely covered by unit tests.
It also automatically handles usage of client-side and CA certificates,
to make it easier for downstreams like fedpkg.
The Commands._do_curl method is now completely unused, and is marked as
deprecated. However its implementation code is maintained, in case
downstreams were actually using it.
---
src/pyrpkg/__init__.py | 11 ++--
src/pyrpkg/lookaside.py | 61 ++++++++++++++++++++++
test/test_lookaside.py | 135 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 199 insertions(+), 8 deletions(-)
diff --git a/src/pyrpkg/__init__.py b/src/pyrpkg/__init__.py
index 391085c..d366395 100644
--- a/src/pyrpkg/__init__.py
+++ b/src/pyrpkg/__init__.py
@@ -914,7 +914,8 @@ class Commands(object):
return os.path.getmtime(file1) > os.path.getmtime(file2)
def _do_curl(self, file_hash, file):
- """Use curl manually to upload a file"""
+ warn_deprecated(self.__class__.__name__, '_do_curl',
+ 'lookasidecache.upload')
cmd = ['curl', '--fail', '-o', '/dev/null', '--show-error',
'--progress-bar', '-F', 'name=%s' % self.module_name,
@@ -2223,13 +2224,7 @@ class Commands(object):
if not gitignore.match(file_basename):
gitignore.add('/%s' % file_basename)
- if self.lookasidecache.remote_file_exists(
- self.module_name, file_basename, file_hash):
- self.log.info("File already uploaded: %s" % file_basename)
-
- else:
- self.log.info("Uploading: %s %s" % (file_hash, f))
- self._do_curl(file_hash, f)
+ self.lookasidecache.upload(self.module_name, f, file_hash)
sourcesf.write()
gitignore.write()
diff --git a/src/pyrpkg/lookaside.py b/src/pyrpkg/lookaside.py
index 240bd6a..6bf5c42 100644
--- a/src/pyrpkg/lookaside.py
+++ b/src/pyrpkg/lookaside.py
@@ -236,3 +236,64 @@ class CGILookasideCache(object):
self.log.debug(output)
raise UploadError('Error checking for %s at %s'
% (filename, self.upload_url))
+
+ def upload(self, name, filepath, hash):
+ """Upload a source file
+
+ Args:
+ name (str): The name of the module. (usually the name of the SRPM)
+ filepath (str): The full path to the file to upload.
+ hash (str): The known good hash of the file.
+ """
+ filename = os.path.basename(filepath)
+
+ if self.remote_file_exists(name, filename, hash):
+ self.log.info("File already uploaded: %s" % filepath)
+ return
+
+ self.log.info("Uploading: %s" % filepath)
+ post_data = [('name', name),
+ ('%ssum' % self.hashtype, hash),
+ ('file', (pycurl.FORM_FILE, filepath))]
+
+ with io.BytesIO() as buf:
+ c = pycurl.Curl()
+ c.setopt(pycurl.URL, self.upload_url)
+ c.setopt(pycurl.NOPROGRESS, False)
+ c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress)
+ c.setopt(pycurl.WRITEFUNCTION, buf.write)
+ c.setopt(pycurl.HTTPPOST, post_data)
+
+ if self.client_cert is not None:
+ if os.path.exists(self.client_cert):
+ c.setopt(pycurl.SSLCERT, self.client_cert)
+ else:
+ self.log.warn("Missing certificate: %s" % self.client_cert)
+
+ if self.ca_cert is not None:
+ if os.path.exists(self.ca_cert):
+ c.setopt(pycurl.CAINFO, self.ca_cert)
+ else:
+ self.log.warn("Missing certificate: %s" % self.ca_cert)
+
+ try:
+ c.perform()
+ status = c.getinfo(pycurl.RESPONSE_CODE)
+
+ except Exception as e:
+ raise UploadError(e)
+
+ finally:
+ c.close()
+
+ output = buf.getvalue().strip()
+
+ # Get back a new line, after displaying the download progress
+ sys.stdout.write('\n')
+ sys.stdout.flush()
+
+ if status != 200:
+ raise UploadError(output)
+
+ if output:
+ self.log.debug(output)
diff --git a/test/test_lookaside.py b/test/test_lookaside.py
index b6cc71d..eced096 100644
--- a/test/test_lookaside.py
+++ b/test/test_lookaside.py
@@ -454,3 +454,138 @@ class CGILookasideCacheTestCase(unittest.TestCase):
lc = CGILookasideCache('_', '_', '_')
self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg',
'pyrpkg-0.tar.xz', 'thehash')
+
+ @mock.patch('pyrpkg.lookaside.logging.getLogger')
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_upload(self, mock_curl, mock_logger):
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ def mock_perform():
+ curlopts[pycurl.WRITEFUNCTION](b'Some output')
+
+ def mock_debug(msg):
+ debug_messages.append(msg)
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 200
+ curl.perform.side_effect = mock_perform
+ curl.setopt.side_effect = mock_setopt
+
+ debug_messages = []
+ log = mock_logger.return_value
+ log.debug.side_effect = mock_debug
+
+ lc = CGILookasideCache('sha512', '_', '_')
+
+ with mock.patch.object(lc, 'remote_file_exists', lambda *x: False):
+ lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', 'thehash')
+
+ self.assertTrue(pycurl.HTTPPOST in curlopts)
+ self.assertEqual(curlopts[pycurl.HTTPPOST], [
+ ('name', 'pyrpkg'), ('sha512sum', 'thehash'),
+ ('file', (pycurl.FORM_FILE, 'pyrpkg-0.0.tar.xz'))])
+
+ self.assertEqual(debug_messages, [b'Some output'])
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_upload_already_exists(self, mock_curl):
+ curl = mock_curl.return_value
+
+ lc = CGILookasideCache('_', '_', '_')
+ hash = 'thehash'
+
+ with mock.patch.object(lc, 'remote_file_exists', lambda *x: True):
+ lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', hash)
+
+ self.assertEqual(curl.perform.call_count, 0)
+ self.assertEqual(curl.setopt.call_count, 0)
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_upload_with_custom_certs(self, mock_curl):
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 200
+ curl.setopt.side_effect = mock_setopt
+
+ client_cert = os.path.join(self.workdir, 'my-client-cert.cert')
+ with open(client_cert, 'w'):
+ pass
+
+ ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert')
+ with open(ca_cert, 'w'):
+ pass
+
+ lc = CGILookasideCache('_', '_', '_', client_cert=client_cert,
+ ca_cert=ca_cert)
+
+ with mock.patch.object(lc, 'remote_file_exists', lambda *x: False):
+ lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', 'thehash')
+
+ self.assertEqual(curlopts[pycurl.SSLCERT], client_cert)
+ self.assertEqual(curlopts[pycurl.CAINFO], ca_cert)
+
+ @mock.patch('pyrpkg.lookaside.logging.getLogger')
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_upload_missing_custom_certs(self, mock_curl, mock_logger):
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ def mock_warn(msg):
+ warn_messages.append(msg)
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 200
+ curl.setopt.side_effect = mock_setopt
+
+ warn_messages = []
+ log = mock_logger.return_value
+ log.warn.side_effect = mock_warn
+
+ client_cert = os.path.join(self.workdir, 'my-client-cert.cert')
+ ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert')
+
+ lc = CGILookasideCache('_', '_', '_', client_cert=client_cert,
+ ca_cert=ca_cert)
+
+ with mock.patch.object(lc, 'remote_file_exists', lambda *x: False):
+ lc.upload('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash')
+
+ self.assertTrue(pycurl.SSLCERT not in curlopts)
+ self.assertTrue(pycurl.CAINFO not in curlopts)
+ self.assertEqual(len(warn_messages), 2)
+ self.assertTrue('Missing certificate: ' in warn_messages[0])
+ self.assertTrue('Missing certificate: ' in warn_messages[1])
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_upload_failed(self, mock_curl):
+ curl = mock_curl.return_value
+ curl.perform.side_effect = Exception(
+ 'Could not resolve host: example.com')
+
+ lc = CGILookasideCache('_', '_', '_')
+
+ with mock.patch.object(lc, 'remote_file_exists', lambda *x: False):
+ self.assertRaises(UploadError, lc.upload, 'pyrpkg',
+ 'pyrpkg-0.tar.xz', 'thehash')
+
+ @mock.patch('pyrpkg.lookaside.pycurl.Curl')
+ def test_upload_failed_status_code(self, mock_curl):
+ def mock_setopt(opt, value):
+ curlopts[opt] = value
+
+ curlopts = {}
+ curl = mock_curl.return_value
+ curl.getinfo.return_value = 500
+ curl.setopt.side_effect = mock_setopt
+
+ lc = CGILookasideCache('sha512', '_', '_')
+
+ with mock.patch.object(lc, 'remote_file_exists', lambda *x: False):
+ self.assertRaises(UploadError, lc.upload, 'pyrpkg',
+ 'pyrpkg-0.tar.xz', 'thehash')
--
2.1.0
More information about the rel-eng
mailing list