Createrepo in rawhide now has parallel drpm creation, this patch should hopefully enable it in mash (with 8 workers).
Note that it's missing a createrepo version check right now since there's not yet been a released createrepo with the support.
kevin --- configs/rawhide.mash | 1 + mash/__init__.py | 6 +++++- mash/config.py | 2 ++ mash/metadata.py | 5 +++-- 4 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/configs/rawhide.mash b/configs/rawhide.mash index 1deba24..69992c9 100644 --- a/configs/rawhide.mash +++ b/configs/rawhide.mash @@ -17,6 +17,7 @@ arches = i386 x86_64 armhfp delta = True max_delta_rpm_size = 800000000 max_delta_rpm_age = 604800 +delta_workers = 8 # Change distro_tags as fedora-release version gets bumped distro_tags = cpe:/o:fedoraproject:fedora:21 rawhide hash_packages = True diff --git a/mash/__init__.py b/mash/__init__.py index 8ced405..566b8c1 100644 --- a/mash/__init__.py +++ b/mash/__init__.py @@ -121,7 +121,11 @@ class Mash: max_delta_rpm_age = self.config.max_delta_rpm_age else: max_delta_rpm_age = None # No age specified. Copy all. - md.set_delta(paths, max_delta_rpm_size, max_delta_rpm_age) + if self.config.delta_workers: + delta_workers = self.config.delta_workers + else: + delta_workers = 1 # default to 1 worker + md.set_delta(paths, max_delta_rpm_size, max_delta_rpm_age, delta_workers) if previous: md.set_previous(previous) # Setup the distro tags diff --git a/mash/config.py b/mash/config.py index 1e9f140..e4e9b95 100644 --- a/mash/config.py +++ b/mash/config.py @@ -47,6 +47,7 @@ class MashConfig(config.BaseConfig): repoviewtitle = config.Option('"Mash - %(arch)s"') delta = config.BoolOption(False) delta_dirs = config.ListOption() + delta_workers = config.Option(1) max_delta_rpm_size = config.Option(300000000) max_delta_rpm_age = config.Option() make_ancient = config.BoolOption(False) @@ -90,6 +91,7 @@ class MashDistroConfig(config.BaseConfig): delta_dirs = config.Inherit(MashConfig.delta_dirs) max_delta_rpm_size = config.Inherit(MashConfig.delta_dirs) max_delta_rpm_age = config.Inherit(MashConfig.max_delta_rpm_age) + delta_workers = config.Inherit(MashConfig.delta_workers) make_ancient = config.Inherit(MashConfig.make_ancient) latest = config.Inherit(MashConfig.latest) distro_tags = config.Inherit(MashConfig.distro_tags) diff --git a/mash/metadata.py b/mash/metadata.py index e20b1d6..50a67da 100644 --- a/mash/metadata.py +++ b/mash/metadata.py @@ -71,7 +71,7 @@ class MetadataOld: if skip: self.args.append('--skip-stat')
- def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age): + def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age, delta_workers): # Sorry, can't do that here. pass
@@ -136,12 +136,13 @@ class MetadataNew: def set_skipstat(self, skip): self.conf.skip_stat = skip
- def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age): + def set_delta(self, deltapaths, max_delta_rpm_size, max_delta_rpm_age, delta_workers): if rpm.labelCompare([createrepo.__version__,'0','0'], ['0.9.7', '0', '0']) >= 0: self.conf.deltas = True self.conf.oldpackage_paths = deltapaths self.conf.max_delta_rpm_size = max_delta_rpm_size self.conf.max_delta_rpm_age = max_delta_rpm_age + self.conf.delta_workers = delta_workers
def set_previous(self, previous): if rpm.labelCompare([createrepo.__version__,'0','0'], ['0.9.7', '0', '0']) >= 0:
-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1
On Wed, 4 Jun 2014 18:27:58 -0600 Kevin Fenzi kevin@scrye.com wrote:
Createrepo in rawhide now has parallel drpm creation, this patch should hopefully enable it in mash (with 8 workers).
Note that it's missing a createrepo version check right now since there's not yet been a released createrepo with the support.
kevin
Applied thanks
Dennis
On 06/05/2014 03:27 AM, Kevin Fenzi wrote:
Createrepo in rawhide now has parallel drpm creation, this patch should hopefully enable it in mash (with 8 workers).
Note that it's missing a createrepo version check right now since there's not yet been a released createrepo with the support.
Is going to cause the createrepo server to run out of RAM? I only ask because drpm generation used to take (uncompressed size of rpm * 3) RAM. This was supposed to be fixed in deltarpm-3.6, but in my tests it still took at least (uncompressed size of rpm) RAM.
I know createrepo sets a maximum size on the rpms that it will delta, but I don't know the specs of the createrepo server, and I don't remember what the maximum size was, so if the server doesn't have enough RAM and createrepo ends up generating eight large drpms at the same time, it might run out.
FWIW, I think the patch should go in, but we should be aware that any drpm generation errors will most likely be because of running out of memory.
Jonathan
On Thu, 05 Jun 2014 15:23:49 +0300 Jonathan Dieter jdieter@lesbg.com wrote:
On 06/05/2014 03:27 AM, Kevin Fenzi wrote:
Createrepo in rawhide now has parallel drpm creation, this patch should hopefully enable it in mash (with 8 workers).
Note that it's missing a createrepo version check right now since there's not yet been a released createrepo with the support.
Is going to cause the createrepo server to run out of RAM? I only ask because drpm generation used to take (uncompressed size of rpm * 3) RAM. This was supposed to be fixed in deltarpm-3.6, but in my tests it still took at least (uncompressed size of rpm) RAM.
I know createrepo sets a maximum size on the rpms that it will delta, but I don't know the specs of the createrepo server, and I don't remember what the maximum size was, so if the server doesn't have enough RAM and createrepo ends up generating eight large drpms at the same time, it might run out.
FWIW, I think the patch should go in, but we should be aware that any drpm generation errors will most likely be because of running out of memory.
Sure. Right now the rawhide compose machine has 16gb ram and 16 cpus. If we run out of memory, we can back off on the drpm workers, or add more memory.
kevin
buildsys@lists.fedoraproject.org
-
Dennis Gilmore -
Jonathan Dieter -
Kevin Fenzi