Freeze Break: fix pkgdb_sync_git_branches.py for git being too clever
Stephen John Smoogen
smooge at gmail.com
Fri Mar 6 21:47:05 UTC 2015
Read and reviewed. +1
On 6 March 2015 at 14:44, Kevin Fenzi <kevin at scrye.com> wrote:
> ok, here's a final final version...
>
> This one doesn't output anything normally (unless you set VERBOSE=True).
>
> Patrick +1'ed this on irc. One more?
>
> kevin
> --
> diff --git a/roles/distgit/templates/pkgdb_sync_git_branches.py
> b/roles/distgit/templates/pkgdb_sync_git_branches.py
> index 8643165..9055e48 100644
> --- a/roles/distgit/templates/pkgdb_sync_git_branches.py
> +++ b/roles/distgit/templates/pkgdb_sync_git_branches.py
> @@ -26,8 +26,10 @@ the missing branches (or even the missing repo)
>
> """
>
> +import multiprocessing.pool
> import os
> import subprocess
> +import time
>
> import requests
>
> @@ -56,6 +58,7 @@ GIT_FOLDER = '/srv/git/rpms/'
> MKBRANCH = '/usr/local/bin/mkbranch'
> SETUP_PACKAGE = '/usr/local/bin/setup_git_package'
>
> +THREADS = 20
> VERBOSE = False
>
>
> @@ -67,7 +70,7 @@ class ProcessError(InternalError):
> pass
>
>
> -def _invoke(program, args):
> +def _invoke(program, args, cwd=None):
> '''Run a command and raise an exception if an error occurred.
>
> :arg program: The program to invoke
> @@ -79,63 +82,60 @@ def _invoke(program, args):
> cmdLine.extend(args)
> if VERBOSE:
> print ' '.join(cmdLine)
> + print ' in', cwd
>
> - if VERBOSE:
> - program = subprocess.Popen(
> - cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
> - else:
> - program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT)
> + program = subprocess.Popen(
> + cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
> cwd=cwd)
> +
> + stdout, stderr = program.communicate()
>
> - retCode = program.wait()
> - if retCode != 0:
> + if program.returncode != 0:
> e = ProcessError()
> - e.returnCode = retCode
> + e.returnCode = program.returncode
> e.cmd = ' '.join(cmdLine)
> - if VERBOSE:
> - output = program.stdout.read()
> - e.message = 'Error, "%s" returned %s: %s' % (
> - e.cmd, e.returnCode, output)
> - print e.message
> - else:
> - e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode)
> + e.cwd = cwd
> + e.message = 'Error, "%s" (in %r) returned %s\n stdout: %s\n
> stderr: %s' % (
> + e.cmd, e.cwd, e.returnCode, stdout, stderr)
> + print e.message
> raise e
>
> + return stdout.strip()
>
> -def _create_branch(pkgname, branch):
> +
> +def _create_branch(pkgname, branch, existing_branches):
> '''Create a specific branch for a package.
>
> :arg pkgname: Name of the package to branch
> :arg branch: Name of the branch to create
> + :arg existing_branches: A list of the branches that already exist
> locally.
>
> '''
> + branch = branch.replace('*', '').strip()
> if branch == 'master':
> print 'ERROR: Proudly refusing to create master branch. Invalid
> repo?'
> print 'INFO: Please check %s repo' % pkgname
> return
>
> - branchpath = os.path.join(
> - GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch)
> - if not os.path.exists(branchpath):
> - try:
> - _invoke(MKBRANCH, [branch, pkgname])
> - except ProcessError, e:
> - if e.returnCode == 255:
> - # This is a warning, not an error
> - return
> - raise
> - finally:
> - fedmsg.publish(
> - topic='branch',
> - modname='git',
> - msg=dict(
> - agent='pkgdb',
> - name=pkgname,
> - branch=branch,
> - ),
> - )
> - elif VERBOSE:
> - print 'Was asked to create branch %s of package %s, but it '\
> - 'already exists' % (pkgname, branch)
> + if branch in existing_branches:
> + print 'ERROR: Refusing to create a branch %s that exists' % branch
> + return
> +
> + try:
> + _invoke(MKBRANCH, [branch, pkgname])
> + fedmsg.publish(
> + topic='branch',
> + modname='git',
> + msg=dict(
> + agent='pkgdb',
> + name=pkgname,
> + branch=branch,
> + ),
> + )
> + except ProcessError, e:
> + if e.returnCode == 255:
> + # This is a warning, not an error
> + return
> + raise
>
>
> def pkgdb_pkg_branch():
> @@ -168,43 +168,48 @@ def get_git_branch(pkg):
> """
> git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg)
> if not os.path.exists(git_folder):
> - print 'Could not find %s' % git_folder
> + if VERBOSE:
> + print 'Could not find %s' % git_folder
> return set()
>
> - head_folder = os.path.join(git_folder, 'refs', 'heads')
> - return set(os.listdir(head_folder))
> + branches = [
> + lclbranch.replace('*', '').strip()
> + for lclbranch in _invoke('git', ['branch'],
> cwd=git_folder).split('\n')
> + ]
> + return set(branches)
>
>
> -def branch_package(pkgname, branches):
> +def branch_package(pkgname, requested_branches, existing_branches):
> '''Create all the branches that are listed in the pkgdb for a package.
>
> :arg pkgname: The package to create branches for
> - :arg branches: The branches to creates
> + :arg requested_branches: The branches to creates
> + :arg existing_branches: A list of existing local branches
>
> '''
> if VERBOSE:
> - print 'Fixing package %s for branches %s' % (pkgname, branches)
> + print 'Fixing package %s for branches %s' % (pkgname,
> requested_branches)
>
> # Create the devel branch if necessary
> - if not os.path.exists(
> - os.path.join(GIT_FOLDER, '%s.git/refs/heads/master' %
> pkgname)):
> + exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname))
> + if not exists or 'master' not in existing_branches:
> _invoke(SETUP_PACKAGE, [pkgname])
> - if 'master' in branches:
> - branches.remove('master') # SETUP_PACKAGE creates master
> - fedmsg.publish(
> - topic='branch',
> - modname='git',
> - msg=dict(
> - agent='pkgdb',
> - name=pkgname,
> - branch='master',
> - ),
> - )
> + if 'master' in requested_branches:
> + requested_branches.remove('master') # SETUP_PACKAGE creates
> master
> + fedmsg.publish(
> + topic='branch',
> + modname='git',
> + msg=dict(
> + agent='pkgdb',
> + name=pkgname,
> + branch='master',
> + ),
> + )
>
> # Create all the required branches for the package
> # Use the translated branch name until pkgdb falls inline
> - for branch in branches:
> - _create_branch(pkgname, branch)
> + for branch in requested_branches:
> + _create_branch(pkgname, branch, existing_branches)
>
>
> def main():
> @@ -214,10 +219,14 @@ def main():
>
> local_pkgs = set(os.listdir(GIT_FOLDER))
> local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
> + if VERBOSE:
> + print "Found %i local packages" % len(local_pkgs)
>
> pkgdb_info = pkgdb_pkg_branch()
>
> pkgdb_pkgs = set(pkgdb_info.keys())
> + if VERBOSE:
> + print "Found %i pkgdb packages" % len(pkgdb_pkgs)
>
> ## Commented out as we keep the git of retired packages while they
> won't
> ## show up in the information retrieved from pkgdb.
> @@ -230,19 +239,38 @@ def main():
> print 'Some packages are present in pkgdb but not locally:'
> print ', '.join(sorted(pkgdb_pkgs - local_pkgs))
>
> +
> + if VERBOSE:
> + print "Finding the lists of local branches for local repos."
> + start = time.time()
> + if THREADS == 1:
> + git_branch_lookup = map(get_git_branch, sorted(pkgdb_info))
> + else:
> + threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
> + git_branch_lookup = threadpool.map(get_git_branch,
> sorted(pkgdb_info))
> +
> + # Zip that list of results up into a lookup dict.
> + git_branch_lookup = dict(zip(sorted(pkgdb_info), git_branch_lookup))
> +
> + if VERBOSE:
> + print "Found all local git branches in %0.2fs" % (time.time() -
> start)
> +
> tofix = set()
> for pkg in sorted(pkgdb_info):
> pkgdb_branches = pkgdb_info[pkg]
> - git_branches = get_git_branch(pkg)
> + git_branches = git_branch_lookup[pkg]
> diff = (pkgdb_branches - git_branches)
> if diff:
> print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
> tofix.add(pkg)
> - branch_package(pkg, diff)
> + branch_package(pkg, diff, git_branches)
>
> if tofix:
> print 'Packages fixed (%s): %s' % (
> len(tofix), ', '.join(sorted(tofix)))
> + else:
> + if VERBOSE:
> + print 'Didn\'t find any packages to fix.'
>
>
> if __name__ == '__main__':
>
> _______________________________________________
> infrastructure mailing list
> infrastructure at lists.fedoraproject.org
> https://admin.fedoraproject.org/mailman/listinfo/infrastructure
>
--
Stephen J Smoogen.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.fedoraproject.org/pipermail/infrastructure/attachments/20150306/7f809080/attachment.html>
More information about the infrastructure
mailing list