#!/usr/bin/python
# -*- python -*-
#
# find-missing-debuginfo
# Karel Klic <kklic@redhat.com>
#
# Checks the corectness and completness of debuginfo packages.
#
# This script checks for selected issues in packages:
#
# 1) A binary (an executable or a shared library) does not have an
# associated debuginfo file in -debuginfo packages, and the binary
# does not contain debugging symbols (it is stripped).  It might be
# caused by the component's build script stripping the binary before
# rpmbuild can generate the -debuginfo package (by calling
# /usr/lib/rpm/find-debuginfo.sh).  This breaks debugging of a crash
# in this binary.
#
# 2) A binary does not have an associated debuginfo file in -debuginfo
# packages, and the binary contains debugging symbols (it is not
# stripped).  It might be caused by the component's build script
# installing the binary with invalid permissions - common issue is
# that the executable bits not set for a shared library.  This breaks
# some debugging scenarios, e.g. the retrace server: when it analyzes
# a coredump, it can get the build ids of all binaries from the
# coredump, and use the build-ids to find appropriate
# packages. However, if the binary is not stripped and thus no
# debuginfo is present, the package repository cannot be searched for
# the exact package that participated in the crash, because the
# build-id cannot be found in yum metadata.
#
# 3) A binary has an associated debuginfo, but the symlink in the
# debuginfo points to another binary, which does not exist.  It might
# be caused by packaging the binary under different name from what has
# been installed into the build root by component's build script.
# This prevents GDB from finding the binary and breaks some debugging
# scenarios.
#
# 4) A binary has an associated debuginfo, but the symlink in the
# debuginfo points to another binary, and the package containing that
# binary is not present in the dependencies of the package of the
# checked binary.  It might be caused by packaging the same binary
# multiple times in multiple packages, or by building the same binary
# multiple times under different names.  It is ok when all the
# packages containing the binary (=same build-id) depend on the
# package with the binary the debuginfo symlink is pointing to.  This
# issue can be fixed for all packages at once by fixing packages
# rpm-build and gdb - see rhbz#641377.
#
# 5) There are debug symbols present for unpackaged binaries in the
# -debuginfo package.  It might be caused by leaving an intentionally
# unpackaged binary in the build root, where
# /usr/lib/rpm/find-debuginfo.sh finds it.  The unused debuginfo files
# are not a serious problem, they just waste space.
#
# Requirements:
#
# Packages python, yum, elfutils, cpio, file; 10 GB free disk space;
# it might take several days to get the results for Fedora, depending
# on the access to the nearest package repository
#
# Usage:
#  terminal one (results):
#   $ ./find-missing-debuginfo --repos="rawhide" \
#         --log=rawhide.log
#  terminal two (progress logging):
#   $ tail -f rawhide.log
#
# Use the --ignore-unused-debuginfo option on RHEL to suppress noise
# caused by operating system variants.  Use --fedora-component-owners
# on Fedora to include the component owners to the report.
#
import subprocess
import yum
import sys
import argparse
import os
import os.path
import re
import shutil
import urllib
import json
parser = argparse.ArgumentParser(description='Get packages for coredump processing.')
parser.add_argument('--repos', default='fedora', metavar='WILDCARD',
                    help='Yum repositories to be checked')
parser.add_argument('--log', metavar='FILENAME',
                    help='Store debug output to a file')
parser.add_argument('--ignore-unused-debuginfo', action='store_true')
parser.add_argument('--fedora-component-owners', action='store_true')
parser.add_argument('--offset', type=int)
args = parser.parse_args()
if args.log:
    log = open(args.log, "w", 0)
else:
    log = open("/dev/null", "w")
# Initialize yum, enable only repositories specified via command line
# --repos option.
stdout = sys.stdout
sys.stdout = log
yumbase = yum.YumBase()
yumbase.doConfigSetup()
if not yumbase.setCacheDir():
    exit(2)
log.write("Closing all enabled repositories...\n")
for repo in yumbase.repos.listEnabled():
    log.write(" - {0}\n".format(repo.name))
    repo.close()
    yumbase.repos.disableRepo(repo.id)
log.write("Enabling repositories matching \'{0}\'...\n".format(args.repos))
for repo in yumbase.repos.findRepos(args.repos):
    log.write(" - {0}\n".format(repo.name))
    repo.enable()
    repo.skip_if_unavailable = True
yumbase.repos.doSetup()
yumbase.repos.populateSack(mdtype='metadata', cacheonly=1)
yumbase.repos.populateSack(mdtype='filelists', cacheonly=1)
sys.stdout = stdout
# Use the enabled repos to get all their packages.
log.write("Getting the list of all packages...")
package_list = yumbase.pkgSack.returnPackages()
log.write("{0} packages found\n".format(len(package_list)))
# Partition the packages by component
log.write("Partitioning packages by component...\n")
components = {}
for package in package_list:
    if not package.base_package_name in components:
        components[package.base_package_name] = [package]
    else:
        components[package.base_package_name].append(package)
def unpack_package(package):
    # Download the package.
    log.write("  - downloading {0}\n".format(package))
    repo = yumbase.repos.getRepo(package.repoid)
    remote = package.returnSimple('relativepath')
    local = os.path.basename(remote)
    package.localpath = local
    path = repo.getPackage(package)
    if not os.path.exists(local) or not os.path.samefile(path, local):
        shutil.copy2(path, local)
    # Convert it to cpio
    log.write("  - converting {0} to cpio\n".format(local))
    cpio = open(local + ".cpio", "wb")
    rpm2cpio_proc = subprocess.Popen(['rpm2cpio', local], stdout=cpio)
    rpm2cpio_proc.communicate()
    if rpm2cpio_proc.returncode != 0:
        sys.stderr.write("Rpm2cpio failed.\n")
        exit(1)
    cpio.close()
    # Unpack the cpio
    rpmdir = re.sub('\\.rpm$', '', local)
    if os.path.exists(rpmdir):
        shutil.rmtree(rpmdir)
    os.makedirs(rpmdir)
    cpio = open(local + ".cpio", "rb")
    cpio_proc = subprocess.Popen(["cpio", "--extract", "-d", "--quiet"],
                                 stdin=cpio, cwd=rpmdir, bufsize=-1)
    cpio_proc.communicate()
    if cpio_proc.returncode != 0:
        sys.stderr.write("Cpio failed.\n")
        exit(1)
    cpio.close()
    os.unlink(local)
    os.unlink(local + ".cpio")
    # Set sane access rights. The file command fails to work reliably
    # on suid binaries without user read access, and also some files
    # and directories are not removable by default.
    for root, dirs, files in os.walk(rpmdir):
        for f in files:
            ff = os.path.join(root, f)
            if not os.path.islink(ff):
                os.chmod(ff, 0644)
        for d in dirs:
            dd = os.path.join(root, d)
            if not os.path.islink(dd):
                os.chmod(dd, 0755)
    return rpmdir
# Check all the components.
index = 0
log.write("Checking components...\n")
components_keys = sorted(components.keys())
if args.offset:
    args.offset -= 1
    components_keys = components_keys[args.offset:]
    index = args.offset
for component in components_keys:
    index += 1
    log.write("[{0}/{1}] Checking {2}\n".format(index, len(components.keys()), component))
    if component == "udev":
        log.write("  - skipping because cpio cannot extract the udev package\n")
        continue
    # Download and unpack all packages.
    packages = components[component]
    common_package_dirs = []
    debuginfo_package_dirs = []
    # Mapping of a local directory with extracted rpm to package name.
    rpmdir_to_package_name = {}
    # Mapping from package name to a list of package names the package
    # depends on. The list contains recursive dependencies.
    log.write("  - building provides/requires list\n")
    requires = {}
    provides = {}
    for package in packages:
        if package.arch == "noarch":
            continue
        rpmdir = unpack_package(package)
        rpmdir_to_package_name[rpmdir] = package.name
        # Build the provides and requires lists for the package.
        provides[package.name] = []
        for prov in package.returnPrco('provides'):
            provides[package.name].append(prov[0])
        requires[package.name] = set()
        for req in package.returnPrco('requires'):
            requires[package.name].add(req[0])
        # Put the package to debuginfo or non-debuginfo (common) bucket.
        if -1 == package.name.find("-debuginfo"):
            common_package_dirs.append(str(rpmdir))
        else:
            debuginfo_package_dirs.append(str(rpmdir))
    # Propagate requires and provides recursively.  First step is to
    # change all known file dependencies to package dependencies in
    # requires.  Use provides for this.
    for provides_package, package_provides in provides.items():
        for provide in package_provides:
            for requires_package, package_requires in requires.items():
                for require in package_requires.copy():
                    if require == provide:
                        package_requires.remove(require)
                        package_requires.add(provides_package)
    # Build the transitive closure of requires on the requires of one
    # component's packages.
    for requires_package, package_requires in requires.items():
        while True:
            original_package_requires = package_requires.copy()
            for require in original_package_requires:
                if require in requires:
                    package_requires |= requires[require]
            if len(package_requires - original_package_requires) == 0:
                break
    # Find all ELF binaries in the common packages and do the checking.
    component_problems = {}
    used_debuginfo_paths = []
    def add_problem(f, problem):
        if not f in component_problems:
            component_problems[f] = [problem]
        else:
            component_problems[f].append(problem)
    for package_dir in common_package_dirs:
        rpm_files = []
        for root, dirs, files in os.walk(package_dir):
            for f in files:
                rpm_files.append(os.path.join(root, f))
        for f in rpm_files:
            # The file utility recognizes an ELF binary and also tells
            # whether it is stripped or not.
            file_proc = subprocess.Popen(["file", f], stdout=subprocess.PIPE)
            file_out = file_proc.communicate()[0]
            if file_proc.returncode != 0:
                sys.stderr.write("File call failed.\n")
                exit(1)
            if -1 != file_out.find(" ELF "):
                log.write("  - checking {0}\n".format(f))
                # Get its build id
                readelf_proc = subprocess.Popen(["eu-readelf", "--notes", f],
                                                stdout=subprocess.PIPE)
                readelf_out = readelf_proc.communicate()[0]
                if readelf_proc.returncode != 0:
                    sys.stderr.write("Readelf call failed.\n")
                    exit(1)
                match = re.search("Build ID: ([a-fA-F0-9]+)", readelf_out)
                if match is None:
                    # This is usually ok: there are many false
                    # positives (non-GCC generated ELFs, ARM stuff,
                    # firmware) here.  Silently ignore.  It might be
                    # interesting to check them later, this seems to
                    # catch accidentally packaged object files.
                    continue
                build_id = match.group(1)
                # Try to find the associated debuginfo package.
                debuginfo_path = "usr/lib/debug/.build-id/{0}/{1}".format(build_id[:2], build_id[2:])
                used_debuginfo_paths.append(debuginfo_path)
                found = False
                for debuginfo_dir in debuginfo_package_dirs:
                    fullpath = os.path.join(debuginfo_dir, debuginfo_path)
                    if os.path.islink(fullpath):
                        if found:
                            # Never happens.
                            add_problem(f, "debuginfo found in multiple debuginfo packages")
                        # Check that the debuginfo symlink points to
                        # our binary.
                        pointer_relpath = os.readlink(fullpath)
                        pointer_fullpath = os.path.join(os.path.dirname(fullpath), pointer_relpath)
                        pointer_abspath = os.path.normpath(pointer_fullpath).replace(debuginfo_dir, "")
                        file_abspath = f.replace(package_dir, "")
                        if pointer_abspath != file_abspath:
                            # Problem: symlink points to another binary!
                            # Let's check if the binary is at least in
                            # the same RPM package.
                            found_in_rpm = False
                            for rpm_file in rpm_files:
                                file_abspath = rpm_file.replace(package_dir, "")
                                if pointer_abspath == file_abspath:
                                    found_in_rpm = True
                                    break
                            if not found_in_rpm:
                                # Find the binary referenced by the
                                # symlink, and the package where it is
                                # If our package depends on this
                                # package, then it is available when
                                # debugging a crash of the
                                # binary. Otherwise we report a problem.
                                path_in_another_rpm = None
                                another_rpm_is_in_requires = False
                                for x_package_dir in common_package_dirs:
                                    file_abspath = os.path.join(x_package_dir, pointer_abspath[1:])
                                    if os.path.isfile(file_abspath):
                                        path_in_another_rpm = file_abspath
                                        # Check if it is at least in
                                        # the dependencies.
                                        another_rpm_is_in_requires = rpmdir_to_package_name[x_package_dir] \
                                            in requires[rpmdir_to_package_name[package_dir]]
                                        break
                                if path_in_another_rpm is None:
                                    add_problem(f, "debuginfo symlink points" +
                                                " to another binary which" +
                                                " is not found in RPMs: {0}".format(pointer_abspath))
                                elif not another_rpm_is_in_requires:
                                    add_problem(f, "debuginfo symlink points" +
                                                " to another binary in another" +
                                                " RPM package which might not" +
                                                " be installed: {0}".format(path_in_another_rpm))
                        found = True
                if not found:
                    # Check if the binary is stripped or not
                    if -1 != file_out.find("not stripped"):
                        add_problem(f, "debuginfo missing; ELF is not stripped")
                    elif -1 != file_out.find("stripped"):
                        add_problem(f, "debuginfo missing; ELF stripped")
    # Check the debuginfo packages for unused debuginfo.
    for debuginfo_dir in debuginfo_package_dirs:
        for root, dirs, files in os.walk(debuginfo_dir):
            if -1 == root.find("usr/lib/debug/.build-id"):
                continue
            for f in files:
                if -1 != f.find(".debug"):
                    continue
                fullpath = os.path.join(root, f)
                found = False
                for used_debuginfo_path in used_debuginfo_paths:
                    if -1 != fullpath.find(used_debuginfo_path):
                        found = True
                        break
                if not found and not args.ignore_unused_debuginfo:
                    # Get the path of unused binary and report the
                    # problem.
                    pointer_relpath = os.readlink(fullpath)
                    pointer_fullpath = os.path.join(os.path.dirname(fullpath), pointer_relpath)
                    pointer_abspath = os.path.normpath(pointer_fullpath).replace(debuginfo_dir, "")
                    add_problem(fullpath, "unused debuginfo, binary is" +
                                " not packaged: {0}".format(pointer_abspath))
    # Print all the problems found
    if len(component_problems) > 0:
        sys.stdout.write("--\n")
        if args.fedora_component_owners:
            network_object = urllib.urlopen("https://admin.fedoraproject.org/pkgdb/acls/" +
                                            "name/{0}?tg_format=json".format(component))
            json_lines = network_object.readlines()
            component_info = json.loads("\n".join(json_lines))
            collections = component_info['packageListings']
            collection = filter(lambda x: x['collection']['version'] == "devel", collections)
            sys.stdout.write("component: {0} ({1})\n".format(component, collection[0]["owner"]))
        else:
            sys.stdout.write("component: {0}\n".format(component))
        for f, problems in component_problems.items():
            sys.stdout.write("  file: {0}\n".format(f))
            for problem in problems:
                sys.stdout.write("   - {0}\n".format(problem))
    # Remove all the extracted packages for the component.
    for d in common_package_dirs + debuginfo_package_dirs:
        shutil.rmtree(d)
