#!/usr/bin/python # # find-missing-debuginfo # Karel Klic # # Checks the correctness and completness of debuginfo packages. # # # This script checks for the following issues in the relationship of # binary and its debuginfo counterpart: # # 1) A binary (an executable or a shared library) does not have an # associated debuginfo file in -debuginfo packages, and the binary # does not contain debugging symbols (it is stripped). It might be # caused by the component's build script stripping the binary before # rpmbuild can generate the -debuginfo package (rpmbuild calls # /usr/lib/rpm/find-debuginfo.sh to do that). This breaks debugging # of a crash in this binary. # # 2) A binary does not have an associated debuginfo file in -debuginfo # packages, and the binary contains debugging symbols (it is not # stripped). It might be caused by the component's build script # installing the binary with invalid permissions - common issue is # that the executable bits not set for a shared library. This breaks # some debugging scenarios, e.g. the retrace server: when it analyzes # a coredump, it can get the build ids of all binaries from the # coredump, and use the build-ids to find appropriate # packages. However, if the binary is not stripped and thus no # debuginfo is present, the package repository cannot be searched for # the exact package that participated in the crash, because the # build-id cannot be found in yum metadata. # # 3) A binary has an associated debuginfo, but the symlink in the # debuginfo points to another binary, which does not exist. It might # be caused by packaging the binary under different name from what has # been installed into the build root by component's build script. # This prevents GDB from finding the binary and breaks some debugging # scenarios. # # 4) A binary has an associated debuginfo, but the symlink in the # debuginfo points to another binary, and the package containing that # binary is not present in the dependencies of the package of the # checked binary. It might be caused by packaging the same binary # multiple times in multiple packages, or by building the same binary # multiple times under different names. It is ok when all the # packages containing the binary (=same build-id) depend on the # package with the binary the debuginfo symlink is pointing to. This # issue can be fixed for all packages at once by fixing packages # rpm-build and gdb - see rhbz#641377. However, better way how to fix # this issue is to avoid packaging the same binary multiple times. Use # symlinks. # # 5) There are debug symbols present for unpackaged binaries in the # -debuginfo package. It might be caused by leaving an intentionally # unpackaged binary in the build root, where # /usr/lib/rpm/find-debuginfo.sh finds it, and using %exclude to skip # it in %files. The unused debuginfo files are not a serious problem, # they just waste space. # # # This script checks for the following issues in the relationship of # debuginfo and its source code files: # # 1) A source file path specified in a .debug_info compilation unit is # relative, but comp_dir entry is missing, thus the full path to the # source file is not known. # # 2) A source file name specified in a .debug_line table uses # directory pointer pointing to relative directory, this the full path # to the source file is not known. # # 3) A source file name specified in a .debug_line table uses invalid # (out of range) directory pointer to the corresponding directory # table. # # 4) A source file name specified in a .debug_line table uses # directory pointer to comp_dir from .debug_info, but comp_dir is not # present there. # # 5) A source file specified in .debug_info or .debug_line is missing # in the debuginfo package. # # # Requirements: # # Packages python, yum, elfutils, cpio, file; 10 GB free disk space; # it might take several days to get the results for Fedora, depending # on the access to the nearest package repository # # Usage: # terminal one (results): # $ ./find-missing-debuginfo --repos="rawhide" \ # --log=rawhide.log # terminal two (progress logging): # $ tail -f rawhide.log # # Use the --ignore-unused-debuginfo option on RHEL to suppress noise # caused by operating system variants. Use --fedora-component-owners # on Fedora to include the component owners to the report. # import subprocess import yum import sys import argparse import os import os.path import re import shutil import urllib import json import signal def sigint_handler(signum, frame): sys.exit(1) signal.signal(signal.SIGINT, sigint_handler) parser = argparse.ArgumentParser(description='check correctness and completness of debuginfo packages') parser.add_argument('--repos', default='fedora', metavar='WILDCARD', help='yum repositories to be checked') parser.add_argument('--log', metavar='FILENAME', help='store debug/progress output to a file') parser.add_argument('--ignore-unused-debuginfo', action='store_true', help='do not report superfluous debug files') parser.add_argument('--fedora-component-owners', action='store_true', help='include component owners in the stdout report') parser.add_argument('--keep-dirs', action='store_true', help='keep package directories in the work directory') parser.add_argument('--offset', type=int, metavar='N', help='start from Nth component (see logfile for component numbers)') parser.add_argument('--component', type=str, metavar='NAME', help='check packages from a single component found in a repository') args = parser.parse_args() if args.log: log = open(args.log, "w", 0) else: log = open("/dev/null", "w") # Initialize yum, enable only repositories specified via command line --repos option. stdout = sys.stdout sys.stdout = log yumbase = yum.YumBase() yumbase.doConfigSetup() if not yumbase.setCacheDir(): exit(2) log.write("Closing all enabled repositories...\n") for repo in yumbase.repos.listEnabled(): log.write(" - {0}\n".format(repo.name)) repo.close() yumbase.repos.disableRepo(repo.id) log.write("Enabling repositories matching \'{0}\'...\n".format(args.repos)) for repo in yumbase.repos.findRepos(args.repos): log.write(" - {0}\n".format(repo.name)) repo.enable() repo.skip_if_unavailable = True yumbase.repos.doSetup() yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) sys.stdout = stdout # Use the enabled repos to get all their packages. log.write("Getting the list of all packages...") package_list = yumbase.pkgSack.returnPackages() log.write("{0} packages found\n".format(len(package_list))) # Partition the packages by component. log.write("Partitioning packages by component...\n") components = {} for package in package_list: if args.component and not package.base_package_name == args.component: continue if not package.base_package_name in components: components[package.base_package_name] = [package] else: components[package.base_package_name].append(package) def unpack_package(package): # Download the package. log.write(" - downloading {0}\n".format(package)) repo = yumbase.repos.getRepo(package.repoid) remote = package.returnSimple('relativepath') local = os.path.basename(remote) package.localpath = local path = repo.getPackage(package) if not os.path.exists(local) or not os.path.samefile(path, local): shutil.copy2(path, local) # Convert it to cpio. log.write(" - converting {0} to cpio\n".format(local)) cpio = open(local + ".cpio", "wb") rpm2cpio_proc = subprocess.Popen(['rpm2cpio', local], stdout=cpio) rpm2cpio_proc.communicate() if rpm2cpio_proc.returncode != 0: sys.stderr.write("Rpm2cpio failed.\n") exit(1) cpio.close() # Unpack the cpio. rpmdir = re.sub('\\.rpm$', '', local) if os.path.exists(rpmdir): shutil.rmtree(rpmdir) os.makedirs(rpmdir) cpio = open(local + ".cpio", "rb") cpio_args = ["cpio", "--extract", "-d", "--quiet"] cpio_proc = subprocess.Popen(cpio_args, stdin=cpio, cwd=rpmdir, bufsize=-1) cpio_proc.communicate() if cpio_proc.returncode != 0: sys.stderr.write("Cpio failed: {0}\n".format(cpio_args)) exit(1) cpio.close() os.unlink(local) os.unlink(local + ".cpio") # Set sane access rights. The file command fails to work reliably # on suid binaries without user read access, and also some files # and directories are not removable by default. for root, dirs, files in os.walk(rpmdir): for f in files: ff = os.path.join(root, f) if not os.path.islink(ff): os.chmod(ff, 0644) for d in dirs: dd = os.path.join(root, d) if not os.path.islink(dd): os.chmod(dd, 0755) return rpmdir # Check all the components. index = 0 log.write("Checking components...\n") components_keys = sorted(components.keys()) if args.offset: args.offset -= 1 components_keys = components_keys[args.offset:] index = args.offset for component in components_keys: index += 1 log.write("[{0}/{1}] Checking {2}\n".format(index, len(components.keys()), component)) if component == "udev": log.write(" - skipping because cpio cannot extract the udev package\n") continue elif component.startswith("compat-"): # compat- packages are not actively maintained log.write(" - skipping compat package (not actively maintained)") continue # Download and unpack all packages. packages = components[component] common_package_dirs = [] debuginfo_package_dirs = [] # Mapping of a local directory with extracted rpm to package name. rpmdir_to_package_name = {} # Mapping from package name to a list of package names the package # depends on. The list contains recursive dependencies. log.write(" - building provides/requires list\n") requires = {} provides = {} for package in packages: if package.arch == "noarch": continue rpmdir = unpack_package(package) rpmdir_to_package_name[rpmdir] = package.name # Build the provides and requires lists for the package. provides[package.name] = [] for prov in package.returnPrco('provides'): provides[package.name].append(prov[0]) requires[package.name] = set() for req in package.returnPrco('requires'): requires[package.name].add(req[0]) # Put the package to debuginfo or non-debuginfo (common) bucket. if -1 == package.name.find("-debuginfo"): common_package_dirs.append(str(rpmdir)) else: debuginfo_package_dirs.append(str(rpmdir)) # Propagate requires and provides recursively. First step is to # change all known file dependencies to package dependencies in # requires. Use provides for this. for provides_package, package_provides in provides.items(): for provide in package_provides: for requires_package, package_requires in requires.items(): for require in package_requires.copy(): if require == provide: package_requires.remove(require) package_requires.add(provides_package) # Build the transitive closure of requires on the requires of one # component's packages. for requires_package, package_requires in requires.items(): while True: original_package_requires = package_requires.copy() for require in original_package_requires: if require in requires: package_requires |= requires[require] if len(package_requires - original_package_requires) == 0: break # Skip ocaml and ghc packages, which are built by gcc but do not # include DWARF. Ocaml packages can be recognized by depending on # ocaml runtime. There seems to be no 100% way of recognize that # a binary was build from Haskell or Ocaml sources, see # `eu-readelf --all /usr/bin/xmonad`. is_ocaml = False is_ghc = False for package in packages: if not package.name in requires: continue if "ocaml(runtime)" in requires[package.name]: is_ocaml = True for r in requires[package.name]: if -1 != r.find("ghc-"): is_ghc = True if is_ocaml: log.write(" - skipping ocaml component\n") continue if is_ghc: log.write(" - skipping ghc component\n") continue # Find all ELF binaries in the common packages and do the checking. component_problems = {} used_debuginfo_paths = [] def add_problem(f, problem): if not f in component_problems: component_problems[f] = [problem] elif not problem in component_problems[f]: component_problems[f].append(problem) for package_dir in common_package_dirs: rpm_files = [] for root, dirs, files in os.walk(package_dir): for f in files: rpm_files.append(os.path.join(root, f)) for f in rpm_files: # The file utility recognizes an ELF binary and also tells # whether it is stripped or not. file_proc = subprocess.Popen(["file", f], stdout=subprocess.PIPE) file_out = file_proc.communicate()[0] if file_proc.returncode != 0: sys.stderr.write("File call failed.\n") exit(1) if -1 != file_out.find(" ELF "): log.write(" - checking {0}\n".format(f)) # Get its build id readelf_proc = subprocess.Popen(["eu-readelf", "--notes", f], stdout=subprocess.PIPE) readelf_out = readelf_proc.communicate()[0] if readelf_proc.returncode != 0: sys.stderr.write("Readelf call failed.\n") exit(1) match = re.search("Build ID: ([a-fA-F0-9]+)", readelf_out) if match is None: # This is usually ok: there are many false positives (non-GCC generated ELFs, ARM stuff, # firmware) here. Silently ignore. It might be interesting to check them later, this seems to # catch accidentally packaged object files. continue build_id = match.group(1) # Try to find the associated debuginfo package. debuginfo_path = "usr/lib/debug/.build-id/{0}/{1}".format(build_id[:2], build_id[2:]) used_debuginfo_paths.append(debuginfo_path) found = False for debuginfo_dir in debuginfo_package_dirs: fullpath = os.path.join(debuginfo_dir, debuginfo_path) if os.path.islink(fullpath): if found: # Never happens. add_problem(f, "debuginfo found in multiple debuginfo packages") # Check that the debuginfo symlink points to our binary. pointer_relpath = os.readlink(fullpath) pointer_fullpath = os.path.join(os.path.dirname(fullpath), pointer_relpath) pointer_abspath = os.path.normpath(pointer_fullpath).replace(debuginfo_dir, "") file_abspath = f.replace(package_dir, "") if pointer_abspath != file_abspath: # Problem: symlink points to another binary! Let's check if the binary is at least in # the same RPM package. found_in_rpm = False for rpm_file in rpm_files: file_abspath = rpm_file.replace(package_dir, "") if pointer_abspath == file_abspath: found_in_rpm = True break if not found_in_rpm: # Find the binary referenced by the symlink, and the package where it is # If our package depends on this package, then it is available when # debugging a crash of the binary. Otherwise we report a problem. path_in_another_rpm = None another_rpm_is_in_requires = False for x_package_dir in common_package_dirs: file_abspath = os.path.join(x_package_dir, pointer_abspath[1:]) if os.path.isfile(file_abspath): path_in_another_rpm = file_abspath # Check if it is at least in # the dependencies. another_rpm_is_in_requires = rpmdir_to_package_name[x_package_dir] in requires[rpmdir_to_package_name[package_dir]] break if path_in_another_rpm is None: add_problem(f, "debuginfo symlink points to another binary which is not found in RPMs: {0}".format(pointer_abspath)) elif not another_rpm_is_in_requires: add_problem(f, "debuginfo symlink points to another binary in another RPM package which might not be installed: {0}".format(path_in_another_rpm)) found = True if not found: # Check if the binary is stripped or not. # Valgrind requires presence of debug info and symbol tables in the files for its own shared libraries. if -1 != file_out.find("not stripped") and component != "valgrind": add_problem(f, "debuginfo missing; ELF is not stripped") elif -1 != file_out.find("stripped"): if re.search("/usr/sbin/libgcc_post_upgrade", f): # Intentional, there is nothing to debug on it and the binary is used just in %post. pass else: add_problem(f, "debuginfo missing; ELF stripped") # Check the debuginfo packages for unused debuginfo and for source files. for debuginfo_dir in debuginfo_package_dirs: for root, dirs, files in os.walk(debuginfo_dir): if -1 == root.find("usr/lib/debug/.build-id"): continue for f in files: if -1 != f.find(".debug"): continue fullpath = os.path.join(root, f) found = False for used_debuginfo_path in used_debuginfo_paths: if -1 != fullpath.find(used_debuginfo_path): found = True break if not found and not args.ignore_unused_debuginfo: # Get the path of unused binary and report the problem. pointer_relpath = os.readlink(fullpath) pointer_fullpath = os.path.join(os.path.dirname(fullpath), pointer_relpath) pointer_abspath = os.path.normpath(pointer_fullpath).replace(debuginfo_dir, "") add_problem(fullpath, "unused debuginfo, binary is not packaged: {0}".format(pointer_abspath)) elif found: # Used debuginfo, check if it contains all source files. fullpath_debug = "{0}.debug".format(fullpath) # Get real debug file fullpath_debug_real = os.path.normpath(os.path.join(os.path.dirname(fullpath_debug), os.readlink(fullpath_debug))) log.write(" - checking {0}\n".format(fullpath_debug_real)) log.write(" - reading .debug_info\n") readelf_args = ["eu-readelf", "-winfo", fullpath_debug_real] readelf_proc = subprocess.Popen(readelf_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # The .debug_info output might take several gigabytes, so let's filter unnecessary parts. # amanith-debuginfo-0.3-14.fc13.i686/usr/lib/debug/usr/lib/libamanith.so.1.0.0.debug # has 6.7 MB and its -winfo output has 1.5 milion lines. There are many .debug files having # more than 100 MB. winfo = "" deleting = False # Precompile regexps as it improves the performance. compile_unit_re = re.compile("\\s*\\[\\s*[0-9a-f]+\\]\\s*compile_unit") other_section_re = re.compile("\\s*\\[\\s*[0-9a-f]+\\]\\s*[a-z_]+") while True: line = readelf_proc.stdout.readline() if not line: break # The line.find() call is performace optimization only. if -1 != line.find("compile_unit") and compile_unit_re.match(line): deleting = False # The line.find() call is performace optimization only. elif -1 != line.find("[") and other_section_re.match(line): deleting = True if not deleting: winfo += line + "\n" readelf_proc.wait() if readelf_proc.returncode != 0: err = "" for line in readelf_proc.stderr.readlines(): err += line.strip() + "\n" add_problem(fullpath_debug_real, "{0} (return code {1})".format(err.strip(), readelf_proc.returncode)) continue # Get the .debug_lines contents log.write(" - reading .debug_line\n") readelf_args = ["eu-readelf", "-wline", fullpath_debug_real] readelf_proc = subprocess.Popen(readelf_args, stdout=subprocess.PIPE) wlines = readelf_proc.communicate()[0] if readelf_proc.returncode != 0: sys.stderr.write("Readelf call failed: {0}.\n".format(readelf_args)) exit(1) # Parse it and build a list of source files log.write(" - examining\n") source_files = set() winfo_pos = 0 while True: winfo_pos = winfo.find("Compilation unit at offset", winfo_pos + 1) if -1 == winfo_pos: break compilation_unit_offset = re.search("offset (\\d+):", winfo[winfo_pos:]).group(1) next_winfo_pos = winfo.find("Compilation unit at offset", winfo_pos + 1) compilation_unit = winfo[winfo_pos:next_winfo_pos] comp_dir_match = re.search("\\s+comp_dir\\s+\\(strp\\)\\s+\"(.*?)\"", compilation_unit) name_match = re.search("\\s+name\\s+\\(strp\\)\\s+\"(.*?)\"", compilation_unit) if comp_dir_match: comp_dir = comp_dir_match.group(1) # Add name to the list. if name_match: name = name_match.group(1) if os.path.isabs(name): source_files.add(name) else: if comp_dir_match: name = os.path.normpath(os.path.join(comp_dir, name)) source_files.add(name) else: print "No comp dir match:", compilation_unit add_problem(fullpath_debug_real, "relative source name {0} without comp_dir in .debug_info compilation unit at offset {1}".format(name, compilation_unit_offset)) stmt_list_match = re.search("\\s+stmt_list\\s+\\(.*?\\)\\s+(\\d+)", compilation_unit) if not stmt_list_match: continue table_offset = stmt_list_match.group(1) # Build directory list from wlines. directory_table = [] if comp_dir_match: directory_table.append(comp_dir) else: # This is a problem, the comp_dir is missing in .debug_section. # Issue error only if this directory is used. directory_table.append("") table_pos = wlines.find("Table at offset {0}:".format(table_offset)) next_table_pos = wlines.find("Table at offset", table_pos + 1) table = wlines[table_pos:next_table_pos] in_directory_table = False for line in table.splitlines(False): if -1 != line.find("Directory table:"): in_directory_table = True continue if in_directory_table and "" == line.strip(): break if in_directory_table: entry = line.strip() if os.path.isabs(entry): directory_table.append(entry) else: if comp_dir_match: entry = os.path.normpath(os.path.join(comp_dir, entry)) directory_table.append(entry) else: # This is a problem, the directory is relative, but comp_dir is missing in .debug_section. # # We do not issue error here, because some DWARF files contain unused relative directory entries named # "XXXXXX"; see `eu-readelf -wline CGAL-debuginfo-3.6.1-4.fc15.i686/usr/lib/debug/usr/lib/libCGAL_Qt4.so.5.0.1.debug` # for an example. Issue this error later, when such relative directory is used. directory_table.append(entry) if not in_directory_table: add_problem(fullpath_debug_real, "directory table not found in .debug_lines table at offset {0}".format(table_offset)) # Build file list from wlines in_filelist_table = 0 for line in table.splitlines(False): if -1 != line.find("File name table:"): in_filelist_table = 1 continue if in_filelist_table > 0 and "" == line.strip(): break if in_filelist_table == 1: # Skip the header in_filelist_table = 2 continue if in_filelist_table == 2: file_match = re.search("\\s*\\d+\\s+(\\d+)\\s+\\d+\\s+\\d+\\s+(.*)", line) if not file_match: add_problem(fullpath_debug_real, "invalid line in the file name table in .debug_lines table at offset {0}".format(table_offset)) continue dirno = int(file_match.group(1)) filename = file_match.group(2) if "" == filename: continue if not os.path.isabs(filename): if dirno >= len(directory_table): add_problem(fullpath_debug_real, "missing entry in directory table in .debug_lines table at offset {0} for file {1}".format(table_offset, filename)) continue if dirno == 0 and len(directory_table[dirno]) == 0: # The file references comp_dir which is not present in .debug_info. add_problem(fullpath_debug_real, "comp_dir missing in .debug_section compilation unit at offset {0}, but file {1} references it in .debug_lines table at offset {2}".format(compilation_unit_offset, filename, table_offset)) elif not os.path.isabs(directory_table[dirno]): # This is the issue with relative directory used without comp_dir present. add_problem(fullpath_debug_real, "relative source directory \"{0}\" in .debug_lines table at offset {1} without comp_dir in .debug_section compilation unit at offset {2} used for file {3}".format(entry, table_offset, compilation_unit_offset, filename)) filename = os.path.join(directory_table[dirno], filename) source_files.add(filename) # Check the existence of source files for source_file in source_files: if not source_file.startswith("/usr/src/debug"): continue fullpath = os.path.join(debuginfo_dir, source_file[1:]) if not os.path.isfile(fullpath): add_problem(fullpath_debug_real, "missing source file {0} in the debuginfo package".format(fullpath, fullpath_debug_real)) # Print all the problems found if len(component_problems) > 0: sys.stdout.write("--\n") if args.fedora_component_owners: network_object = urllib.urlopen("https://admin.fedoraproject.org/pkgdb/acls/name/{0}?tg_format=json".format(component)) json_lines = network_object.readlines() component_info = json.loads("\n".join(json_lines)) collections = component_info['packageListings'] collection = filter(lambda x: x['collection']['version'] == "devel", collections) sys.stdout.write("component: {0} ({1})\n".format(component, collection[0]["owner"])) else: sys.stdout.write("component: {0}\n".format(component)) for f, problems in component_problems.items(): sys.stdout.write(" file: {0}\n".format(f)) for problem in problems: sys.stdout.write(" - {0}\n".format(problem)) # Remove all the extracted packages for the component. if not args.keep_dirs: for d in common_package_dirs + debuginfo_package_dirs: shutil.rmtree(d)