commit d2276cf0c8cac90934d4588d633cfdebf8244b4f Author: Michal Srb msrb@redhat.com Date: Thu Dec 4 14:03:13 2014 +0100
Move all caching related code to separate module, introduce MetadataCache class
python/javapackages/cache/cache.py | 55 +++++++++++++++ python/javapackages/cache/metadata.py | 69 +++++++++++++++++++ python/javapackages/cache/osgi.py | 99 ++++++++++++++++++++++++++++ python/javapackages/common/osgi.py | 117 --------------------------------- 4 files changed, 223 insertions(+), 117 deletions(-) --- diff --git a/python/javapackages/cache/__init__.py b/python/javapackages/cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/javapackages/cache/cache.py b/python/javapackages/cache/cache.py new file mode 100644 index 0000000..87f2e23 --- /dev/null +++ b/python/javapackages/cache/cache.py @@ -0,0 +1,55 @@ +import os +import logging +import pickle +import javapackages.common.config as config + + +class Cache(object): + def __init__(self, cachedir, scl=None): + self._cachedir = cachedir + self._scl = scl + + def _process_buildroot(self): + cache = {} + # TODO: implement in subclass + return cache + + def _find_paths(self): + buildroot = config.get_buildroot() + paths = [] + for dirpath, _, filenames in os.walk(buildroot): + for filename in filenames: + fpath = os.path.abspath(os.path.join(dirpath, filename)) + if self._check_path(fpath): + paths.append(fpath) + return paths + + def _check_path(self, path): + # TODO: implement in subclass + return False + + def _read_cache(self): + try: + cachepath = os.path.join(self._cachedir, self._config_name) + cachefile = open(cachepath, 'rb') + ppid, cache = pickle.load(cachefile) + cachefile.close() + # check if the cache was most likely created during current build + if ppid != os.getppid(): + logging.warning("Cache in {path} is outdated, skipping" + .format(path=cachepath)) + return None + except IOError: + return None + return cache + + def _write_cache(self, cache): + try: + cachefile = open(os.path.join(self._cachedir, + self._config_name), 'wb') + content = (os.getppid(), cache) + pickle.dump(content, cachefile) + cachefile.close() + except IOError: + return None + return cache diff --git a/python/javapackages/cache/metadata.py b/python/javapackages/cache/metadata.py new file mode 100644 index 0000000..362ac2d --- /dev/null +++ b/python/javapackages/cache/metadata.py @@ -0,0 +1,69 @@ +import javapackages.common.config as config +from javapackages.metadata.metadata import Metadata, MetadataLoadingException +from javapackages.cache.cache import Cache + + +class MetadataCache(Cache): + def __init__(self, cachedir, scl=None): + self._cachedir = cachedir + self._scl = scl + self._config_name = config.metadata_cache_f + self._cache = self._read_cache() + + if self._cache is None: + self._cache = self._process_buildroot() + self._write_cache(self._cache) + + def _process_buildroot(self): + # "path: Metadata" mapping + cache = {} + + metadata_paths = self._find_paths() + for path in metadata_paths: + try: + metadata = Metadata(path) + if metadata: + cache.update({path: metadata}) + except MetadataLoadingException: + continue + + return cache + + def _check_path(self, path): + # TODO + if "/usr/share/maven-metadata/" in path and path.endswith(".xml"): + return True + return False + + def get_artifact_for_path(self, path, can_be_dir=False): + for metadata in self._cache: + artifact = metadata.get_artifact_for_path(path, + can_be_dir=can_be_dir) + if artifact: + return artifact + return None + + def get_metadata_for_path(self, path): + try: + return self._cache[path] + except KeyError: + pass + return None + + def get_provided_artifacts(self): + artifacts = [] + for metadata in self._cache.values(): + artifacts.extend(metadata.artifacts) + return artifacts + + def get_skipped_artifacts(self): + artifacts = [] + for metadata in self._cache.values(): + artifacts.extend(metadata.skipped_artifacts) + return artifacts + + def get_provided_osgi(self): + bundles = [] + for metadata in self._cache.values(): + bundles += metadata.get_osgi_provides() + return bundles diff --git a/python/javapackages/cache/osgi.py b/python/javapackages/cache/osgi.py new file mode 100644 index 0000000..af1fde3 --- /dev/null +++ b/python/javapackages/cache/osgi.py @@ -0,0 +1,99 @@ +#!/usr/bin/python +# Copyright (c) 2014, Red Hat, Inc. +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the Red Hat nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Alexander Kurtakov akurtako@redhat.com +# Michal Srb msrb@redhat.com + +import os + +import javapackages.common.config as config +from javapackages.common.osgi import OSGiBundle +from javapackages.cache.cache import Cache +from javapackages.cache.metadata import MetadataCache + + +class OSGiCache(Cache): + + def __init__(self, cachedir, scl=None): + self._cachedir = cachedir + self._scl = scl + self._config_name = config.osgi_cache_f + self._cache = self._read_cache() + self._metadata_cache = MetadataCache(cachedir, scl) + + if self._cache is None: + self._cache = self._process_buildroot() + self._write_cache(self._cache) + + def get_bundle_for_path(self, path): + try: + return self._cache[path] + except KeyError: + pass + return None + + def get_bundle(self, name): + for bundle in self._cache.values(): + if bundle == name: + return bundle + return None + + def _process_buildroot(self): + # "path: OSGiBundle" mapping + cache = {} + + bundle_paths = self._find_paths() + for path in bundle_paths: + bundle = OSGiBundle.from_manifest(path) + if bundle: + cache.update({path: bundle}) + + return cache + + def _check_path(self, path): + if os.path.islink(path): + return False + if path.endswith(".jar"): + return True + if path.endswith("/MANIFEST.MF"): + # who knows where the manifest can be in buildroot. + # this is an attempt to identify only MANIFEST.MF files + # which are in %{_datadir} or %{_prefix}/lib + if "/usr/share/" in path or "/usr/lib" in path: + return True + return False + + def check_path_in_metadata(self, path): + artifact = self._metadata_cache.get_artifact_for_path(path, + can_be_dir=True) + if artifact and artifact.has_osgi_information(): + return True + return False diff --git a/python/javapackages/common/osgi.py b/python/javapackages/common/osgi.py index 541bc03..73ea210 100644 --- a/python/javapackages/common/osgi.py +++ b/python/javapackages/common/osgi.py @@ -32,11 +32,8 @@ # Authors: Alexander Kurtakov akurtako@redhat.com # Michal Srb msrb@redhat.com
-import os -import pickle import re
-import javapackages.common.config as config from javapackages.common.manifest import Manifest
@@ -168,117 +165,3 @@ class OSGiBundle(object): d="-" if self.namespace else "", bundle=self.bundle, version=version or self.version) - - -def check_path_in_metadata(path, cachedir_path): - buildroot = config.get_buildroot() - - from javapackages.metadata.metadata import Metadata, MetadataInvalidException - artifacts = Metadata.read_provided_artifacts_from_cache(cachedir_path) - if artifacts is None: - artifacts = [] - metadata_paths = [] - for dirpath, dirnames, filenames in os.walk(buildroot): - for filename in filenames: - fpath = os.path.abspath(os.path.join(dirpath, filename)) - # FIXME: add path to metadata directory to config file? - if "/maven-metadata/" in fpath: - metadata_paths.append(fpath) - try: - mdata = Metadata(metadata_paths) - artifacts = mdata.write_provided_artifacts_to_cache(cachedir_path) - except MetadataInvalidException: - pass - - for a in artifacts: - path = os.path.abspath(path) - if path.startswith(buildroot): - path = path[len(buildroot):] - path = os.path.join('/', path) - if a.path and a.has_osgi_information(): - if (os.path.abspath(a.path) == path or - (path.startswith(os.path.abspath(a.path)) and - os.path.realpath(buildroot + path))): - return True - return False - - -class OSGiCache(object): - - def __init__(self, cachedir, scl=None): - self._cachedir = cachedir - self._cache = self._read_osgi_cache() - self._scl = scl - - if self._cache is None: - cache = self._process_osgi_in_buildroot() - self._write_osgi_cache(cache) - self._cache = cache - - def get_bundle_for_path(self, path): - try: - return self._cache[path] - except KeyError: - pass - return None - - def get_bundle(self, name): - for bundle in self._cache.values(): - if bundle == name: - return bundle - return None - - def _process_osgi_in_buildroot(self): - # "path: OSGiBundle" mapping - cache = {} - - bundle_paths = self._find_possible_bundles() - for path in bundle_paths: - bundle = OSGiBundle.from_manifest(path) - if bundle: - cache.update({path: bundle}) - - return cache - - def _find_possible_bundles(self): - buildroot = config.get_buildroot() - paths = [] - for dirpath, _, filenames in os.walk(buildroot): - for filename in filenames: - fpath = os.path.abspath(os.path.join(dirpath, filename)) - if self._check_path(fpath): - paths.append(fpath) - return paths - - def _check_path(self, path): - if os.path.islink(path): - return False - if path.endswith(".jar"): - return True - if path.endswith("/MANIFEST.MF"): - # who knows where the manifest can be in buildroot. - # this is an attempt to identify only MANIFEST.MF files - # which are in %{_datadir} or %{_prefix}/lib - if "/usr/share/" in path or "/usr/lib" in path: - return True - return False - - def _read_osgi_cache(self): - try: - cachefile = open(os.path.join(self._cachedir, - config.osgi_cache_f), 'rb') - cache = pickle.load(cachefile) - cachefile.close() - except IOError: - return None - return cache - - def _write_osgi_cache(self, cache): - try: - cachefile = open(os.path.join(self._cachedir, - config.osgi_cache_f), 'wb') - pickle.dump(cache, cachefile) - cachefile.close() - except IOError: - return None - return cache