[python3/python3.4] Update to Python 3.4 final
Matej Stuchlik
mstuchli at fedoraproject.org
Tue Apr 15 09:01:14 UTC 2014
commit 11fb599edb14569b903abafeed7fd2349d96dfbf
Author: Matej Stuchlik <mstuchli at redhat.com>
Date: Tue Apr 15 09:52:32 2014 +0200
Update to Python 3.4 final
Also merge patches from master and add the rewheel module
00189-add-rewheel-module.patch | 224 ++++++++++++++++++++
00190-fix-tests-with-sqlite-3.8.4.patch | 21 ++
...rect-num-of-pycfile-bytes-in-modulefinder.patch | 65 ++++++
python3.spec | 53 +++++-
4 files changed, 358 insertions(+), 5 deletions(-)
---
diff --git a/00189-add-rewheel-module.patch b/00189-add-rewheel-module.patch
new file mode 100644
index 0000000..ad80e9f
--- /dev/null
+++ b/00189-add-rewheel-module.patch
@@ -0,0 +1,224 @@
+unchanged:
+--- Python-3.4.0rc3/Lib/ensurepip/__init__.py 2014-03-10 07:56:33.000000000 +0100
++++ Python-3.4.0rc3-rewheel/Lib/ensurepip/__init__.py 2014-03-12 09:57:12.917120853 +0100
+@@ -1,8 +1,10 @@
+ import os
+ import os.path
+ import pkgutil
++import shutil
+ import sys
+ import tempfile
++from ensurepip import rewheel
+
+
+ __all__ = ["version", "bootstrap"]
+@@ -38,6 +40,8 @@ def _run_pip(args, additional_paths=None
+
+ # Install the bundled software
+ import pip
++ if args[0] in ["install", "list", "wheel"]:
++ args.append('--pre')
+ pip.main(args)
+
+
+@@ -87,20 +90,40 @@ def bootstrap(*, root=None, upgrade=Fals
+ # omit pip and easy_install
+ os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+- with tempfile.TemporaryDirectory() as tmpdir:
+- # Put our bundled wheels into a temporary directory and construct the
+- # additional paths that need added to sys.path
+- additional_paths = []
++ whls = []
++ rewheel_dir = None
++ # try to see if we have system-wide versions of _PROJECTS
++ dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS])
++ # TODO: check if system-wide versions are the newest ones
++ # if --upgrade is used?
++ if all(dep_records):
++ # if we have all _PROJECTS installed system-wide, we'll recreate
++ # wheels from them and install those
++ rewheel_dir = tempfile.TemporaryDirectory()
++ for dr in dep_records:
++ new_whl = rewheel.rewheel_from_record(dr, rewheel_dir.name)
++ whls.append(os.path.join(rewheel_dir.name, new_whl))
++ else:
++ # if we don't have all the _PROJECTS installed system-wide,
++ # let's just fall back to bundled wheels
+ for project, version in _PROJECTS:
+- wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+- whl = pkgutil.get_data(
++ whl = os.path.join(
++ os.path.dirname(__file__),
+ "ensurepip",
+- "_bundled/{}".format(wheel_name),
++ "bundled",
++ "{}-{}-py2.py3-none-any.whl".format(project, version)
+ )
+- with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+- fp.write(whl)
++ whls.append(whl)
+
+- additional_paths.append(os.path.join(tmpdir, wheel_name))
++ with tempfile.TemporaryDirectory() as tmpdir:
++ # Put our bundled wheels into a temporary directory and construct the
++ # additional paths that need added to sys.path
++ additional_paths = []
++ for whl in whls:
++ shutil.copy(whl, tmpdir)
++ additional_paths.append(os.path.join(tmpdir, os.path.basename(whl)))
++ if rewheel_dir:
++ rewheel_dir.cleanup()
+
+ # Construct the arguments to be passed to the pip command
+ args = ["install", "--no-index", "--find-links", tmpdir]
+unchanged:
+--- Python-3.4.0rc3/Lib/ensurepip/rewheel/__init__.py 1970-01-01 01:00:00.000000000 +0100
++++ Python-3.4.0rc3-rewheel/Lib/ensurepip/rewheel/__init__.py 2014-03-12 09:55:30.413152104 +0100
+@@ -0,0 +1,133 @@
++import argparse
++import csv
++import email.parser
++import os
++import io
++import re
++import site
++import subprocess
++import sys
++import zipfile
++
++def run():
++ parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.')
++ parser.add_argument('record_path',
++ help='Path to RECORD file')
++ parser.add_argument('-o', '--output-dir',
++ help='Dir where to place the wheel, defaults to current working dir.',
++ dest='outdir',
++ default=os.path.curdir)
++
++ ns = parser.parse_args()
++ retcode = 0
++ try:
++ print(rewheel_from_record(**vars(ns)))
++ except BaseException as e:
++ print('Failed: {}'.format(e))
++ retcode = 1
++ sys.exit(1)
++
++def find_system_records(projects):
++ """Return list of paths to RECORD files for system-installed projects.
++
++ If a project is not installed, the resulting list contains None instead
++ of a path to its RECORD
++ """
++ records = []
++ # get system site-packages dirs
++ sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix])
++ sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)]
++ # try to find all projects in all system site-packages
++ for project in projects:
++ path = None
++ for sp in sys_sitepack:
++ dist_info_re = os.path.join(sp, project) + '-[^\{0}]+\.dist-info'.format(os.sep)
++ candidates = [os.path.join(sp, p) for p in os.listdir(sp)]
++ # filter out candidate dirs based on the above regexp
++ filtered = [c for c in candidates if re.match(dist_info_re, c)]
++ # if we have 0 or 2 or more dirs, something is wrong...
++ if len(filtered) == 1:
++ path = filtered[0]
++ records.append(os.path.join(path, 'RECORD'))
++ return records
++
++def rewheel_from_record(record_path, outdir):
++ """Recreates a whee of package with given record_path and returns path
++ to the newly created wheel."""
++ site_dir = os.path.dirname(os.path.dirname(record_path))
++ record_relpath = record_path[len(site_dir):].strip(os.path.sep)
++ to_write, to_omit = get_records_to_pack(site_dir, record_relpath)
++ new_wheel_name = get_wheel_name(record_path)
++ new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl')
++
++ new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED)
++ # we need to write a new record with just the files that we will write,
++ # e.g. not binaries and *.pyc/*.pyo files
++ new_record = io.StringIO()
++ writer = csv.writer(new_record)
++
++ # handle files that we can write straight away
++ for f, sha_hash, size in to_write:
++ new_wheel.write(os.path.join(site_dir, f), arcname=f)
++ writer.writerow([f, sha_hash,size])
++
++ # rewrite the old wheel file with a new computed one
++ writer.writerow([record_relpath, '', ''])
++ new_wheel.writestr(record_relpath, new_record.getvalue())
++
++ new_wheel.close()
++
++ return new_wheel.filename
++
++def get_wheel_name(record_path):
++ """Return proper name of the wheel, without .whl."""
++ wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL')
++ wheel_info = email.parser.Parser().parsestr(open(wheel_info_path).read())
++ metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA')
++ metadata = email.parser.Parser().parsestr(open(metadata_path).read())
++
++ # construct name parts according to wheel spec
++ distribution = metadata.get('Name')
++ version = metadata.get('Version')
++ build_tag = '' # nothing for now
++ lang_tag = []
++ for t in wheel_info.get_all('Tag'):
++ lang_tag.append(t.split('-')[0])
++ lang_tag = '.'.join(lang_tag)
++ abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3]
++ # leave out build tag, if it is empty
++ to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag])
++ return '-'.join(list(to_join))
++
++def get_records_to_pack(site_dir, record_relpath):
++ """Accepts path of sitedir and path of RECORD file relative to it.
++ Returns two lists:
++ - list of files that can be written to new RECORD straight away
++ - list of files that shouldn't be written or need some processing
++ (pyc and pyo files, scripts)
++ """
++ record_contents = open(os.path.join(site_dir, record_relpath)).read()
++ # temporary fix for https://github.com/pypa/pip/issues/1376
++ # we need to ignore files under ".data" directory
++ data_dir = os.path.dirname(record_relpath).strip(os.path.sep)
++ data_dir = data_dir[:-len('dist-info')] + 'data'
++
++ to_write = []
++ to_omit = []
++ for l in record_contents.splitlines():
++ spl = l.split(',')
++ if len(spl) == 3:
++ # new record will omit (or write differently):
++ # - abs paths, paths with ".." (entry points),
++ # - pyc+pyo files
++ # - the old RECORD file
++ # TODO: is there any better way to recognize an entry point?
++ if os.path.isabs(spl[0]) or spl[0].startswith('..') or \
++ spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \
++ spl[0] == record_relpath or spl[0].startswith(data_dir):
++ to_omit.append(spl)
++ else:
++ to_write.append(spl)
++ else:
++ pass # bad RECORD or empty line
++ return to_write, to_omit
+only in patch2:
+unchanged:
+--- Python-3.4.0/Makefile.pre.in 2014-04-01 12:02:48.188136172 +0200
++++ Python-3.4.0-new/Makefile.pre.in 2014-04-01 12:03:23.770394025 +0200
+@@ -1140,7 +1140,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter
+ test/test_asyncio \
+ collections concurrent concurrent/futures encodings \
+ email email/mime test/test_email test/test_email/data \
+- ensurepip ensurepip/_bundled \
++ ensurepip ensurepip/_bundled ensurepip/rewheel \
+ html json test/test_json http dbm xmlrpc \
+ sqlite3 sqlite3/test \
+ logging csv wsgiref urllib \
diff --git a/00190-fix-tests-with-sqlite-3.8.4.patch b/00190-fix-tests-with-sqlite-3.8.4.patch
new file mode 100644
index 0000000..8a94f5c
--- /dev/null
+++ b/00190-fix-tests-with-sqlite-3.8.4.patch
@@ -0,0 +1,21 @@
+
+# HG changeset patch
+# User Benjamin Peterson <benjamin at python.org>
+# Date 1394679139 18000
+# Node ID 4d626a9df062104b61c44c8a5be8b0fd52fae953
+# Parent 6f93ab911d5dafcde364013e21723259fe2c85a8# Parent dbc9e3ed5e9f1bd11240eaa971f6c75d6a7013b5
+merge 3.3 (#20901)
+
+diff --git a/Lib/sqlite3/test/hooks.py b/Lib/sqlite3/test/hooks.py
+--- a/Lib/sqlite3/test/hooks.py
++++ b/Lib/sqlite3/test/hooks.py
+@@ -162,7 +162,7 @@ class ProgressTests(unittest.TestCase):
+ create table bar (a, b)
+ """)
+ second_count = len(progress_calls)
+- self.assertGreater(first_count, second_count)
++ self.assertGreaterEqual(first_count, second_count)
+
+ def CheckCancelOperation(self):
+ """
+
diff --git a/00193-skip-correct-num-of-pycfile-bytes-in-modulefinder.patch b/00193-skip-correct-num-of-pycfile-bytes-in-modulefinder.patch
new file mode 100644
index 0000000..4a82309
--- /dev/null
+++ b/00193-skip-correct-num-of-pycfile-bytes-in-modulefinder.patch
@@ -0,0 +1,65 @@
+
+# HG changeset patch
+# User Brett Cannon <brett at python.org>
+# Date 1393602285 18000
+# Node ID 432cb56db05d73f55d211501bf0dfc767768923b
+# Parent ade5e4922a54cb84c99ec924ab7c700a014893da
+Issue #20778: Fix modulefinder to work with bytecode-only modules.
+
+Bug filed and initial attempt at a patch by Bohuslav Kabrda.
+
+diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
+--- a/Lib/modulefinder.py
++++ b/Lib/modulefinder.py
+@@ -290,7 +290,7 @@ class ModuleFinder:
+ if fp.read(4) != imp.get_magic():
+ self.msgout(2, "raise ImportError: Bad magic number", pathname)
+ raise ImportError("Bad magic number in %s" % pathname)
+- fp.read(4)
++ fp.read(8) # Skip mtime and size.
+ co = marshal.load(fp)
+ else:
+ co = None
+diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py
+--- a/Lib/test/test_modulefinder.py
++++ b/Lib/test/test_modulefinder.py
+@@ -1,5 +1,7 @@
+ import os
+ import errno
++import importlib.machinery
++import py_compile
+ import shutil
+ import unittest
+ import tempfile
+@@ -208,6 +210,14 @@ a/module.py
+ from . import *
+ """]
+
++bytecode_test = [
++ "a",
++ ["a"],
++ [],
++ [],
++ ""
++]
++
+
+ def open_file(path):
+ dirname = os.path.dirname(path)
+@@ -288,6 +298,16 @@ class ModuleFinderTest(unittest.TestCase
+ def test_relative_imports_4(self):
+ self._do_test(relative_import_test_4)
+
++ def test_bytecode(self):
++ base_path = os.path.join(TEST_DIR, 'a')
++ source_path = base_path + importlib.machinery.SOURCE_SUFFIXES[0]
++ bytecode_path = base_path + importlib.machinery.BYTECODE_SUFFIXES[0]
++ with open_file(source_path) as file:
++ file.write('testing_modulefinder = True\n')
++ py_compile.compile(source_path, cfile=bytecode_path)
++ os.remove(source_path)
++ self._do_test(bytecode_test)
++
+
+ def test_main():
+ support.run_unittest(ModuleFinderTest)
diff --git a/python3.spec b/python3.spec
index 9967e52..e152006 100644
--- a/python3.spec
+++ b/python3.spec
@@ -2,14 +2,13 @@
# Conditionals and other variables controlling the build
# ======================================================
+%global with_rewheel 0
+
%global pybasever 3.4
# pybasever without the dot:
%global pyshortver 34
-# prereleasetag
-%global prerel rc2
-
%global pylibdir %{_libdir}/python%{pybasever}
%global dynload_dir %{pylibdir}/lib-dynload
@@ -129,7 +128,7 @@
Summary: Version 3 of the Python programming language aka Python 3000
Name: python3
Version: %{pybasever}.0
-Release: %{?prerel:0.}1%{?prerel:.%{prerel}}%{?dist}
+Release: 1%{?dist}
License: Python
Group: Development/Languages
@@ -187,12 +186,17 @@ BuildRequires: valgrind-devel
BuildRequires: xz-devel
BuildRequires: zlib-devel
+%if 0%{?with_rewheel}
+BuildRequires: python3-setuptools
+BuildRequires: python3-pip
+%endif
+
# =======================
# Source code and patches
# =======================
-Source: http://www.python.org/ftp/python/%{version}/Python-%{version}%{?prerel}.tar.xz
+Source: http://www.python.org/ftp/python/%{version}/Python-%{version}.tar.xz
# Avoid having various bogus auto-generated Provides lines for the various
# python c modules' SONAMEs:
@@ -632,6 +636,28 @@ Patch186: 00186-dont-raise-from-py_compile.patch
# relying on this will fail (test_filename_changing_on_output_single_dir)
Patch188: 00188-fix-lib2to3-tests-when-hashlib-doesnt-compile-properly.patch
+# 00189 #
+#
+# Add the rewheel module, allowing to recreate wheels from already installed
+# ones
+# https://github.com/bkabrda/rewheel
+%if 0%{with_rewheel}
+Patch189: 00189-add-rewheel-module.patch
+%endif
+
+# 00190 #
+#
+# Fix tests with SQLite >= 3.8.4
+# http://bugs.python.org/issue20901
+# http://hg.python.org/cpython/rev/4d626a9df062
+Patch190: 00190-fix-tests-with-sqlite-3.8.4.patch
+
+# 00193
+#
+# Skip correct number of *.pyc file bytes in ModuleFinder.load_module
+# rhbz#1060338
+# http://bugs.python.org/issue20778
+Patch193: 00193-skip-correct-num-of-pycfile-bytes-in-modulefinder.patch
# (New patches go here ^^^)
#
@@ -895,6 +921,13 @@ done
# 00187: upstream as of Python 3.4.0b1
%patch188 -p1
+%if 0%{with_rewheel}
+%patch189 -p1
+%endif
+
+%patch190 -p1
+%patch193 -p1
+
# Currently (2010-01-15), http://docs.python.org/library is for 2.6, and there
# are many differences between 2.6 and the Python 3 library.
#
@@ -1506,6 +1539,11 @@ rm -fr %{buildroot}
%{pylibdir}/ensurepip/__pycache__/*%{bytecode_suffixes}
%exclude %{pylibdir}/ensurepip/_bundled
+%dir %{pylibdir}/ensurepip/rewheel/
+%dir %{pylibdir}/ensurepip/rewheel/__pycache__/
+%{pylibdir}/ensurepip/rewheel/*.py
+%{pylibdir}/ensurepip/rewheel/__pycache__/*%{bytecode_suffixes}
+
%{pylibdir}/html
%{pylibdir}/http
%{pylibdir}/idlelib
@@ -1759,6 +1797,11 @@ rm -fr %{buildroot}
# ======================================================
%changelog
+* Tue Apr 15 2014 Matej Stuchlik <mstuchli at redhat.com> - 3.4.0-1
+- Update to Python 3.4 final
+- Add patch adding the rewheel module
+- Merge patches from master
+
* Wed Jan 08 2014 Bohuslav Kabrda <bkabrda at redhat.com> - 3.4.0-0.1.b2
- Update to Python 3.4 beta 2.
- Refreshed patches: 55 (systemtap), 146 (hashlib-fips), 154 (test_gdb noise)
More information about the scm-commits
mailing list