From 043a51c0c2a025b84b0fb14c157add7236d7a526 Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Sun, 2 Oct 2016 18:48:56 +0200 Subject: guix: python-build-system: Fix an outdated comment. --- guix/build/python-build-system.scm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'guix/build') diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 9109fb4ac7..e906e60699 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -137,8 +137,7 @@ installed with setuptools." #t)) (define %standard-phases - ;; 'configure' and 'build' phases are not needed. Everything is done during - ;; 'install'. + ;; 'configure' phase is not needed. (modify-phases gnu:%standard-phases (add-after 'unpack 'ensure-no-mtimes-pre-1980 ensure-no-mtimes-pre-1980) (delete 'configure) -- cgit v1.2.3 From 7db40bce58e149ecb541d295e01cfbfe953d39a3 Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Wed, 28 Sep 2016 10:42:35 +0200 Subject: guix: build all Python packages with --single-version-externally-managed. This requires setuptools to be installed together with python, which is the case for Python 3 anyway and which we do for our build of Python 2 (see last commit). * guix/build/python-build-system.scm (install): Add "--single-version-externally-managed" and "--root=/" to params to be passed to call-setuppy. Remove thus needless manipulation of PYTHONPATH. Remove now unused argument "inputs". --- guix/build/python-build-system.scm | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) (limited to 'guix/build') diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index e906e60699..2424fed310 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2013 Andreas Enge ;;; Copyright © 2013 Nikita Karetnikov ;;; Copyright © 2015 Mark H Weaver +;;; Copyright © 2016 Hartmut Goebel ;;; ;;; This file is part of GNU Guix. ;;; @@ -60,25 +61,15 @@ (major+minor (take components 2))) (string-join major+minor "."))) -(define* (install #:key outputs inputs (configure-flags '()) +(define* (install #:key outputs (configure-flags '()) #:allow-other-keys) "Install a given Python package." (let* ((out (assoc-ref outputs "out")) - (params (append (list (string-append "--prefix=" out)) - configure-flags)) - (python-version (get-python-version (assoc-ref inputs "python"))) - (old-path (getenv "PYTHONPATH")) - (add-path (string-append out "/lib/python" python-version - "/site-packages/"))) - ;; create the module installation directory and add it to PYTHONPATH - ;; to make setuptools happy - (mkdir-p add-path) - (setenv "PYTHONPATH" - (string-append (if old-path - (string-append old-path ":") - "") - add-path)) - (call-setuppy "install" params))) + (params (append (list (string-append "--prefix=" out) + "--single-version-externally-managed" + "--root=/") + configure-flags))) + (call-setuppy "install" params))) (define* (wrap #:key inputs outputs #:allow-other-keys) (define (list-of-files dir) -- cgit v1.2.3 From 46bcdcc287ecfc1db8b7a0429e72517f407b580d Mon Sep 17 00:00:00 2001 From: Marius Bakke Date: Thu, 29 Sep 2016 18:41:35 +0100 Subject: guix: python-build-system: Import setuptools before calling `setup.py'. This is needed for packages using "distutils" instead of "setuptools" since the former does not understand the "--single-version-externally-managed" flag. Also export __file__ since it will be unset when setup.py is called from python "exec". * guix/build/python-build-system.scm (call-setuppy): extend "python setup.py" call to import setuptools, export __file__, and call setup.py from setuptools python environment. Co-Authored-By: Hartmut Goebel --- guix/build/python-build-system.scm | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'guix/build') diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 2424fed310..6086df3e82 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -36,13 +36,25 @@ ;; ;; Code: +(define setuptools-shim + ;; Run setup.py with "setuptools" being imported, which will patch + ;; "distutils". This is needed for packages using "distutils" instead of + ;; "setuptools" since the former does not understand the + ;; "--single-version-externally-managed" flag. + ;; Python code taken from pip 9.0.1 pip/utils/setuptools_build.py + (string-append + "import setuptools, tokenize;__file__='setup.py';" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))")) (define (call-setuppy command params) (if (file-exists? "setup.py") (begin (format #t "running \"python setup.py\" with command ~s and parameters ~s~%" command params) - (zero? (apply system* "python" "setup.py" command params))) + (zero? (apply system* "python" "-c" setuptools-shim command params))) (error "no setup.py found"))) (define* (build #:rest empty) -- cgit v1.2.3 From 5f7565d190cf380b7bae2ce12dba38aff98c4eb9 Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Sun, 2 Oct 2016 14:03:32 +0200 Subject: guix: python-build-system: Add option "#:use-setuptools?" (default true). * guix/build-system/python.scm (python-build): New keyword argument "#:use-setuptools?", defaulting to #t. * guix/build/python-build-system.scm (call-setup-py): New positional parameter "use-setuptools?". If false, do not use the shim-wrapper for addin setuptools. (build, check): accept keyword- parameter, and pass to call-setuppy. (install): same; if "use-setuptools?" is false, do not use options "--root" and "--single-version-externally-managed" for setup.py. * doc/guix.texi (Build Systems): Document it. --- doc/guix.texi | 5 +++++ guix/build-system/python.scm | 2 ++ guix/build/python-build-system.scm | 28 +++++++++++++++++----------- 3 files changed, 24 insertions(+), 11 deletions(-) (limited to 'guix/build') diff --git a/doc/guix.texi b/doc/guix.texi index 2691e24faf..5f2807654b 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -3137,6 +3137,11 @@ the @code{#:python} parameter. This is a useful way to force a package to be built for a specific version of the Python interpreter, which might be necessary if the package is only compatible with a single interpreter version. + +By default guix calls @code{setup.py} under control of +@code{setuptools}, much like @command{pip} does. Some packages are not +compatible with setuptools (and pip), thus you can disable this by +setting the @code{#:use-setuptools} parameter to @code{#f}. @end defvr @defvr {Scheme Variable} perl-build-system diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm index adeceb4a89..d4d3d28f2a 100644 --- a/guix/build-system/python.scm +++ b/guix/build-system/python.scm @@ -177,6 +177,7 @@ pre-defined variants." #:key (tests? #t) (test-target "test") + (use-setuptools? #t) (configure-flags ''()) (phases '(@ (guix build python-build-system) %standard-phases)) @@ -204,6 +205,7 @@ provides a 'setup.py' file as its build system." #:system ,system #:test-target ,test-target #:tests? ,tests? + #:use-setuptools? ,use-setuptools? #:phases ,phases #:outputs %outputs #:search-paths ',(map search-path-specification->sexp diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 6086df3e82..7ccc9386cf 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -49,22 +49,25 @@ "f.close();" "exec(compile(code, __file__, 'exec'))")) -(define (call-setuppy command params) +(define (call-setuppy command params use-setuptools?) (if (file-exists? "setup.py") (begin (format #t "running \"python setup.py\" with command ~s and parameters ~s~%" command params) - (zero? (apply system* "python" "-c" setuptools-shim command params))) + (if use-setuptools? + (zero? (apply system* "python" "-c" setuptools-shim + command params)) + (zero? (apply system* "python" "./setup.py" command params)))) (error "no setup.py found"))) -(define* (build #:rest empty) +(define* (build #:key use-setuptools? #:allow-other-keys) "Build a given Python package." - (call-setuppy "build" '())) + (call-setuppy "build" '() use-setuptools?)) -(define* (check #:key tests? test-target #:allow-other-keys) +(define* (check #:key tests? test-target use-setuptools? #:allow-other-keys) "Run the test suite of a given Python package." (if tests? - (call-setuppy test-target '()) + (call-setuppy test-target '() use-setuptools?) #t)) (define (get-python-version python) @@ -73,15 +76,18 @@ (major+minor (take components 2))) (string-join major+minor "."))) -(define* (install #:key outputs (configure-flags '()) +(define* (install #:key outputs (configure-flags '()) use-setuptools? #:allow-other-keys) "Install a given Python package." (let* ((out (assoc-ref outputs "out")) - (params (append (list (string-append "--prefix=" out) - "--single-version-externally-managed" - "--root=/") + (params (append (list (string-append "--prefix=" out)) + (if use-setuptools? + ;; distutils does not accept these flags + (list "--single-version-externally-managed" + "--root=/") + '()) configure-flags))) - (call-setuppy "install" params))) + (call-setuppy "install" params use-setuptools?))) (define* (wrap #:key inputs outputs #:allow-other-keys) (define (list-of-files dir) -- cgit v1.2.3 From a2ff4f0240f0fac484836bb8ffb2f86917369666 Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Sat, 8 Oct 2016 12:04:20 +0200 Subject: guix: python-build-system: Add helpers for getting and setting PYTHONPATH. * guix/build/python-build-system.scm (add-installed-pythonpath, site-packages): New exported procedures. --- guix/build/python-build-system.scm | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'guix/build') diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 7ccc9386cf..22c4f7d38a 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -28,6 +28,8 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:export (%standard-phases + add-installed-pythonpath + site-packages python-build)) ;; Commentary: @@ -76,6 +78,24 @@ (major+minor (take components 2))) (string-join major+minor "."))) +(define (site-packages inputs outputs) + "Return the path of the current output's Python site-package." + (let* ((out (assoc-ref outputs "out")) + (python (assoc-ref inputs "python"))) + (string-append out "/lib/python" + (get-python-version python) + "/site-packages/"))) + +(define (add-installed-pythonpath inputs outputs) + "Prepend the Python site-package of OUTPUT to PYTHONPATH. This is useful +when running checks after installing the package." + (let ((old-path (getenv "PYTHONPATH")) + (add-path (site-packages inputs outputs))) + (setenv "PYTHONPATH" + (string-append add-path + (if old-path (string-append ":" old-path) ""))) + #t)) + (define* (install #:key outputs (configure-flags '()) use-setuptools? #:allow-other-keys) "Install a given Python package." -- cgit v1.2.3 From b002f964bb3d69c77856ea7dcadfe82383050512 Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Fri, 7 Oct 2016 17:17:00 +0200 Subject: guix: python-build-system: Delete .egg-info file created in phase check. * guix/build/python-build-system.scm (check): Delete .egg-info dirs which did not exist prior to calling setup.py but afterwards. --- guix/build/python-build-system.scm | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'guix/build') diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 22c4f7d38a..310ba8aa2e 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -69,7 +69,15 @@ (define* (check #:key tests? test-target use-setuptools? #:allow-other-keys) "Run the test suite of a given Python package." (if tests? - (call-setuppy test-target '() use-setuptools?) + ;; Running `setup.py test` creates an additional .egg-info directory in + ;; build/lib in some cases, e.g. if the source is in a sub-directory + ;; (given with `package_dir`). This will by copied to the output, too, + ;; so we need to remove. + (let ((before (find-files "build" "\\.egg-info$" #:directories? #t))) + (call-setuppy test-target '() use-setuptools?) + (let* ((after (find-files "build" "\\.egg-info$" #:directories? #t)) + (inter (lset-difference eqv? after before))) + (for-each delete-file-recursively inter))) #t)) (define (get-python-version python) -- cgit v1.2.3 From c1019287a4aab55ebffab4710b9a85b6c9f1b7ed Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Tue, 15 Nov 2016 16:57:21 +0100 Subject: guix: python-build-system: Add background about Python installation methods. --- guix/build/python-build-system.scm | 68 +++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) (limited to 'guix/build') diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 310ba8aa2e..3f280b0ac0 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -36,7 +36,70 @@ ;; ;; Builder-side code of the standard Python package build procedure. ;; -;; Code: +;; +;; Backgound about the Python installation methods +;; +;; In Python there are different ways to install packages: distutils, +;; setuptools, easy_install and pip. All of these are sharing the file +;; setup.py, introduced with distutils in Python 2.0. The setup.py file can be +;; considered as a kind of Makefile accepting targets (or commands) like +;; "build" and "install". As of autumn 2016 the recommended way to install +;; Python packages is using pip. +;; +;; For both distutils and setuptools, running "python setup.py install" is the +;; way to install Python packages. With distutils the "install" command +;; basically copies all packages into /lib/pythonX.Y/site-packages. +;; +;; Some time later "setuptools" was established to enhance distutils. To use +;; setuptools, the developer imports setuptools in setup.py. When importing +;; setuptools, the original "install" command gets overwritten by setuptools' +;; "install" command. +;; +;; The command-line tools easy_install and pip are both capable of finding and +;; downloading the package source from PyPI (the Python Package Index). Both +;; of them import setuptools and execute the "setup.py" file under their +;; control. Thus the "setup.py" behaves as if the developer had imported +;; setuptools within setup.py - even is still using only distutils. +;; +;; Setuptools' "install" command (to be more precise: the "easy_install" +;; command which is called by "install") will put the path of the currently +;; installed version of each package and it's dependencies (as declared in +;; setup.py) into an "easy-install.pth" file. In Guix each packages gets its +;; own "site-packages" directory and thus an "easy-install.pth" of its own. +;; To avoid conflicts, the python build system renames the file to +;; .pth in the phase rename-pth-file. To ensure that Python will +;; process the .pth file, easy_install also creates a basic "site.py" in each +;; "site-packages" directory. The file is the same for all packages, thus +;; there is no need to rename it. For more information about .pth files and +;; the site module, please refere to +;; https://docs.python.org/3/library/site.html. +;; +;; The .pth files contain the file-system paths (pointing to the store) of all +;; dependencies. So the dependency is hidden in the .pth file but is not +;; visible in the file-system. Now if packages A and B both required packages +;; P, but in different versions, Guix will not detect this when installing +;; both A and B to a profile. (For details and example see +;; https://lists.gnu.org/archive/html/guix-devel/2016-10/msg01233.html.) +;; +;; Pip behaves a bit different then easy_install: it always executes +;; "setup.py" with the option "--single-version-externally-managed" set. This +;; makes setuptools' "install" command run the original "install" command +;; instead of the "easy_install" command, so no .pth file (and no site.py) +;; will be created. The "site-packages" directory only contains the package +;; and the related .egg-info directory. +;; +;; This is exactly what we need for Guix and this is what we mimic in the +;; install phase below. +;; +;; As a draw back, the magic of the .pth file of linking to the other required +;; packages is gone and these packages have now to be declared as +;; "propagated-inputs". +;; +;; Note: Importing setuptools also adds two sub-commands: "install_egg_info" +;; and "install_scripts". These sub-commands are executed even if +;; "--single-version-externally-managed" is set, thus the .egg-info directory +;; and the scripts defined in entry-points will always be created. + (define setuptools-shim ;; Run setup.py with "setuptools" being imported, which will patch @@ -149,6 +212,9 @@ when running checks after installing the package." (define* (rename-pth-file #:key name inputs outputs #:allow-other-keys) "Rename easy-install.pth to NAME.pth to avoid conflicts between packages installed with setuptools." + ;; Even if the "easy-install.pth" is not longer created, we kept this phase. + ;; There still may be packages creating an "easy-install.pth" manually for + ;; some good reason. (let* ((out (assoc-ref outputs "out")) (python (assoc-ref inputs "python")) (site-packages (string-append out "/lib/python" -- cgit v1.2.3 From c062b1eb6c9d799f0015e26b14cd77eaf8d946dd Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 28 Nov 2016 22:08:17 +0100 Subject: pull: Set '%nix-instantiate' to a sensible value. Reported by ng0 . Fixes . * guix/build/pull.scm (build-guix): Replace "@NIX_INSTANTIATE@" in guix/config.scm with "nix-instantiate". --- guix/build/pull.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'guix/build') diff --git a/guix/build/pull.scm b/guix/build/pull.scm index 871bf6f535..6034e93cbf 100644 --- a/guix/build/pull.scm +++ b/guix/build/pull.scm @@ -84,7 +84,7 @@ containing the source code. Write any debugging output to DEBUG-PORT." (("@GZIP@") (string-append gzip "/bin/gzip")) (("@BZIP2@") (string-append bzip2 "/bin/bzip2")) (("@XZ@") (string-append xz "/bin/xz")) - (("@NIX_INSTANTIATE@") "")) ;remnants from the past + (("@NIX_INSTANTIATE@") "nix-instantiate")) ;for (guix import nix) ;; Augment the search path so Scheme code can be compiled. (set! %load-path (cons out %load-path)) -- cgit v1.2.3