summaryrefslogtreecommitdiff
path: root/guix/build/pyproject-build-system.scm
blob: c69ccc9d64b2133fd9e34f567f08f19307ca5862 (about) (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2021 Lars-Dominik Braun <lars@6xq.net>
;;; Copyright © 2022 Marius Bakke <marius@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (guix build pyproject-build-system)
  #:use-module ((guix build python-build-system) #:prefix python:)
  #:use-module (guix build utils)
  #:use-module (guix build json)
  #:use-module (ice-9 match)
  #:use-module (ice-9 ftw)
  #:use-module (ice-9 format)
  #:use-module (ice-9 rdelim)
  #:use-module (ice-9 regex)
  #:use-module (srfi srfi-1)
  #:use-module (srfi srfi-26)
  #:use-module (srfi srfi-34)
  #:use-module (srfi srfi-35)
  #:export (%standard-phases
            add-installed-pythonpath
            site-packages
            python-version
            pyproject-build))

;;; Commentary:
;;;
;;; PEP 517-compatible build system for Python packages.
;;;
;;; PEP 517 mandates the use of a TOML file called pyproject.toml at the
;;; project root, describing build and runtime dependencies, as well as the
;;; build system, which can be different from setuptools. This module uses
;;; that file to extract the build system used and call its wheel-building
;;; entry point build_wheel (see 'build). setuptools’ wheel builder is
;;; used as a fallback if either no pyproject.toml exists or it does not
;;; declare a build-system. It supports config_settings through the
;;; standard #:configure-flags argument.
;;;
;;; This wheel, which is just a ZIP file with a file structure defined
;;; by PEP 427 (https://www.python.org/dev/peps/pep-0427/), is then unpacked
;;; and its contents are moved to the appropriate locations in 'install.
;;;
;;; Then entry points, as defined by the PyPa Entry Point Specification
;;; (https://packaging.python.org/specifications/entry-points/) are read
;;; from a file called entry_points.txt in the package’s site-packages
;;; subdirectory and scripts are written to bin/. These are not part of a
;;; wheel and expected to be created by the installing utility.
;;; TODO: Add support for PEP-621 entry points.
;;;
;;; Caveats:
;;; - There is no support for in-tree build backends.
;;;
;;; Code:
;;;

;; Re-export these variables from python-build-system as many packages
;; rely on these.
(define python-version python:python-version)
(define site-packages python:site-packages)
(define add-installed-pythonpath python:add-installed-pythonpath)

;; Base error type.
(define-condition-type &python-build-error &error python-build-error?)

;; Raised when 'check cannot find a valid test system in the inputs.
(define-condition-type &test-system-not-found &python-build-error
                       test-system-not-found?)

;; Raised when multiple wheels are created by 'build.
(define-condition-type &cannot-extract-multiple-wheels &python-build-error
                       cannot-extract-multiple-wheels?)

;; Raised, when no wheel has been built by the build system.
(define-condition-type &no-wheels-built &python-build-error no-wheels-built?)

(define* (build #:key outputs build-backend configure-flags #:allow-other-keys)
  "Build a given Python package."

  (define (pyproject.toml->build-backend file)
    "Look up the build backend in a pyproject.toml file."
    (call-with-input-file file
      (lambda (in)
        (let loop
          ((line (read-line in 'concat)))
          (if (eof-object? line) #f
              (let ((m (string-match "build-backend = [\"'](.+)[\"']" line)))
                (if m
                    (match:substring m 1)
                    (loop (read-line in 'concat)))))))))

  (let* ((wheel-output (assoc-ref outputs "wheel"))
         (wheel-dir (if wheel-output wheel-output "dist"))
         ;; There is no easy way to get data from Guile into Python via
         ;; s-expressions, but we have JSON serialization already, which Python
         ;; also supports out-of-the-box.
         (config-settings (call-with-output-string
                            (cut write-json configure-flags <>)))
         ;; python-setuptools’ default backend supports setup.py *and*
         ;; pyproject.toml. Allow overriding this automatic detection via
         ;; build-backend.
         (auto-build-backend (if (file-exists? "pyproject.toml")
                                 (pyproject.toml->build-backend
                                  "pyproject.toml")
                                 #f))
         ;; Use build system detection here and not in importer, because a) we
         ;; have alot of legacy packages and b) the importer cannot update arbitrary
         ;; fields in case a package switches its build system.
         (use-build-backend (or build-backend
                                auto-build-backend
                                "setuptools.build_meta")))
    (format #t
     "Using '~a' to build wheels, auto-detected '~a', override '~a'.~%"
     use-build-backend auto-build-backend build-backend)
    (mkdir-p wheel-dir)
    ;; Call the PEP 517 build function, which drops a .whl into wheel-dir.
    (invoke "python" "-c"
     "import sys, importlib, json
config_settings = json.loads (sys.argv[3])
builder = importlib.import_module(sys.argv[1])
builder.build_wheel(sys.argv[2], config_settings=config_settings)"
     use-build-backend
     wheel-dir
     config-settings)))

(define* (check #:key tests? test-backend test-flags #:allow-other-keys)
  "Run the test suite of a given Python package."
  (if tests?
      ;; Unfortunately with PEP 517 there is no common method to specify test
      ;; systems.  Guess test system based on inputs instead.
      (let* ((pytest (which "pytest"))
             (nosetests (which "nosetests"))
             (nose2 (which "nose2"))
             (have-setup-py (file-exists? "setup.py"))
             (use-test-backend
              (or test-backend
                  ;; Prefer pytest
                  (if pytest 'pytest #f)
                  (if nosetests 'nose #f)
                  (if nose2 'nose2 #f)
                  ;; But fall back to setup.py, which should work for most
                  ;; packages. XXX: would be nice not to depend on setup.py here?
                  ;; fails more often than not to find any tests at all.  Maybe
                  ;; we can run `python -m unittest`?
                  (if have-setup-py 'setup.py #f))))
        (format #t "Using ~a~%" use-test-backend)
        (match use-test-backend
          ('pytest
           (apply invoke pytest "-vv" test-flags))
          ('nose
           (apply invoke nosetests "-v" test-flags))
          ('nose2
           (apply invoke nose2 "-v" "--pretty-assert" test-flags))
          ('setup.py
           (apply invoke "python" "setup.py"
                  (if (null? test-flags)
                      '("test" "-v")
                      test-flags)))
          ;; The developer should explicitly disable tests in this case.
          (else (raise (condition (&test-system-not-found))))))
      (format #t "test suite not run~%")))

(define* (install #:key inputs outputs #:allow-other-keys)
  "Install a wheel file according to PEP 427"
  ;; See https://www.python.org/dev/peps/pep-0427/#installing-a-wheel-distribution-1-0-py32-none-any-whl
  (let ((site-dir (site-packages inputs outputs))
        (python (assoc-ref inputs "python"))
        (out (assoc-ref outputs "out")))
    (define (extract file)
      "Extract wheel (ZIP file) into site-packages directory"
      ;; Use Python’s zipfile to avoid extra dependency
      (invoke "python" "-m" "zipfile" "-e" file site-dir))

    (define python-hashbang
      (string-append "#!" python "/bin/python"))

    (define* (merge-directories source destination
                                #:optional (post-move #f))
      "Move all files in SOURCE into DESTINATION, merging the two directories."
      (format #t "Merging directory ~a into ~a~%" source destination)
      (for-each (lambda (file)
                  (format #t "~a/~a -> ~a/~a~%"
                          source file destination file)
                  (mkdir-p destination)
                  (rename-file (string-append source "/" file)
                               (string-append destination "/" file))
                  (when post-move
                    (post-move file)))
                (scandir source
                         (negate (cut member <> '("." "..")))))
      (rmdir source))

    (define (expand-data-directory directory)
      "Move files from all .data subdirectories to their respective\ndestinations."
      ;; Python’s distutils.command.install defines this mapping from source to
      ;; destination mapping.
      (let ((source (string-append directory "/scripts"))
            (destination (string-append out "/bin")))
        (when (file-exists? source)
          (merge-directories source destination
                             (lambda (f)
                               (let ((dest-path (string-append destination
                                                               "/" f)))
                                 (chmod dest-path #o755)
                                 ;; PEP 427 recommends that installers rewrite
                                 ;; this odd shebang.
                                 (substitute* dest-path
                                   (("#!python")
                                    python-hashbang)))))))
      ;; Data can be contained in arbitrary directory structures.  Most
      ;; commonly it is used for share/.
      (let ((source (string-append directory "/data"))
            (destination out))
        (when (file-exists? source)
          (merge-directories source destination)))
      (let* ((distribution (car (string-split (basename directory) #\-)))
             (source (string-append directory "/headers"))
             (destination (string-append out "/include/python"
                                         (python-version python)
                                         "/" distribution)))
        (when (file-exists? source)
          (merge-directories source destination))))

    (define (list-directories base predicate)
      ;; Cannot use find-files here, because it’s recursive.
      (scandir base
               (lambda (name)
                 (let ((stat (lstat (string-append base "/" name))))
                   (and (not (member name '("." "..")))
                        (eq? (stat:type stat) 'directory)
                        (predicate name stat))))))

    (let* ((wheel-output (assoc-ref outputs "wheel"))
           (wheel-dir (if wheel-output wheel-output "dist"))
           (wheels (map (cut string-append wheel-dir "/" <>)
                        (scandir wheel-dir
                                 (cut string-suffix? ".whl" <>)))))
      (cond
        ((> (length wheels) 1)
         ;; This code does not support multiple wheels yet, because their
         ;; outputs would have to be merged properly.
         (raise (condition (&cannot-extract-multiple-wheels))))
        ((= (length wheels) 0)
         (raise (condition (&no-wheels-built)))))
      (for-each extract wheels))
    (let ((datadirs (map (cut string-append site-dir "/" <>)
                         (list-directories site-dir
                                           (file-name-predicate "\\.data$")))))
      (for-each (lambda (directory)
                  (expand-data-directory directory)
                  (rmdir directory)) datadirs))))

(define* (compile-bytecode #:key inputs outputs #:allow-other-keys)
  "Compile installed byte-code in site-packages."
  (let* ((site-dir (site-packages inputs outputs))
         (python (assoc-ref inputs "python"))
         (major-minor (map string->number
                           (take (string-split (python-version python) #\.) 2)))
         (<3.7? (match major-minor
                  ((major minor)
                   (or (< major 3)
                       (and (= major 3)
                            (< minor 7)))))))
    (if <3.7?
        ;; These versions don’t have the hash invalidation modes and do
        ;; not produce reproducible bytecode files.
        (format #t "Skipping bytecode compilation for Python version ~a < 3.7~%"
                (python-version python))
        (invoke "python" "-m" "compileall"
                "--invalidation-mode=unchecked-hash" site-dir))))

(define* (create-entrypoints #:key inputs outputs #:allow-other-keys)
  "Implement Entry Points Specification
(https://packaging.python.org/specifications/entry-points/) by PyPa,
which creates runnable scripts in bin/ from entry point specification
file entry_points.txt.  This is necessary, because wheels do not contain
these binaries and installers are expected to create them."

  (define (entry-points.txt->entry-points file)
    "Specialized parser for Python configfile-like files, in particular
entry_points.txt.  Returns a list of console_script and gui_scripts
entry points."
    (call-with-input-file file
      (lambda (in)
        (let loop ((line (read-line in))
                   (inside #f)
                   (result '()))
          (if (eof-object? line)
              result
              (let* ((group-match (string-match "^\\[(.+)\\]$" line))
                     (group-name (if group-match
                                     (match:substring group-match 1)
                                     #f))
                     (next-inside (if (not group-name)
                                      inside
                                      (or (string=? group-name
                                                    "console_scripts")
                                          (string=? group-name "gui_scripts"))))
                     (item-match (string-match
                                  "^([^ =]+)\\s*=\\s*([^:]+):(.+)$" line)))
                (if (and inside item-match)
                    (loop (read-line in)
                          next-inside
                          (cons (list (match:substring item-match 1)
                                      (match:substring item-match 2)
                                      (match:substring item-match 3))
                                result))
                    (loop (read-line in) next-inside result))))))))

  (define (create-script path name module function)
    "Create a Python script from an entry point’s NAME, MODULE and FUNCTION
and return write it to PATH/NAME."
    (let ((interpreter (which "python"))
          (file-path (string-append path "/" name)))
      (format #t "Creating entry point for '~a.~a' at '~a'.~%"
              module function file-path)
      (call-with-output-file file-path
        (lambda (port)
          ;; Technically the script could also include search-paths,
          ;; but having a generic 'wrap phases also handles manually
          ;; written entry point scripts.
          (format port "#!~a
# Auto-generated entry point script.
import sys
import ~a as mod
sys.exit (mod.~a ())~%" interpreter module function)))
        (chmod file-path #o755)))

  (let* ((site-dir (site-packages inputs outputs))
         (out (assoc-ref outputs "out"))
         (bin-dir (string-append out "/bin"))
         (entry-point-files (find-files site-dir "^entry_points.txt$")))
    (mkdir-p bin-dir)
    (for-each (lambda (f)
                (for-each (lambda (ep)
                            (apply create-script
                                   (cons bin-dir ep)))
                          (entry-points.txt->entry-points f)))
              entry-point-files)))

(define* (set-SOURCE-DATE-EPOCH* #:rest _)
  "Set the 'SOURCE_DATE_EPOCH' environment variable.  This is used by tools
that incorporate timestamps as a way to tell them to use a fixed timestamp.
See https://reproducible-builds.org/specs/source-date-epoch/."
  ;; Use a post-1980 timestamp because the Zip format used in wheels do
  ;; not support timestamps before 1980.
  (setenv "SOURCE_DATE_EPOCH" "315619200"))

(define %standard-phases
  ;; The build phase only builds C extensions and copies the Python sources,
  ;; while the install phase copies then byte-compiles the sources to the
  ;; prefix directory.  The check phase is moved after the installation phase
  ;; to ease testing the built package.
  (modify-phases python:%standard-phases
    (replace 'set-SOURCE-DATE-EPOCH set-SOURCE-DATE-EPOCH*)
    (replace 'build build)
    (replace 'install install)
    (delete 'check)
    ;; Must be before tests, so they can use installed packages’ entry points.
    (add-before 'wrap 'create-entrypoints create-entrypoints)
    (add-after 'wrap 'check check)
    (add-before 'check 'compile-bytecode compile-bytecode)))

(define* (pyproject-build #:key inputs (phases %standard-phases)
                          #:allow-other-keys #:rest args)
  "Build the given Python package, applying all of PHASES in order."
  (apply python:python-build #:inputs inputs #:phases phases args))

;;; pyproject-build-system.scm ends here