diff options
Diffstat (limited to 'guix/scripts/substitute.scm')
| -rwxr-xr-x | guix/scripts/substitute.scm | 169 |
1 files changed, 125 insertions, 44 deletions
diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index cdf591ac4d..fedb33019d 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013-2022 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013-2023 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2014 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> ;;; Copyright © 2020 Christopher Baines <mail@cbaines.net> @@ -366,7 +366,7 @@ authorized substitutes." When FRESH? is true, delete any cached connections for URI and open a new one. Return #f if URI's scheme is 'file' or #f. -When true, TIMEOUT is the maximum number of milliseconds to wait for +When true, TIMEOUT is the maximum number of seconds to wait for connection establishment. When VERIFY-CERTIFICATE? is true, verify HTTPS server certificates." (define host (uri-host uri)) @@ -437,20 +437,13 @@ server certificates." "Bind PORT with EXP... to a socket connected to URI." (call-with-cached-connection uri (lambda (port) exp ...))) -(define* (process-substitution port store-item destination - #:key cache-urls acl - deduplicate? print-build-trace?) - "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to -DESTINATION as a nar file. Verify the substitute against ACL, and verify its -hash against what appears in the narinfo. When DEDUPLICATE? is true, and if -DESTINATION is in the store, deduplicate its files. Print a status line to -PORT." - (define narinfo - (lookup-narinfo cache-urls store-item - (if (%allow-unauthenticated-substitutes?) - (const #t) - (cut valid-narinfo? <> acl)))) - +(define* (download-nar narinfo destination + #:key status-port + deduplicate? print-build-trace?) + "Download the nar prescribed in NARINFO, which is assumed to be authentic +and authorized, and write it to DESTINATION. When DEDUPLICATE? is true, and +if DESTINATION is in the store, deduplicate its files. Print a status line to +STATUS-PORT." (define destination-in-store? (string-prefix? (string-append (%store-prefix) "/") destination)) @@ -467,33 +460,24 @@ PORT." (let ((port (open-file (uri-path uri) "r0b"))) (values port (stat:size (stat port))))) ((http https) - (guard (c ((http-get-error? c) - (leave (G_ "download from '~a' failed: ~a, ~s~%") - (uri->string (http-get-error-uri c)) - (http-get-error-code c) - (http-get-error-reason c)))) - ;; Test this with: - ;; sudo tc qdisc add dev eth0 root netem delay 1500ms - ;; and then cancel with: - ;; sudo tc qdisc del dev eth0 root - (with-timeout %fetch-timeout - (begin - (warning (G_ "while fetching ~a: server is somewhat slow~%") - (uri->string uri)) - (warning (G_ "try `--no-substitutes' if the problem persists~%"))) - (with-cached-connection uri port - (http-fetch uri #:text? #f - #:port port - #:keep-alive? #t - #:buffered? #f))))) + ;; Test this with: + ;; sudo tc qdisc add dev eth0 root netem delay 1500ms + ;; and then cancel with: + ;; sudo tc qdisc del dev eth0 root + (with-timeout %fetch-timeout + (begin + (warning (G_ "while fetching ~a: server is somewhat slow~%") + (uri->string uri)) + (warning (G_ "try `--no-substitutes' if the problem persists~%"))) + (with-cached-connection uri port + (http-fetch uri #:text? #f + #:port port + #:keep-alive? #t + #:buffered? #f)))) (else (leave (G_ "unsupported substitute URI scheme: ~a~%") (uri->string uri))))) - (unless narinfo - (leave (G_ "no valid substitute for '~a'~%") - store-item)) - (let ((uri compression file-size (narinfo-best-uri narinfo #:fast-decompression? @@ -575,14 +559,109 @@ PORT." (let ((actual (get-hash))) (if (bytevector=? actual expected) ;; Tell the daemon that we're done. - (format port "success ~a ~a~%" + (format status-port "success ~a ~a~%" (narinfo-hash narinfo) (narinfo-size narinfo)) ;; The actual data has a different hash than that in NARINFO. - (format port "hash-mismatch ~a ~a ~a~%" + (format status-port "hash-mismatch ~a ~a ~a~%" (hash-algorithm-name algorithm) (bytevector->nix-base32-string expected) (bytevector->nix-base32-string actual))))))) +(define system-error? + (let ((kind-and-args? (exception-predicate &exception-with-kind-and-args))) + (lambda (exception) + "Return true if EXCEPTION is a Guile 'system-error exception." + (and (kind-and-args? exception) + (eq? 'system-error (exception-kind exception)))))) + +(define network-error? + (let ((kind-and-args? (exception-predicate &exception-with-kind-and-args))) + (lambda (exception) + "Return true if EXCEPTION denotes a networking error." + (or (and (system-error? exception) + (let ((errno (system-error-errno + (cons 'system-error (exception-args exception))))) + (memv errno (list ECONNRESET ECONNABORTED + ECONNREFUSED EHOSTUNREACH + ENOENT)))) ;for "file://" + (and (kind-and-args? exception) + (memq (exception-kind exception) + '(gnutls-error getaddrinfo-error))) + (and (http-get-error? exception) + (begin + (warning (G_ "download from '~a' failed: ~a, ~s~%") + (uri->string (http-get-error-uri exception)) + (http-get-error-code exception) + (http-get-error-reason exception)) + #t)))))) + +(define* (process-substitution/fallback port narinfo destination + #:key cache-urls acl + deduplicate? print-build-trace?) + "Attempt to substitute NARINFO, which is assumed to be authorized or +equivalent, by trying to download its nar from each entry in CACHE-URLS. + +This can be less efficient than 'lookup-narinfo', which stops at the first +entry that provides a valid narinfo, but it makes sure we eventually find a +way to download the nar." + ;; Note: Keep NARINFO's uri-base in CACHE-URLS: that lets us retry in case + ;; this was a transient issue. + (let loop ((cache-urls cache-urls)) + (match cache-urls + (() + (leave (G_ "failed to find alternative substitute for '~a'~%") + (narinfo-path narinfo))) + ((cache-url rest ...) + (match (lookup-narinfos cache-url + (list (narinfo-path narinfo)) + #:open-connection + open-connection-for-uri/cached) + ((alternate) + (if (or (equivalent-narinfo? narinfo alternate) + (valid-narinfo? alternate acl) + (%allow-unauthenticated-substitutes?)) + (guard (c ((network-error? c) (loop rest))) + (download-nar alternate destination + #:status-port port + #:deduplicate? deduplicate? + #:print-build-trace? print-build-trace?)) + (loop rest))) + (() + (loop rest))))))) + +(define* (process-substitution port store-item destination + #:key cache-urls acl + deduplicate? print-build-trace?) + "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to +DESTINATION as a nar file. Verify the substitute against ACL, and verify its +hash against what appears in the narinfo. When DEDUPLICATE? is true, and if +DESTINATION is in the store, deduplicate its files. Print a status line to +PORT." + (define narinfo + (lookup-narinfo cache-urls store-item + (if (%allow-unauthenticated-substitutes?) + (const #t) + (cut valid-narinfo? <> acl)))) + + (unless narinfo + (leave (G_ "no valid substitute for '~a'~%") + store-item)) + + (guard (c ((network-error? c) + (format (current-error-port) + (G_ "retrying download of '~a' with other substitute URLs...~%") + store-item) + (process-substitution/fallback port narinfo destination + #:cache-urls cache-urls + #:acl acl + #:deduplicate? deduplicate? + #:print-build-trace? + print-build-trace?))) + (download-nar narinfo destination + #:status-port port + #:deduplicate? deduplicate? + #:print-build-trace? print-build-trace?))) + ;;; ;;; Entry point. @@ -627,10 +706,12 @@ substitutes may be unavailable\n"))))) (string-drop option=value (+ 1 equal-sign)))))) (string-tokenize newline-separated %not-newline))))) -(define (find-daemon-option option) - "Return the value of build daemon option OPTION, or #f if it could not be +(define find-daemon-option + (let ((options (delay (daemon-options)))) + (lambda (option) + "Return the value of build daemon option OPTION, or #f if it could not be found." - (assoc-ref (daemon-options) option)) + (assoc-ref (force options) option)))) (define %default-substitute-urls (match (and=> (or (find-daemon-option "untrusted-substitute-urls") ;client |