From 2c6ab6ccd430550dfbc95fbdd22ae017f39e5901 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Mon, 1 Apr 2013 16:08:31 +0200 Subject: store: Add `store-path-hash-part'. * guix/store.scm (store-path-hash-part): New procedure. * tests/store.scm ("store-path-hash-part", "store-path-hash-part #f"): New tests. --- guix/store.scm | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index 4d078c5899..3bb2656bb6 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -83,7 +83,8 @@ (define-module (guix store) %store-prefix store-path? derivation-path? - store-path-package-name)) + store-path-package-name + store-path-hash-part)) (define %protocol-version #x10c) @@ -751,3 +752,12 @@ (define store-path-rx (and=> (regexp-exec store-path-rx path) (cut match:substring <> 1))) + +(define (store-path-hash-part path) + "Return the hash part of PATH as a base32 string, or #f if PATH is not a +syntactically valid store path." + (let ((path-rx (make-regexp + (string-append"^" (regexp-quote (%store-prefix)) + "/([0-9a-df-np-sv-z]{32})-[^/]+$")))) + (and=> (regexp-exec path-rx path) + (cut match:substring <> 1)))) -- cgit v1.2.3 From f65cf81a3cd15eab993e129977bca46972508b4b Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 2 Apr 2013 10:44:20 +0200 Subject: Add preliminary binary substituter. * guix/scripts/substitute-binary.scm: New file. * Makefile.am (MODULES): Add it. * nix/scripts/substitute-binary.in: New file. * config-daemon.ac: Produce nix/scripts/substitute-binary. * daemon.am (nodist_pkglibexec_SCRIPTS): Add nix/scripts/substitute-binary. * guix/store.scm (substitutable-path-info): Use the `query-substitutable-path-infos' RPC. * nix/nix-daemon/guix-daemon.cc (main): Honor `NIX_SUBSTITUTERS'. * pre-inst-env.in: Set `NIX_SUBSTITUTERS'. * test-env.in: Leave `NIX_SUBSTITUTERS' unchanged. Set `GUIX_BINARY_SUBSTITUTE_URL, and create $NIX_STATE_DIR/substituter-data. Run `guix-daemon' within `./pre-inst-env'. * tests/store.scm ("substitute query"): New test. --- .gitignore | 1 + Makefile.am | 1 + config-daemon.ac | 5 +- daemon.am | 3 +- guix/scripts/substitute-binary.scm | 232 +++++++++++++++++++++++++++++++++++++ guix/store.scm | 2 +- nix/nix-daemon/guix-daemon.cc | 12 +- nix/scripts/substitute-binary.in | 11 ++ pre-inst-env.in | 3 +- test-env.in | 17 ++- tests/store.scm | 39 +++++++ 11 files changed, 313 insertions(+), 13 deletions(-) create mode 100755 guix/scripts/substitute-binary.scm create mode 100644 nix/scripts/substitute-binary.in (limited to 'guix/store.scm') diff --git a/.gitignore b/.gitignore index 302e473fd8..f2b1f1cd39 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ stamp-h[0-9] /doc/guix.tp /doc/guix.vr /doc/guix.vrs +/nix/scripts/substitute-binary diff --git a/Makefile.am b/Makefile.am index 74977c5cf7..888302bd96 100644 --- a/Makefile.am +++ b/Makefile.am @@ -31,6 +31,7 @@ MODULES = \ guix/scripts/package.scm \ guix/scripts/gc.scm \ guix/scripts/pull.scm \ + guix/scripts/substitute-binary.scm \ guix/base32.scm \ guix/utils.scm \ guix/derivations.scm \ diff --git a/config-daemon.ac b/config-daemon.ac index f48741dfda..eed1e23f9e 100644 --- a/config-daemon.ac +++ b/config-daemon.ac @@ -93,8 +93,9 @@ if test "x$guix_build_daemon" = "xyes"; then AC_MSG_RESULT([$GUIX_TEST_ROOT]) AC_SUBST([GUIX_TEST_ROOT]) - AC_CONFIG_FILES([nix/scripts/list-runtime-roots], - [chmod +x nix/scripts/list-runtime-roots]) + AC_CONFIG_FILES([nix/scripts/list-runtime-roots + nix/scripts/substitute-binary], + [chmod +x nix/scripts/list-runtime-roots nix/scripts/substitute-binary]) fi AM_CONDITIONAL([BUILD_DAEMON], [test "x$guix_build_daemon" = "xyes"]) diff --git a/daemon.am b/daemon.am index 0c9bc9fb69..1d4d955a0c 100644 --- a/daemon.am +++ b/daemon.am @@ -159,7 +159,8 @@ nix/libstore/schema.sql.hh: nix/libstore/schema.sql (write (get-string-all in) out)))))" nodist_pkglibexec_SCRIPTS = \ - nix/scripts/list-runtime-roots + nix/scripts/list-runtime-roots \ + nix/scripts/substitute-binary EXTRA_DIST += \ nix/sync-with-upstream \ diff --git a/guix/scripts/substitute-binary.scm b/guix/scripts/substitute-binary.scm new file mode 100755 index 0000000000..6e886b6c96 --- /dev/null +++ b/guix/scripts/substitute-binary.scm @@ -0,0 +1,232 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2013 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (guix scripts substitute-binary) + #:use-module (guix ui) + #:use-module (guix store) + #:use-module (guix utils) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 regex) + #:use-module (ice-9 match) + #:use-module (ice-9 threads) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-9) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-26) + #:use-module (web uri) + #:use-module (web client) + #:use-module (web response) + #:export (guix-substitute-binary)) + +;;; Comment: +;;; +;;; This is the "binary substituter". It is invoked by the daemon do check +;;; for the existence of available "substitutes" (pre-built binaries), and to +;;; actually use them as a substitute to building things locally. +;;; +;;; If possible, substitute a binary for the requested store path, using a Nix +;;; "binary cache". This program implements the Nix "substituter" protocol. +;;; +;;; Code: + +(define (fields->alist port) + "Read recutils-style record from PORT and return them as a list of key/value +pairs." + (define field-rx + (make-regexp "^([[:graph:]]+): (.*)$")) + + (let loop ((line (read-line port)) + (result '())) + (cond ((eof-object? line) + (reverse result)) + ((regexp-exec field-rx line) + => + (lambda (match) + (loop (read-line port) + (alist-cons (match:substring match 1) + (match:substring match 2) + result)))) + (else + (error "unmatched line" line))))) + +(define (alist->record alist make keys) + "Apply MAKE to the values associated with KEYS in ALIST." + (let ((args (map (cut assoc-ref alist <>) keys))) + (apply make args))) + +(define (fetch uri) + (case (uri-scheme uri) + ((file) + (open-input-file (uri-path uri))) + ((http) + (let*-values (((resp port) + ;; XXX: `http-get*' was introduced in 2.0.7, and deprecated + ;; in 2.0.8 (!). Assume it is available here. + (if (version>? "2.0.7" (version)) + (http-get* uri #:decode-body? #f) + (http-get uri #:streaming? #t))) + ((code) + (response-code resp)) + ((size) + (response-content-length resp))) + (case code + ((200) ; OK + port) + ((301 ; moved permanently + 302) ; found (redirection) + (let ((uri (response-location resp))) + (format #t "following redirection to `~a'...~%" + (uri->string uri)) + (fetch uri))) + (else + (error "download failed" (uri->string uri) + code (response-reason-phrase resp)))))))) + +(define-record-type + (%make-cache url store-directory wants-mass-query?) + cache? + (url cache-url) + (store-directory cache-store-directory) + (wants-mass-query? cache-wants-mass-query?)) + +(define (open-cache url) + "Open the binary cache at URL. Return a object on success, or #f on +failure." + (define (download-cache-info url) + ;; Download the `nix-cache-info' from URL, and return its contents as an + ;; list of key/value pairs. + (and=> (false-if-exception (fetch (string->uri url))) + fields->alist)) + + (and=> (download-cache-info (string-append url "/nix-cache-info")) + (lambda (properties) + (alist->record properties + (cut %make-cache url <...>) + '("StoreDir" "WantMassQuery"))))) + +(define-record-type + (%make-narinfo path url compression file-hash file-size nar-hash nar-size + references deriver system) + narinfo? + (path narinfo-path) + (url narinfo-url) + (compression narinfo-compression) + (file-hash narinfo-file-hash) + (file-size narinfo-file-size) + (nar-hash narinfo-hash) + (nar-size narinfo-size) + (references narinfo-references) + (deriver narinfo-deriver) + (system narinfo-system)) + +(define (make-narinfo path url compression file-hash file-size nar-hash nar-size + references deriver system) + "Return a new object." + (%make-narinfo path url compression file-hash + (and=> file-size string->number) + nar-hash + (and=> nar-size string->number) + (string-tokenize references) + (match deriver + ((or #f "") #f) + (_ deriver)) + system)) + +(define (fetch-narinfo cache path) + "Return the record for PATH, or #f if CACHE does not hold PATH." + (define (download url) + ;; Download the `nix-cache-info' from URL, and return its contents as an + ;; list of key/value pairs. + (and=> (false-if-exception (fetch (string->uri url))) + fields->alist)) + + (and=> (download (string-append (cache-url cache) "/" + (store-path-hash-part path) + ".narinfo")) + (lambda (properties) + (alist->record properties make-narinfo + '("StorePath" "URL" "Compression" + "FileHash" "FileSize" "NarHash" "NarSize" + "References" "Deriver" "System"))))) + +(define %cache-url + (or (getenv "GUIX_BINARY_SUBSTITUTE_URL") + "http://hydra.gnu.org")) + + +;;; +;;; Entry point. +;;; + +(define (guix-substitute-binary . args) + "Implement the build daemon's substituter protocol." + (match args + (("--query") + (let ((cache (open-cache %cache-url))) + (let loop ((command (read-line))) + (or (eof-object? command) + (begin + (match (string-tokenize command) + (("have" paths ..1) + ;; Return the subset of PATHS available in CACHE. + (let ((substitutable + (if cache + (par-map (cut fetch-narinfo cache <>) + paths) + '()))) + (for-each (lambda (narinfo) + (when narinfo + (display (narinfo-path narinfo)) + (newline))) + substitutable))) + (("info" paths ..1) + ;; Reply info about PATHS if it's in CACHE. + (let ((substitutable + (if cache + (par-map (cut fetch-narinfo cache <>) + paths) + '()))) + (for-each (lambda (narinfo) + (format #t "~a\n~a\n~a\n" + (narinfo-path narinfo) + (or (and=> (narinfo-deriver narinfo) + (cute string-append + (%store-prefix) "/" + <>)) + "") + (length (narinfo-references narinfo))) + (for-each (cute format #t "~a/~a~%" + (%store-prefix) <>) + (narinfo-references narinfo)) + (format #t "~a\n~a\n" + (or (narinfo-file-size narinfo) 0) + (or (narinfo-size narinfo) 0)) + (newline)) + substitutable))) + (wtf + (error "unknown `--query' command" wtf))) + (loop (read-line))))))) + (("--substitute" store-path destination) + ;; Download PATH and add it to the store. + ;; TODO: Implement. + (format (current-error-port) "substitution not implemented yet~%") + #f) + (("--version") + (show-version-and-exit "guix substitute-binary")))) + +;;; substitute-binary.scm ends here diff --git a/guix/store.scm b/guix/store.scm index 3bb2656bb6..de9785c835 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -662,7 +662,7 @@ (define substitutable-paths store-path-list)) (define substitutable-path-info - (operation (query-substitutable-paths (store-path-list paths)) + (operation (query-substitutable-path-infos (store-path-list paths)) "Return information about the subset of PATHS that is substitutable. For each substitutable path, a `substitutable?' object is returned." diff --git a/nix/nix-daemon/guix-daemon.cc b/nix/nix-daemon/guix-daemon.cc index 1611840bd4..0e2f36150b 100644 --- a/nix/nix-daemon/guix-daemon.cc +++ b/nix/nix-daemon/guix-daemon.cc @@ -200,9 +200,17 @@ main (int argc, char *argv[]) { settings.processEnvironment (); - /* FIXME: Disable substitutes until we have something that works. */ - settings.useSubstitutes = false; + /* Use our substituter by default. */ settings.substituters.clear (); + string subs = getEnv ("NIX_SUBSTITUTERS", "default"); + if (subs == "default") + /* XXX: No substituters until we have something that works. */ + settings.substituters.clear (); + // settings.substituters.push_back (settings.nixLibexecDir + // + "/guix/substitute-binary"); + else + settings.substituters = tokenizeString (subs, ":"); + argp_parse (&argp, argc, argv, 0, 0, 0); diff --git a/nix/scripts/substitute-binary.in b/nix/scripts/substitute-binary.in new file mode 100644 index 0000000000..48d7bb8ff1 --- /dev/null +++ b/nix/scripts/substitute-binary.in @@ -0,0 +1,11 @@ +#!@SHELL@ +# A shorthand for "guix substitute-binary", for use by the daemon. + +if test "x$GUIX_UNINSTALLED" = "x" +then + prefix="@prefix@" + exec_prefix="@exec_prefix@" + exec "@bindir@/guix" substitute-binary "$@" +else + exec guix substitute-binary "$@" +fi diff --git a/pre-inst-env.in b/pre-inst-env.in index 4e079c8d41..5e7758cd7c 100644 --- a/pre-inst-env.in +++ b/pre-inst-env.in @@ -35,8 +35,9 @@ export PATH # Daemon helpers. NIX_ROOT_FINDER="@abs_top_builddir@/nix/scripts/list-runtime-roots" +NIX_SUBSTITUTERS="@abs_top_builddir@/nix/scripts/substitute-binary" NIX_SETUID_HELPER="@abs_top_builddir@/nix-setuid-helper" -export NIX_ROOT_FINDER NIX_SETUID_HELPER +export NIX_ROOT_FINDER NIX_SETUID_HELPER NIX_SUBSTITUTERS # The following variables need only be defined when compiling Guix # modules, but we define them to be on the safe side in case of diff --git a/test-env.in b/test-env.in index 491a45c7b4..9a6257197c 100644 --- a/test-env.in +++ b/test-env.in @@ -1,7 +1,7 @@ #!/bin/sh # GNU Guix --- Functional package management for GNU -# Copyright © 2012 Ludovic Courtès +# Copyright © 2012, 2013 Ludovic Courtès # # This file is part of GNU Guix. # @@ -26,7 +26,6 @@ if [ -x "@abs_top_builddir@/guix-daemon" ] then - NIX_SUBSTITUTERS="" # don't resort to substituters NIX_SETUID_HELPER="@abs_top_builddir@/nix-setuid-helper" # normally unused NIX_IGNORE_SYMLINK_STORE=1 # in case the store is a symlink NIX_STORE_DIR="@GUIX_TEST_ROOT@/store" @@ -39,18 +38,24 @@ then # that the directory name must be chosen so that the socket's file # name is less than 108-char long (the size of `sun_path' in glibc). # Currently, in Nix builds, we're at ~106 chars... - NIX_STATE_DIR="@GUIX_TEST_ROOT@/var/$$" # allow for parallel tests + NIX_STATE_DIR="@GUIX_TEST_ROOT@/var/$$" - export NIX_SUBSTITUTERS NIX_IGNORE_SYMLINK_STORE NIX_STORE_DIR \ + # A place to store data of the substituter. + GUIX_BINARY_SUBSTITUTE_URL="file://$NIX_STATE_DIR/substituter-data" + rm -rf "$NIX_STATE_DIR/substituter-data" + mkdir -p "$NIX_STATE_DIR/substituter-data" + + export NIX_IGNORE_SYMLINK_STORE NIX_STORE_DIR \ NIX_LOCALSTATE_DIR NIX_LOG_DIR NIX_STATE_DIR NIX_DB_DIR \ - NIX_ROOT_FINDER NIX_SETUID_HELPER + NIX_ROOT_FINDER NIX_SETUID_HELPER GUIX_BINARY_SUBSTITUTE_URL # Do that because store.scm calls `canonicalize-path' on it. mkdir -p "$NIX_STORE_DIR" # Launch the daemon without chroot support because is may be # unavailable, for instance if we're not running as root. - "@abs_top_builddir@/guix-daemon" --disable-chroot & + "@abs_top_builddir@/pre-inst-env" \ + "@abs_top_builddir@/guix-daemon" --disable-chroot & daemon_pid=$! trap "kill $daemon_pid ; rm -rf $NIX_STATE_DIR" EXIT diff --git a/tests/store.scm b/tests/store.scm index d6e1aa54e3..c75b99c6a9 100644 --- a/tests/store.scm +++ b/tests/store.scm @@ -26,6 +26,7 @@ (define-module (test-store) #:use-module (gnu packages) #:use-module (gnu packages bootstrap) #:use-module (ice-9 match) + #:use-module (web uri) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-64)) @@ -128,6 +129,44 @@ (define (random-text) (null? (substitutable-paths s o)) (null? (substitutable-path-info s o))))) +(test-skip (if (getenv "GUIX_BINARY_SUBSTITUTE_URL") 0 1)) + +(test-assert "substitute query" + (let* ((s (open-connection)) + (d (package-derivation s %bootstrap-guile (%current-system))) + (o (derivation-path->output-path d)) + (dir (and=> (getenv "GUIX_BINARY_SUBSTITUTE_URL") + (compose uri-path string->uri)))) + ;; Create fake substituter data, to be read by `substitute-binary'. + (call-with-output-file (string-append dir "/nix-cache-info") + (lambda (p) + (format p "StoreDir: ~a\nWantMassQuery: 0\n" + (getenv "NIX_STORE_DIR")))) + (call-with-output-file (string-append dir "/" (store-path-hash-part o) + ".narinfo") + (lambda (p) + (format p "StorePath: ~a +URL: ~a +Compression: none +NarSize: 1234 +References: +System: ~a +Deriver: ~a~%" + o ; StorePath + (string-append dir "/example.nar") ; URL + (%current-system) ; System + (basename d)))) ; Deriver + + ;; Make sure `substitute-binary' correctly communicates the above data. + (set-build-options s #:use-substitutes? #t) + (and (has-substitutes? s o) + (equal? (list o) (substitutable-paths s (list o))) + (match (pk 'spi (substitutable-path-info s (list o))) + (((? substitutable? s)) + (and (equal? (substitutable-deriver s) d) + (null? (substitutable-references s)) + (equal? (substitutable-nar-size s) 1234))))))) + (test-end "store") -- cgit v1.2.3 From 0f41c26f9b9c981d5d5ecaa8c2ccda4f4c6ab147 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Thu, 4 Apr 2013 22:29:08 +0200 Subject: Add (guix nar) and (guix serialization). * guix/store.scm (write-int, read-int, write-long-long, read-long-long, write-padding, write-string, read-string, read-latin1-string, write-string-list, read-string-list, write-store-path, read-store-path, write-store-path-list, read-store-path-list): Move to serialization.scm. (write-contents, write-file): Move to nar.scm. * guix/nar.scm, guix/serialization.scm: New files. * Makefile.am (MODULES): Add them. --- Makefile.am | 2 + guix/nar.scm | 110 ++++++++++++++++++++++++++++++++++++ guix/serialization.scm | 114 +++++++++++++++++++++++++++++++++++++ guix/store.scm | 149 +------------------------------------------------ 4 files changed, 228 insertions(+), 147 deletions(-) create mode 100644 guix/nar.scm create mode 100644 guix/serialization.scm (limited to 'guix/store.scm') diff --git a/Makefile.am b/Makefile.am index 888302bd96..136c01bf3f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -34,6 +34,8 @@ MODULES = \ guix/scripts/substitute-binary.scm \ guix/base32.scm \ guix/utils.scm \ + guix/serialization.scm \ + guix/nar.scm \ guix/derivations.scm \ guix/download.scm \ guix/gnu-maintenance.scm \ diff --git a/guix/nar.scm b/guix/nar.scm new file mode 100644 index 0000000000..b42f03c514 --- /dev/null +++ b/guix/nar.scm @@ -0,0 +1,110 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2012, 2013 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (guix nar) + #:use-module (guix utils) + #:use-module (guix serialization) + #:use-module (rnrs bytevectors) + #:use-module (rnrs io ports) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) + #:use-module (ice-9 ftw) + #:export (write-file)) + +;;; Comment: +;;; +;;; Read and write Nix archives, aka. ‘nar’. +;;; +;;; Code: + +(define (write-contents file p size) + "Write SIZE bytes from FILE to output port P." + (define (call-with-binary-input-file file proc) + ;; Open FILE as a binary file. This avoids scan-for-encoding, and thus + ;; avoids any initial buffering. Disable file name canonicalization to + ;; avoid stat'ing like crazy. + (with-fluids ((%file-port-name-canonicalization #f)) + (let ((port (open-file file "rb"))) + (catch #t (cut proc port) + (lambda args + (close-port port) + (apply throw args)))))) + + (define (dump in size) + (define buf-size 65536) + (define buf (make-bytevector buf-size)) + + (let loop ((left size)) + (if (<= left 0) + 0 + (let ((read (get-bytevector-n! in buf 0 buf-size))) + (if (eof-object? read) + left + (begin + (put-bytevector p buf 0 read) + (loop (- left read)))))))) + + (write-string "contents" p) + (write-long-long size p) + (call-with-binary-input-file file + ;; Use `sendfile' when available (Guile 2.0.8+). + (if (compile-time-value (defined? 'sendfile)) + (cut sendfile p <> size 0) + (cut dump <> size))) + (write-padding size p)) + +(define (write-file file port) + "Write the contents of FILE to PORT in Nar format, recursing into +sub-directories of FILE as needed." + (define %archive-version-1 "nix-archive-1") + (define p port) + + (write-string %archive-version-1 p) + + (let dump ((f file)) + (let ((s (lstat f))) + (write-string "(" p) + (case (stat:type s) + ((regular) + (write-string "type" p) + (write-string "regular" p) + (if (not (zero? (logand (stat:mode s) #o100))) + (begin + (write-string "executable" p) + (write-string "" p))) + (write-contents f p (stat:size s))) + ((directory) + (write-string "type" p) + (write-string "directory" p) + (let ((entries (remove (cut member <> '("." "..")) + (scandir f)))) + (for-each (lambda (e) + (let ((f (string-append f "/" e))) + (write-string "entry" p) + (write-string "(" p) + (write-string "name" p) + (write-string e p) + (write-string "node" p) + (dump f) + (write-string ")" p))) + entries))) + (else + (error "ENOSYS"))) + (write-string ")" p)))) + +;;; nar.scm ends here diff --git a/guix/serialization.scm b/guix/serialization.scm new file mode 100644 index 0000000000..474dc69de5 --- /dev/null +++ b/guix/serialization.scm @@ -0,0 +1,114 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2012, 2013 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (guix serialization) + #:use-module (guix utils) + #:use-module (rnrs bytevectors) + #:use-module (rnrs io ports) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) + #:export (write-int read-int + write-long-long read-long-long + write-padding + write-string read-string read-latin1-string + write-string-list read-string-list + write-store-path read-store-path + write-store-path-list read-store-path-list)) + +;;; Comment: +;;; +;;; Serialization procedures used by the RPCs and the Nar format. This module +;;; is for internal consumption. +;;; +;;; Code: + +;; Similar to serialize.cc in Nix. + +(define (write-int n p) + (let ((b (make-bytevector 8 0))) + (bytevector-u32-set! b 0 n (endianness little)) + (put-bytevector p b))) + +(define (read-int p) + (let ((b (get-bytevector-n p 8))) + (bytevector-u32-ref b 0 (endianness little)))) + +(define (write-long-long n p) + (let ((b (make-bytevector 8 0))) + (bytevector-u64-set! b 0 n (endianness little)) + (put-bytevector p b))) + +(define (read-long-long p) + (let ((b (get-bytevector-n p 8))) + (bytevector-u64-ref b 0 (endianness little)))) + +(define write-padding + (let ((zero (make-bytevector 8 0))) + (lambda (n p) + (let ((m (modulo n 8))) + (or (zero? m) + (put-bytevector p zero 0 (- 8 m))))))) + +(define (write-string s p) + (let* ((s (string->utf8 s)) + (l (bytevector-length s)) + (m (modulo l 8)) + (b (make-bytevector (+ 8 l (if (zero? m) 0 (- 8 m)))))) + (bytevector-u32-set! b 0 l (endianness little)) + (bytevector-copy! s 0 b 8 l) + (put-bytevector p b))) + +(define (read-string p) + (let* ((len (read-int p)) + (m (modulo len 8)) + (bv (get-bytevector-n p len)) + (str (utf8->string bv))) + (or (zero? m) + (get-bytevector-n p (- 8 m))) + str)) + +(define (read-latin1-string p) + (let* ((len (read-int p)) + (m (modulo len 8)) + (str (get-string-n p len))) + (or (zero? m) + (get-bytevector-n p (- 8 m))) + str)) + +(define (write-string-list l p) + (write-int (length l) p) + (for-each (cut write-string <> p) l)) + +(define (read-string-list p) + (let ((len (read-int p))) + (unfold (cut >= <> len) + (lambda (i) + (read-string p)) + 1+ + 0))) + +(define (write-store-path f p) + (write-string f p)) ; TODO: assert path + +(define (read-store-path p) + (read-string p)) ; TODO: assert path + +(define write-store-path-list write-string-list) +(define read-store-path-list read-string-list) + +;;; serialization.scm ends here diff --git a/guix/store.scm b/guix/store.scm index de9785c835..cc21af84e4 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -17,8 +17,10 @@ ;;; along with GNU Guix. If not, see . (define-module (guix store) + #:use-module (guix nar) #:use-module (guix utils) #:use-module (guix config) + #:use-module (guix serialization) #:use-module (rnrs bytevectors) #:use-module (rnrs io ports) #:use-module (srfi srfi-1) @@ -29,7 +31,6 @@ (define-module (guix store) #:use-module (srfi srfi-39) #:use-module (ice-9 match) #:use-module (ice-9 rdelim) - #:use-module (ice-9 ftw) #:use-module (ice-9 regex) #:export (%daemon-socket-file @@ -161,152 +162,6 @@ (define %daemon-socket-file -;; serialize.cc - -(define (write-int n p) - (let ((b (make-bytevector 8 0))) - (bytevector-u32-set! b 0 n (endianness little)) - (put-bytevector p b))) - -(define (read-int p) - (let ((b (get-bytevector-n p 8))) - (bytevector-u32-ref b 0 (endianness little)))) - -(define (write-long-long n p) - (let ((b (make-bytevector 8 0))) - (bytevector-u64-set! b 0 n (endianness little)) - (put-bytevector p b))) - -(define (read-long-long p) - (let ((b (get-bytevector-n p 8))) - (bytevector-u64-ref b 0 (endianness little)))) - -(define write-padding - (let ((zero (make-bytevector 8 0))) - (lambda (n p) - (let ((m (modulo n 8))) - (or (zero? m) - (put-bytevector p zero 0 (- 8 m))))))) - -(define (write-string s p) - (let* ((s (string->utf8 s)) - (l (bytevector-length s)) - (m (modulo l 8)) - (b (make-bytevector (+ 8 l (if (zero? m) 0 (- 8 m)))))) - (bytevector-u32-set! b 0 l (endianness little)) - (bytevector-copy! s 0 b 8 l) - (put-bytevector p b))) - -(define (read-string p) - (let* ((len (read-int p)) - (m (modulo len 8)) - (bv (get-bytevector-n p len)) - (str (utf8->string bv))) - (or (zero? m) - (get-bytevector-n p (- 8 m))) - str)) - -(define (read-latin1-string p) - (let* ((len (read-int p)) - (m (modulo len 8)) - (str (get-string-n p len))) - (or (zero? m) - (get-bytevector-n p (- 8 m))) - str)) - -(define (write-string-list l p) - (write-int (length l) p) - (for-each (cut write-string <> p) l)) - -(define (read-string-list p) - (let ((len (read-int p))) - (unfold (cut >= <> len) - (lambda (i) - (read-string p)) - 1+ - 0))) - -(define (write-store-path f p) - (write-string f p)) ; TODO: assert path - -(define (read-store-path p) - (read-string p)) ; TODO: assert path - -(define write-store-path-list write-string-list) -(define read-store-path-list read-string-list) - -(define (write-contents file p size) - "Write SIZE bytes from FILE to output port P." - (define (call-with-binary-input-file file proc) - ;; Open FILE as a binary file. This avoids scan-for-encoding, and thus - ;; avoids any initial buffering. Disable file name canonicalization to - ;; avoid stat'ing like crazy. - (with-fluids ((%file-port-name-canonicalization #f)) - (let ((port (open-file file "rb"))) - (catch #t (cut proc port) - (lambda args - (close-port port) - (apply throw args)))))) - - (define (dump in size) - (define buf-size 65536) - (define buf (make-bytevector buf-size)) - - (let loop ((left size)) - (if (<= left 0) - 0 - (let ((read (get-bytevector-n! in buf 0 buf-size))) - (if (eof-object? read) - left - (begin - (put-bytevector p buf 0 read) - (loop (- left read)))))))) - - (write-string "contents" p) - (write-long-long size p) - (call-with-binary-input-file file - ;; Use `sendfile' when available (Guile 2.0.8+). - (if (compile-time-value (defined? 'sendfile)) - (cut sendfile p <> size 0) - (cut dump <> size))) - (write-padding size p)) - -(define (write-file f p) - (define %archive-version-1 "nix-archive-1") - - (write-string %archive-version-1 p) - - (let dump ((f f)) - (let ((s (lstat f))) - (write-string "(" p) - (case (stat:type s) - ((regular) - (write-string "type" p) - (write-string "regular" p) - (if (not (zero? (logand (stat:mode s) #o100))) - (begin - (write-string "executable" p) - (write-string "" p))) - (write-contents f p (stat:size s))) - ((directory) - (write-string "type" p) - (write-string "directory" p) - (let ((entries (remove (cut member <> '("." "..")) - (scandir f)))) - (for-each (lambda (e) - (let ((f (string-append f "/" e))) - (write-string "entry" p) - (write-string "(" p) - (write-string "name" p) - (write-string e p) - (write-string "node" p) - (dump f) - (write-string ")" p))) - entries))) - (else - (error "ENOSYS"))) - (write-string ")" p)))) - ;; Information about a substitutable store path. (define-record-type (substitutable path deriver refs dl-size nar-size) -- cgit v1.2.3 From 9adb6934f434436523e6d736cd1b55950d690b8d Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Fri, 12 Apr 2013 18:22:41 +0200 Subject: store: Remove unneeded and conflicting import. * guix/store.scm: Remove unneeded (ice-9 rdelim) import. In Guile 2.0.9 that module exports `read-string', which conflicts with that of (guix serialization). --- guix/store.scm | 1 - 1 file changed, 1 deletion(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index cc21af84e4..b1b60babf0 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -30,7 +30,6 @@ (define-module (guix store) #:use-module (srfi srfi-35) #:use-module (srfi srfi-39) #:use-module (ice-9 match) - #:use-module (ice-9 rdelim) #:use-module (ice-9 regex) #:export (%daemon-socket-file -- cgit v1.2.3 From 0ff3e3aa9bea8c82c921db88fc03cb7361b886f7 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Sat, 20 Apr 2013 10:53:31 +0200 Subject: daemon: Gracefully handle cases where the daemon does not return a status code. * guix/store.scm (process-stderr): Check for EOF before doing (read-int p). --- guix/store.scm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'guix/store.scm') diff --git a/guix/store.scm b/guix/store.scm index b1b60babf0..b82588b2a0 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -336,7 +336,10 @@ (define %stderr-error #x63787470) #f)) ((= k %stderr-error) (let ((error (read-latin1-string p)) - (status (if (>= (nix-server-minor-version server) 8) + ;; Currently the daemon fails to send a status code for early + ;; errors like DB schema version mismatches, so check for EOF. + (status (if (and (>= (nix-server-minor-version server) 8) + (not (eof-object? (lookahead-u8 p)))) (read-int p) 1))) (raise (condition (&nix-protocol-error -- cgit v1.2.3