summaryrefslogtreecommitdiff
path: root/guix/import
diff options
context:
space:
mode:
authorSarah Morgensen <iskarian@mgsn.dev>2021-07-16 21:01:28 -0700
committerMaxim Cournoyer <maxim.cournoyer@gmail.com>2021-07-18 01:57:17 -0400
commit793ba333c6039545684e8af7e384704e76bc0f2f (patch)
tree2705d7a6da51a129ff441820199a485b1cce852b /guix/import
parent3217a04b0352c2dd13323257b369604eeabfccc3 (diff)
import: go: Upgrade go.mod parser.
Upgrade the go.mod parser to handle the full go.mod spec, and to gracefully handle unexpected/malformed syntax. Restructure parser usage, making the parse tree available for other uses. guix/import/go.scm (parse-go.mod): Parse using (ice-9 peg) instead of regex matching for more robustness. Return a list of directives. (go.mod-directives): New procedure. (go.mod-requirements): Likewise. (go-module->guix-package): Use it. (%go.mod-replace-directive-rx): Remove unused variable. tests/go.scm (testing-parse-mod): Adjust accordingly. (go.mod-requirements) (fixture-go-mod-unparseable) (fixture-go-mod-retract) (fixture-go-mod-strings): New variables. ("parse-go.mod: simple") ("parse-go.mod: comments and unparseable lines") ("parse-go.mod: retract") ("parse-go.mod: raw strings and quoted strings") ("parse-go.mod: complete"): New tests. Signed-off-by: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Diffstat (limited to 'guix/import')
-rw-r--r--guix/import/go.scm249
1 files changed, 130 insertions, 119 deletions
diff --git a/guix/import/go.scm b/guix/import/go.scm
index 24d12acd97..a6e2af215f 100644
--- a/guix/import/go.scm
+++ b/guix/import/go.scm
@@ -5,7 +5,7 @@
;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com>
;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org>
;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz>
-;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev>
+;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -41,6 +41,7 @@
#:autoload (guix base32) (bytevector->nix-base32-string)
#:autoload (guix build utils) (mkdir-p)
#:use-module (ice-9 match)
+ #:use-module (ice-9 peg)
#:use-module (ice-9 rdelim)
#:use-module (ice-9 receive)
#:use-module (ice-9 regex)
@@ -244,129 +245,139 @@ and VERSION and return an input port."
(go-path-escape version))))
(http-fetch* url)))
-(define %go.mod-require-directive-rx
- ;; A line in a require directive is composed of a module path and
- ;; a version separated by whitespace and an optionnal '//' comment at
- ;; the end.
- (make-regexp
- (string-append
- "^[[:blank:]]*([^[:blank:]]+)[[:blank:]]+" ;the module path
- "([^[:blank:]]+)" ;the version
- "([[:blank:]]+//.*)?"))) ;an optional comment
-
-(define %go.mod-replace-directive-rx
- ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
- ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
- (make-regexp
- (string-append
- "([^[:blank:]]+)" ;the module path
- "([[:blank:]]+([^[:blank:]]+))?" ;optional version
- "[[:blank:]]+=>[[:blank:]]+"
- "([^[:blank:]]+)" ;the file or module path
- "([[:blank:]]+([^[:blank:]]+))?"))) ;the version (if a module path)
(define (parse-go.mod content)
- "Parse the go.mod file CONTENT, returning a list of requirements."
- ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar
- ;; which we think necessary for our use case.
- (define (toplevel requirements replaced)
- "This is the main parser. The results are accumulated in THE REQUIREMENTS
-and REPLACED lists."
- (let ((line (read-line)))
- (cond
- ((eof-object? line)
- ;; parsing ended, give back the result
- (values requirements replaced))
- ((string=? line "require (")
- ;; a require block begins, delegate parsing to IN-REQUIRE
- (in-require requirements replaced))
- ((string=? line "replace (")
- ;; a replace block begins, delegate parsing to IN-REPLACE
- (in-replace requirements replaced))
- ((string-prefix? "require " line)
- ;; a require directive by itself
- (let* ((stripped-line (string-drop line 8)))
- (call-with-values
- (lambda ()
- (require-directive requirements replaced stripped-line))
- toplevel)))
- ((string-prefix? "replace " line)
- ;; a replace directive by itself
- (let* ((stripped-line (string-drop line 8)))
- (call-with-values
- (lambda ()
- (replace-directive requirements replaced stripped-line))
- toplevel)))
- (#t
- ;; unrecognised line, ignore silently
- (toplevel requirements replaced)))))
-
- (define (in-require requirements replaced)
- (let ((line (read-line)))
- (cond
- ((eof-object? line)
- ;; this should never happen here but we ignore silently
- (values requirements replaced))
- ((string=? line ")")
- ;; end of block, coming back to toplevel
- (toplevel requirements replaced))
- (#t
- (call-with-values (lambda ()
- (require-directive requirements replaced line))
- in-require)))))
-
- (define (in-replace requirements replaced)
- (let ((line (read-line)))
- (cond
- ((eof-object? line)
- ;; this should never happen here but we ignore silently
- (values requirements replaced))
- ((string=? line ")")
- ;; end of block, coming back to toplevel
- (toplevel requirements replaced))
- (#t
- (call-with-values (lambda ()
- (replace-directive requirements replaced line))
- in-replace)))))
-
- (define (replace-directive requirements replaced line)
- "Extract replaced modules and new requirements from the replace directive
-in LINE and add them to the REQUIREMENTS and REPLACED lists."
- (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line))
- (module-path (match:substring rx-match 1))
- (version (match:substring rx-match 3))
- (new-module-path (match:substring rx-match 4))
- (new-version (match:substring rx-match 6))
- (new-replaced (cons (list module-path version) replaced))
- (new-requirements
- (if (string-match "^\\.?\\./" new-module-path)
- requirements
- (cons (list new-module-path new-version) requirements))))
- (values new-requirements new-replaced)))
-
- (define (require-directive requirements replaced line)
- "Extract requirement from LINE and augment the REQUIREMENTS and REPLACED
-lists."
- (let* ((rx-match (regexp-exec %go.mod-require-directive-rx line))
- (module-path (match:substring rx-match 1))
- ;; Double-quoted strings were seen in the wild without escape
- ;; sequences; trim the quotes to be on the safe side.
- (module-path (string-trim-both module-path #\"))
- (version (match:substring rx-match 2)))
- (values (cons (list module-path version) requirements) replaced)))
-
- (with-input-from-string content
- (lambda ()
- (receive (requirements replaced)
- (toplevel '() '())
- ;; At last remove the replaced modules from the requirements list.
- (remove (lambda (r)
- (assoc (car r) replaced))
- requirements)))))
+ "Parse the go.mod file CONTENT, returning a list of directives, comments,
+and unknown lines. Each sublist begins with a symbol (go, module, require,
+replace, exclude, retract, comment, or unknown) and is followed by one or more
+sublists. Each sublist begins with a symbol (module-path, version, file-path,
+comment, or unknown) and is followed by the indicated data."
+ ;; https://golang.org/ref/mod#go-mod-file-grammar
+ (define-peg-pattern NL none "\n")
+ (define-peg-pattern WS none (or " " "\t" "\r"))
+ (define-peg-pattern => none (and (* WS) "=>"))
+ (define-peg-pattern punctuation none (or "," "=>" "[" "]" "(" ")"))
+ (define-peg-pattern comment all
+ (and (ignore "//") (* WS) (* (and (not-followed-by NL) peg-any))))
+ (define-peg-pattern EOL body (and (* WS) (? comment) NL))
+ (define-peg-pattern block-start none (and (* WS) "(" EOL))
+ (define-peg-pattern block-end none (and (* WS) ")" EOL))
+ (define-peg-pattern any-line body
+ (and (* WS) (* (and (not-followed-by NL) peg-any)) EOL))
+
+ ;; Strings and identifiers
+ (define-peg-pattern identifier body
+ (+ (and (not-followed-by (or NL WS punctuation)) peg-any)))
+ (define-peg-pattern string-raw body
+ (and (ignore "`") (+ (and (not-followed-by "`") peg-any)) (ignore "`")))
+ (define-peg-pattern string-quoted body
+ (and (ignore "\"")
+ (+ (or (and (ignore "\\") peg-any)
+ (and (not-followed-by "\"") peg-any)))
+ (ignore "\"")))
+ (define-peg-pattern string-or-ident body
+ (and (* WS) (or string-raw string-quoted identifier)))
+
+ (define-peg-pattern version all string-or-ident)
+ (define-peg-pattern module-path all string-or-ident)
+ (define-peg-pattern file-path all string-or-ident)
+
+ ;; Non-directive lines
+ (define-peg-pattern unknown all any-line)
+ (define-peg-pattern block-line body
+ (or EOL (and (not-followed-by block-end) unknown)))
+
+ ;; GoDirective = "go" GoVersion newline .
+ (define-peg-pattern go all (and (ignore "go") version EOL))
+
+ ;; ModuleDirective = "module" ( ModulePath | "(" newline ModulePath newline ")" ) newline .
+ (define-peg-pattern module all
+ (and (ignore "module") (or (and block-start module-path EOL block-end)
+ (and module-path EOL))))
+
+ ;; The following directives may all be used solo or in a block
+ ;; RequireSpec = ModulePath Version newline .
+ (define-peg-pattern require all (and module-path version EOL))
+ (define-peg-pattern require-top body
+ (and (ignore "require")
+ (or (and block-start (* (or require block-line)) block-end) require)))
+
+ ;; ExcludeSpec = ModulePath Version newline .
+ (define-peg-pattern exclude all (and module-path version EOL))
+ (define-peg-pattern exclude-top body
+ (and (ignore "exclude")
+ (or (and block-start (* (or exclude block-line)) block-end) exclude)))
+
+ ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline
+ ;; | ModulePath [ Version ] "=>" ModulePath Version newline .
+ (define-peg-pattern original all (or (and module-path version) module-path))
+ (define-peg-pattern with all (or (and module-path version) file-path))
+ (define-peg-pattern replace all (and original => with EOL))
+ (define-peg-pattern replace-top body
+ (and (ignore "replace")
+ (or (and block-start (* (or replace block-line)) block-end) replace)))
+
+ ;; RetractSpec = ( Version | "[" Version "," Version "]" ) newline .
+ (define-peg-pattern range all
+ (and (* WS) (ignore "[") version
+ (* WS) (ignore ",") version (* WS) (ignore "]")))
+ (define-peg-pattern retract all (and (or range version) EOL))
+ (define-peg-pattern retract-top body
+ (and (ignore "retract")
+ (or (and block-start (* (or retract block-line)) block-end) retract)))
+
+ (define-peg-pattern go-mod body
+ (* (and (* WS) (or go module require-top exclude-top replace-top
+ retract-top EOL unknown))))
+
+ (let ((tree (peg:tree (match-pattern go-mod content)))
+ (keywords '(go module require replace exclude retract comment unknown)))
+ (keyword-flatten keywords tree)))
;; Prevent inlining of this procedure, which is accessed by unit tests.
(set! parse-go.mod parse-go.mod)
+(define (go.mod-directives go.mod directive)
+ "Return the list of top-level directive bodies in GO.MOD matching the symbol
+DIRECTIVE."
+ (filter-map (match-lambda
+ (((? (cut eq? <> directive) head) . rest) rest)
+ (_ #f))
+ go.mod))
+
+(define (go.mod-requirements go.mod)
+ "Compute and return the list of requirements specified by GO.MOD."
+ (define (replace directive requirements)
+ (define (maybe-replace module-path new-requirement)
+ ;; Do not allow version updates for indirect dependencies (see:
+ ;; https://golang.org/ref/mod#go-mod-file-replace).
+ (if (and (equal? module-path (first new-requirement))
+ (not (assoc-ref requirements module-path)))
+ requirements
+ (cons new-requirement (alist-delete module-path requirements))))
+
+ (match directive
+ ((('original ('module-path module-path) . _) with . _)
+ (match with
+ (('with ('file-path _) . _)
+ (alist-delete module-path requirements))
+ (('with ('module-path new-module-path) ('version new-version) . _)
+ (maybe-replace module-path
+ (list new-module-path new-version)))))))
+
+ (define (require directive requirements)
+ (match directive
+ ((('module-path module-path) ('version version) . _)
+ (cons (list module-path version) requirements))))
+
+ (let* ((requires (go.mod-directives go.mod 'require))
+ (replaces (go.mod-directives go.mod 'replace))
+ (requirements (fold require '() requires)))
+ (fold replace requirements replaces)))
+
+;; Prevent inlining of this procedure, which is accessed by unit tests.
+(set! go.mod-requirements go.mod-requirements)
+
(define-record-type <vcs>
(%make-vcs url-prefix root-regex type)
vcs?
@@ -592,7 +603,7 @@ When VERSION is unspecified, the latest version available is used."
hint: use one of the following available versions ~a\n"
version* available-versions))))
(content (fetch-go.mod goproxy module-path version*))
- (dependencies+versions (parse-go.mod content))
+ (dependencies+versions (go.mod-requirements (parse-go.mod content)))
(dependencies (if pin-versions?
dependencies+versions
(map car dependencies+versions)))