diff options
| author | Maxim Cournoyer <maxim.cournoyer@gmail.com> | 2021-10-01 17:10:49 -0400 |
|---|---|---|
| committer | Maxim Cournoyer <maxim.cournoyer@gmail.com> | 2021-10-01 17:10:49 -0400 |
| commit | 2e65e4834a226c570866f2e8976ed7f252b45cd1 (patch) | |
| tree | 21d625bce8d03627680214df4a6622bf8eb79dc9 /guix | |
| parent | 9c68ecb24dd1660ce736cdcdea0422a73ec318a2 (diff) | |
| parent | f1a3c11407b52004e523ec5de20d326c5661681f (diff) | |
Merge remote-tracking branch 'origin/master' into staging
With resolved conflicts in:
gnu/packages/bittorrent.scm
gnu/packages/databases.scm
gnu/packages/geo.scm
gnu/packages/gnupg.scm
gnu/packages/gstreamer.scm
gnu/packages/gtk.scm
gnu/packages/linux.scm
gnu/packages/python-xyz.scm
gnu/packages/xorg.scm
guix/build/qt-utils.scm
Diffstat (limited to 'guix')
121 files changed, 6729 insertions, 2495 deletions
diff --git a/guix/android-repo-download.scm b/guix/android-repo-download.scm index 5ff3e7edd4..1c3502e673 100644 --- a/guix/android-repo-download.scm +++ b/guix/android-repo-download.scm @@ -78,6 +78,9 @@ generic name if unset." (define zlib (module-ref (resolve-interface '(gnu packages compression)) 'zlib)) + (define guile-json + (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + (define gnutls (module-ref (resolve-interface '(gnu packages tls)) 'gnutls)) @@ -99,7 +102,7 @@ generic name if unset." (define build (with-imported-modules modules - (with-extensions (list gnutls) + (with-extensions (list gnutls guile-json) ;for (guix swh) #~(begin (use-modules (guix build android-repo) (guix build utils) diff --git a/guix/base16.scm b/guix/base16.scm index 6c15a9f588..9ac964dff0 100644 --- a/guix/base16.scm +++ b/guix/base16.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2012, 2014, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> ;;; ;;; This file is part of GNU Guix. ;;; @@ -32,27 +33,28 @@ (define (bytevector->base16-string bv) "Return the hexadecimal representation of BV's contents." - (define len - (bytevector-length bv)) - - (let-syntax ((base16-chars (lambda (s) - (syntax-case s () - (_ - (let ((v (list->vector - (unfold (cut > <> 255) - (lambda (n) - (format #f "~2,'0x" n)) - 1+ - 0)))) - v)))))) - (define chars base16-chars) - (let loop ((i len) - (r '())) - (if (zero? i) - (string-concatenate r) - (let ((i (- i 1))) - (loop i - (cons (vector-ref chars (bytevector-u8-ref bv i)) r))))))) + (define len (bytevector-length bv)) + (define utf8 (make-bytevector (* len 2))) + (let-syntax ((base16-octet-pairs + (lambda (s) + (syntax-case s () + (_ + (string->utf8 + (string-concatenate + (unfold (cut > <> 255) + (lambda (n) + (format #f "~2,'0x" n)) + 1+ + 0)))))))) + (define octet-pairs base16-octet-pairs) + (let loop ((i 0)) + (when (< i len) + (bytevector-u16-native-set! + utf8 (* 2 i) + (bytevector-u16-native-ref octet-pairs + (* 2 (bytevector-u8-ref bv i)))) + (loop (+ i 1)))) + (utf8->string utf8))) (define base16-string->bytevector (let ((chars->value (fold (lambda (i r) diff --git a/guix/base32.scm b/guix/base32.scm index 49f191ba26..8f097d4e77 100644 --- a/guix/base32.scm +++ b/guix/base32.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2015, 2017 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2015, 2017, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -42,6 +42,27 @@ ;;; ;;; Code: +(define-syntax bit-field + (lambda (s) + ;; This inline version of 'bit-field' assumes that START and END are + ;; literals and pre-computes the mask. In an ideal world, using 'define' + ;; or 'define-inlinable' would be enough, but as of 3.0.7, peval doesn't + ;; expand calls to 'expt' (and 'bit-field' is a subr.) + (syntax-case s () + ((_ n start end) + (let* ((s (syntax->datum #'start)) + (e (syntax->datum #'end)) + (mask (- (expt 2 (- e s)) 1))) + ;; The baseline compiler in Guile <= 3.0.7 miscompiles (ash x N) as + ;; (ash x (- N)) when N is a literal: <https://bugs.gnu.org/50696>. + ;; Here we take advantage of another bug in the baseline compiler, + ;; fixed in Guile commit 330c6ea83f492672578b62d0683acbb532d1a5d9: we + ;; introduce 'minus-start' such that it has a different source + ;; location, which in turn means that the baseline compiler pattern + ;; for (ash x N) doesn't match, thus avoiding the bug (!). + (with-syntax ((minus-start (datum->syntax #'start (- s)))) + #`(logand (ash n minus-start) #,mask))))))) + (define bytevector-quintet-ref (let* ((ref bytevector-u8-ref) (ref+ (lambda (bv offset) diff --git a/guix/build-system/asdf.scm b/guix/build-system/asdf.scm index 28403a1960..b4e40ee8c2 100644 --- a/guix/build-system/asdf.scm +++ b/guix/build-system/asdf.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016, 2017 Andy Patterson <ajpatter@uwaterloo.ca> -;;; Copyright © 2019, 2020 Guillaume Le Vaillant <glv@posteo.net> +;;; Copyright © 2019, 2020, 2021 Guillaume Le Vaillant <glv@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -291,16 +291,16 @@ set up using CL source package conventions." (imported-modules %asdf-build-system-modules) (modules %asdf-build-modules)) - ;; FIXME: The definition of 'systems' is pretty hacky. - ;; Is there a more elegant way to do it? (define systems (if (null? (cadr asd-systems)) - `(quote - ,(list - (string-drop - ;; NAME is the value returned from `package-full-name'. - (hyphen-separated-name->name+version name) - (1+ (string-length lisp-type))))) ; drop the "<lisp>-" prefix. + ;; FIXME: Find a more reliable way to get the main system name. + (let* ((lisp-prefix (string-append lisp-type "-")) + (package-name (hyphen-separated-name->name+version + (if (string-prefix? lisp-prefix name) + (string-drop name + (string-length lisp-prefix)) + name)))) + `(quote ,(list package-name))) asd-systems)) (define builder diff --git a/guix/build-system/chicken.scm b/guix/build-system/chicken.scm index 9abae0431a..10f1469e88 100644 --- a/guix/build-system/chicken.scm +++ b/guix/build-system/chicken.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2020 raingloom <raingloom@riseup.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -26,7 +27,14 @@ #:use-module (ice-9 match) #:export (%chicken-build-system-modules chicken-build - chicken-build-system)) + chicken-build-system + egg-uri)) + +(define* (egg-uri name version #:optional (extension ".tar.gz")) + "Return a URI string for the CHICKEN egg corresponding to NAME and VERSION. +EXTENSION is the file name extension, such as '.tar.gz'." + (string-append "https://code.call-cc.org/egg-tarballs/5/" + name "/" name "-" version extension)) (define %chicken-build-system-modules ;; Build-side modules imported and used by default. diff --git a/guix/build-system/dune.scm b/guix/build-system/dune.scm index 6a2f3d16de..5b33ef6841 100644 --- a/guix/build-system/dune.scm +++ b/guix/build-system/dune.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016, 2017, 2018 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2017 Ben Woodcroft <donttrustben@gmail.com> +;;; Copyright © 2021 pukkamustard <pukkamustard@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -59,6 +60,17 @@ #:allow-other-keys #:rest arguments) "Return a bag for NAME." + + ;; Flags that put dune into reproducible build mode. + (define dune-release-flags + (if (version>=? (package-version dune) "2.5.0") + ;; For dune >= 2.5.0 this is just --release. + ''("--release") + ;; --release does not exist before 2.5.0. Replace with flags compatible + ;; with our old ocaml4.07-dune (1.11.3) + ''("--root" "." "--ignore-promoted-rules" "--no-config" + "--profile" "release"))) + (define private-keywords '(#:source #:target #:dune #:findlib #:ocaml #:inputs #:native-inputs)) @@ -78,7 +90,9 @@ (build-inputs `(("dune" ,dune) ,@(bag-build-inputs base))) (build dune-build) - (arguments (strip-keyword-arguments private-keywords arguments)))))) + (arguments (append + `(#:dune-release-flags ,dune-release-flags) + (strip-keyword-arguments private-keywords arguments))))))) (define* (dune-build store name inputs #:key (guile #f) @@ -88,6 +102,7 @@ (out-of-source? #t) (jbuild? #f) (package #f) + (dune-release-flags ''()) (tests? #t) (test-flags ''()) (test-target "test") @@ -127,6 +142,7 @@ provides a 'setup.ml' file as its build system." #:out-of-source? ,out-of-source? #:jbuild? ,jbuild? #:package ,package + #:dune-release-flags ,dune-release-flags #:tests? ,tests? #:test-target ,test-target #:install-target ,install-target diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm index 8f55796e86..4c1a732107 100644 --- a/guix/build-system/go.scm +++ b/guix/build-system/go.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2016 Petter <petter@mykolab.ch> ;;; Copyright © 2017 Leo Famulari <leo@famulari.name> ;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net> +;;; Copyright © 2021 Efraim Flashner <efraim@flashner.co.il> ;;; ;;; This file is part of GNU Guix. ;;; @@ -27,6 +28,7 @@ #:use-module (guix packages) #:use-module (ice-9 match) #:use-module (ice-9 regex) + #:use-module (srfi srfi-1) #:export (%go-build-system-modules go-build go-build-system @@ -78,6 +80,24 @@ present) if a pseudo-version pattern is not recognized." commit hash and its date rather than a proper release tag." (regexp-exec %go-pseudo-version-rx version)) +(define (go-target target) + ;; Parse the nix-system equivalent of the target and set the + ;; target for compilation accordingly. + (match (string-split (gnu-triplet->nix-system target) #\-) + ((arch os) + (list (match arch + ("aarch64" "arm64") + ("armhf" "arm") + ("powerpc64le" "ppc64le") + ("powerpc64" "ppc64") + ("i686" "386") + ("x86_64" "amd64") + ("mips64el" "mips64le") + (_ arch)) + (match os + ((or "mingw32" "cygwin") "windows") + (_ os)))))) + (define %go-build-system-modules ;; Build-side modules imported and used by default. `((guix build go-build-system) @@ -98,22 +118,37 @@ commit hash and its date rather than a proper release tag." (define private-keywords '(#:source #:target #:go #:inputs #:native-inputs)) - (and (not target) ;XXX: no cross-compilation - (bag - (name name) - (system system) - (host-inputs `(,@(if source - `(("source" ,source)) - '()) - ,@inputs + (bag + (name name) + (system system) + (target target) + (build-inputs `(,@(if source + `(("source" ,source)) + '()) + ,@`(("go" ,go)) + ,@native-inputs + ,@(if target '() inputs) + ,@(if target + ;; Use the standard cross inputs of + ;; 'gnu-build-system'. + (standard-cross-packages target 'host) + '()) + ;; Keep the standard inputs of 'gnu-build-system'. + ,@(standard-packages))) + (host-inputs (if target inputs '())) + + ;; The cross-libc is really a target package, but for bootstrapping + ;; reasons, we can't put it in 'host-inputs'. Namely, 'cross-gcc' is a + ;; native package, so it would end up using a "native" variant of + ;; 'cross-libc' (built with 'gnu-build'), whereas all the other packages + ;; would use a target variant (built with 'gnu-cross-build'.) + (target-inputs (if target + (standard-cross-packages target 'target) + '())) - ;; Keep the standard inputs of 'gnu-build-system'. - ,@(standard-packages))) - (build-inputs `(("go" ,go) - ,@native-inputs)) - (outputs outputs) - (build go-build) - (arguments (strip-keyword-arguments private-keywords arguments))))) + (outputs outputs) + (build (if target go-cross-build go-build)) + (arguments (strip-keyword-arguments private-keywords arguments)))) (define* (go-build store name inputs #:key @@ -128,6 +163,8 @@ commit hash and its date rather than a proper release tag." (tests? #t) (allow-go-reference? #f) (system (%current-system)) + (goarch (first (go-target (%current-system)))) + (goos (last (go-target (%current-system)))) (guile #f) (imported-modules %go-build-system-modules) (modules '((guix build go-build-system) @@ -147,6 +184,8 @@ commit hash and its date rather than a proper release tag." #:system ,system #:phases ,phases #:outputs %outputs + #:goarch ,goarch + #:goos ,goos #:search-paths ',(map search-path-specification->sexp search-paths) #:install-source? ,install-source? @@ -174,6 +213,98 @@ commit hash and its date rather than a proper release tag." #:outputs outputs #:guile-for-build guile-for-build)) +(define* (go-cross-build store name + #:key + target native-drvs target-drvs + (phases '(@ (guix build go-build-system) + %standard-phases)) + (outputs '("out")) + (search-paths '()) + (native-search-paths '()) + (install-source? #t) + (import-path "") + (unpack-path "") + (build-flags ''()) + (tests? #f) ; nothing can be done + (allow-go-reference? #f) + (system (%current-system)) + (goarch (first (go-target target))) + (goos (last (go-target target))) + (guile #f) + (imported-modules %go-build-system-modules) + (modules '((guix build go-build-system) + (guix build union) + (guix build utils)))) + "Cross-build NAME using GO, where TARGET is a GNU triplet and with INPUTS." + (define builder + `(begin + (use-modules ,@modules) + (let () + (define %build-host-inputs + ',(map (match-lambda + ((name (? derivation? drv) sub ...) + `(,name . ,(apply derivation->output-path drv sub))) + ((name path) + `(,name . ,path))) + native-drvs)) + + (define %build-target-inputs + ',(map (match-lambda + ((name (? derivation? drv) sub ...) + `(,name . ,(apply derivation->output-path drv sub))) + ((name (? package? pkg) sub ...) + (let ((drv (package-cross-derivation store pkg + target system))) + `(,name . ,(apply derivation->output-path drv sub)))) + ((name path) + `(,name . ,path))) + target-drvs)) + + (go-build #:name ,name + #:source ,(match (assoc-ref native-drvs "source") + (((? derivation? source)) + (derivation->output-path source)) + ((source) + source) + (source + source)) + #:system ,system + #:phases ,phases + #:outputs %outputs + #:target ,target + #:goarch ,goarch + #:goos ,goos + #:inputs %build-target-inputs + #:native-inputs %build-host-inputs + #:search-paths ',(map search-path-specification->sexp + search-paths) + #:native-search-paths ',(map + search-path-specification->sexp + native-search-paths) + #:install-source? ,install-source? + #:import-path ,import-path + #:unpack-path ,unpack-path + #:build-flags ,build-flags + #:tests? ,tests? + #:allow-go-reference? ,allow-go-reference? + #:inputs %build-inputs)))) + + (define guile-for-build + (match guile + ((? package?) + (package-derivation store guile system #:graft? #f)) + (#f ; the default + (let* ((distro (resolve-interface '(gnu packages commencement))) + (guile (module-ref distro 'guile-final))) + (package-derivation store guile system #:graft? #f))))) + + (build-expression->derivation store name builder + #:system system + #:inputs (append native-drvs target-drvs) + #:outputs outputs + #:modules imported-modules + #:guile-for-build guile-for-build)) + (define go-build-system (build-system (name 'go) diff --git a/guix/build-system/linux-module.scm b/guix/build-system/linux-module.scm index fc3d959ce7..548ed7a9aa 100644 --- a/guix/build-system/linux-module.scm +++ b/guix/build-system/linux-module.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019 Danny Milosavljevic <dannym@scratchpost.org> ;;; Copyright © 2020 Mathieu Othacehe <m.othacehe@gmail.com> +;;; Copyright © 2021 Tobias Geerinckx-Rice <me@tobias.gr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -157,7 +158,9 @@ %standard-phases)) (outputs '("out")) (make-flags ''()) + (parallel-build? #t) (system (%current-system)) + (source-directory ".") (guile #f) (substitutable? #t) (imported-modules @@ -175,7 +178,8 @@ ((source) source) (source - source)) + source)) + #:source-directory ,source-directory #:search-paths ',(map search-path-specification->sexp search-paths) #:phases ,phases @@ -185,6 +189,7 @@ #:tests? ,tests? #:outputs %outputs #:make-flags ,make-flags + #:parallel-build? ,parallel-build? #:inputs %build-inputs))) (define guile-for-build @@ -211,6 +216,7 @@ (guile #f) (outputs '("out")) (make-flags ''()) + (parallel-build? #t) (search-paths '()) (native-search-paths '()) (tests? #f) diff --git a/guix/build-system/minetest.scm b/guix/build-system/minetest.scm new file mode 100644 index 0000000000..1fae3a47e9 --- /dev/null +++ b/guix/build-system/minetest.scm @@ -0,0 +1,99 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build-system minetest) + #:use-module (guix build-system copy) + #:use-module (guix build-system gnu) + #:use-module (guix build-system) + #:use-module (guix utils) + #:export (minetest-mod-build-system)) + +;; +;; Build procedure for minetest mods. This is implemented as an extension +;; of ‘copy-build-system’. +;; +;; Code: + +;; Lazily resolve the bindings to avoid circular dependencies. +(define (default-optipng) + ;; Lazily resolve the binding to avoid a circular dependency. + (module-ref (resolve-interface '(gnu packages image)) 'optipng)) + +(define (default-minetest) + (module-ref (resolve-interface '(gnu packages minetest)) 'minetest)) + +(define (default-xvfb-run) + (module-ref (resolve-interface '(gnu packages xorg)) 'xvfb-run)) + +(define %minetest-build-system-modules + ;; Build-side modules imported by default. + `((guix build minetest-build-system) + ,@%copy-build-system-modules)) + +(define %default-modules + ;; Modules in scope in the build-side environment. + '((guix build gnu-build-system) + (guix build minetest-build-system) + (guix build utils))) + +(define (standard-minetest-packages) + "Return the list of (NAME PACKAGE OUTPUT) or (NAME PACKAGE) tuples of +standard packages used as implicit inputs of the Minetest build system." + `(("xvfb-run" ,(default-xvfb-run)) + ("optipng" ,(default-optipng)) + ("minetest" ,(default-minetest)) + ,@(filter (lambda (input) + (member (car input) + '("libc" "tar" "gzip" "bzip2" "xz" "locales"))) + (standard-packages)))) + +(define* (lower-mod name #:key (implicit-inputs? #t) #:allow-other-keys + #:rest arguments) + (define lower (build-system-lower gnu-build-system)) + (apply lower + name + (substitute-keyword-arguments arguments + ;; minetest-mod-build-system adds implicit inputs by itself, + ;; so don't let gnu-build-system add its own implicit inputs + ;; as well. + ((#:implicit-inputs? implicit-inputs? #t) + #f) + ((#:implicit-cross-inputs? implicit-cross-inputs? #t) + #f) + ((#:imported-modules imported-modules %minetest-build-system-modules) + imported-modules) + ((#:modules modules %default-modules) + modules) + ((#:phases phases '%standard-phases) + phases) + ;; Ensure nothing sneaks into the closure. + ((#:allowed-references allowed-references '()) + allowed-references) + ;; Add the implicit inputs. + ((#:native-inputs native-inputs '()) + (if implicit-inputs? + (append native-inputs (standard-minetest-packages)) + native-inputs))))) + +(define minetest-mod-build-system + (build-system + (name 'minetest-mod) + (description "The build system for minetest mods") + (lower lower-mod))) + +;;; minetest.scm ends here diff --git a/guix/build-system/minify.scm b/guix/build-system/minify.scm index 28a6781c06..9d53760685 100644 --- a/guix/build-system/minify.scm +++ b/guix/build-system/minify.scm @@ -44,8 +44,8 @@ (define (default-uglify-js) "Return the default package to minify JavaScript source files." ;; Lazily resolve the binding to avoid a circular dependency. - (let ((mod (resolve-interface '(gnu packages lisp-xyz)))) - (module-ref mod 'uglify-js))) + (let ((mod (resolve-interface '(gnu packages uglifyjs)))) + (module-ref mod 'uglifyjs))) (define* (lower name #:key source inputs native-inputs outputs system diff --git a/guix/build-system/r.scm b/guix/build-system/r.scm index e2bf41f18d..5e4b23c77e 100644 --- a/guix/build-system/r.scm +++ b/guix/build-system/r.scm @@ -59,7 +59,7 @@ release corresponding to NAME and VERSION." "/src/contrib/" name "_" version ".tar.gz") ;; TODO: use %bioconductor-version from (guix import cran) - (string-append "https://bioconductor.org/packages/3.12" + (string-append "https://bioconductor.org/packages/3.13" type-url-part "/src/contrib/" name "_" version ".tar.gz")))) diff --git a/guix/build-system/renpy.scm b/guix/build-system/renpy.scm index 35edc0056d..5ed59bf5a5 100644 --- a/guix/build-system/renpy.scm +++ b/guix/build-system/renpy.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2021 Leo Prikler <leo.prikler@student.tugraz.at> +;;; Copyright © 2021 Liliana Marie Prikler <liliana.prikler@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; diff --git a/guix/build/asdf-build-system.scm b/guix/build/asdf-build-system.scm index 6ad855cab2..6186613e52 100644 --- a/guix/build/asdf-build-system.scm +++ b/guix/build/asdf-build-system.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016, 2017 Andy Patterson <ajpatter@uwaterloo.ca> -;;; Copyright © 2020 Guillaume Le Vaillant <glv@posteo.net> +;;; Copyright © 2020, 2021 Guillaume Le Vaillant <glv@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -52,12 +52,13 @@ (string-append %source-install-prefix "/systems")) (define (main-system-name output) - (let ((package-name (package-name->name+version - (strip-store-file-name output))) - (lisp-prefix (string-append (%lisp-type) "-"))) - (if (string-prefix? lisp-prefix package-name) - (string-drop package-name (string-length lisp-prefix)) - package-name))) + ;; FIXME: Find a more reliable way to get the main system name. + (let* ((full-name (strip-store-file-name output)) + (lisp-prefix (string-append (%lisp-type) "-")) + (package-name (if (string-prefix? lisp-prefix full-name) + (string-drop full-name (string-length lisp-prefix)) + full-name))) + (package-name->name+version package-name))) (define (lisp-source-directory output name) (string-append output (%lisp-source-install-prefix) "/" name)) @@ -71,8 +72,7 @@ (define (output-translation source-path object-output) - "Return a translation for the system's source path -to it's binary output." + "Return a translation for the system's source path to its binary output." `((,source-path :**/ :*.*.*) (,(library-directory object-output) diff --git a/guix/build/download.scm b/guix/build/download.scm index a22d4064ca..1ed623034b 100644 --- a/guix/build/download.scm +++ b/guix/build/download.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr> +;;; Copyright © 2021 Timothy Sample <samplet@ngyro.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -34,6 +35,8 @@ #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) #:autoload (ice-9 ftw) (scandir) + #:autoload (guix base16) (bytevector->base16-string) + #:autoload (guix swh) (swh-download-directory) #:use-module (ice-9 match) #:use-module (ice-9 format) #:export (open-socket-for-uri @@ -278,21 +281,27 @@ host name without trailing dot." ;;(set-log-level! 10) ;;(set-log-procedure! log) - (catch 'gnutls-error - (lambda () - (handshake session)) - (lambda (key err proc . rest) - (cond ((eq? err error/warning-alert-received) - ;; Like Wget, do no stop upon non-fatal alerts such as - ;; 'alert-description/unrecognized-name'. - (format (current-error-port) - "warning: TLS warning alert received: ~a~%" - (alert-description->string (alert-get session))) - (handshake session)) - (else - ;; XXX: We'd use 'gnutls_error_is_fatal' but (gnutls) doesn't - ;; provide a binding for this. - (apply throw key err proc rest))))) + (let loop ((retries 5)) + (catch 'gnutls-error + (lambda () + (handshake session)) + (lambda (key err proc . rest) + (cond ((eq? err error/warning-alert-received) + ;; Like Wget, do no stop upon non-fatal alerts such as + ;; 'alert-description/unrecognized-name'. + (format (current-error-port) + "warning: TLS warning alert received: ~a~%" + (alert-description->string (alert-get session))) + (handshake session)) + (else + (if (or (fatal-error? err) (zero? retries)) + (apply throw key err proc rest) + (begin + ;; We got 'error/again' or similar; try again. + (format (current-error-port) + "warning: TLS non-fatal error: ~a~%" + (error->string err)) + (loop (- retries 1))))))))) ;; Verify the server's certificate if needed. (when verify-certificate? @@ -626,10 +635,66 @@ Return a list of URIs." (else (list uri)))) +(define* (disarchive-fetch/any uris file + #:key (timeout 10) (verify-certificate? #t)) + "Fetch a Disarchive specification from any of URIS, assemble it, +and write the output to FILE." + (define (fetch-specification uris) + (any (lambda (uri) + (false-if-exception* + (let-values (((port size) (http-fetch uri + #:verify-certificate? + verify-certificate? + #:timeout timeout))) + (let ((specification (read port))) + (close-port port) + specification)))) + uris)) + + (define (resolve addresses output) + (any (match-lambda + (('swhid swhid) + (match (string-split swhid #\:) + (("swh" "1" "dir" id) + (format #t "Downloading ~a from Software Heritage...~%" file) + (false-if-exception* + (swh-download-directory id output))) + (_ #f))) + (_ #f)) + addresses)) + + (format #t "Trying to use Disarchive to assemble ~a...~%" file) + (match (and=> (resolve-module '(disarchive) #:ensure #f) + (lambda (disarchive) + (cons (module-ref disarchive '%disarchive-log-port) + (module-ref disarchive 'disarchive-assemble)))) + (#f (format #t "could not load Disarchive~%") + #f) + ((%disarchive-log-port . disarchive-assemble) + (match (fetch-specification uris) + (#f (format #t "could not find its Disarchive specification~%") + #f) + (spec (parameterize ((%disarchive-log-port (current-output-port))) + (false-if-exception* + (disarchive-assemble spec file #:resolver resolve)))))))) + +(define (internet-archive-uri uri) + "Return a URI corresponding to an Internet Archive backup of URI, or #f if +URI does not denote a Web URI." + (and (memq (uri-scheme uri) '(http https)) + (let* ((now (time-utc->date (current-time time-utc))) + (date (date->string now "~Y~m~d~H~M~S"))) + ;; Note: the date in the URL can be anything and web.archive.org + ;; automatically redirects to the closest date. + (build-uri 'https #:host "web.archive.org" + #:path (string-append "/web/" date "/" + (uri->string uri)))))) + (define* (url-fetch url file #:key (timeout 10) (verify-certificate? #t) (mirrors '()) (content-addressed-mirrors '()) + (disarchive-mirrors '()) (hashes '()) print-build-trace?) "Fetch FILE from URL; URL may be either a single string, or a list of @@ -693,27 +758,54 @@ otherwise simply ignore them." hashes)) content-addressed-mirrors)) + (define disarchive-uris + (append-map (lambda (mirror) + (let ((make-url (match mirror + ((? string?) + (lambda (hash-algo hash) + (string-append + mirror + (symbol->string hash-algo) "/" + (bytevector->base16-string hash)))) + ((? procedure?) + mirror)))) + (map (match-lambda + ((hash-algo . hash) + (string->uri (make-url hash-algo hash)))) + hashes))) + disarchive-mirrors)) + ;; Make this unbuffered so 'progress-report/file' works as expected. 'line ;; means '\n', not '\r', so it's not appropriate here. (setvbuf (current-output-port) 'none) (setvbuf (current-error-port) 'line) - (let try ((uri (append uri content-addressed-uris))) + (let try ((uri (append uri content-addressed-uris + (match uri + ((first . _) + (or (and=> (internet-archive-uri first) list) + '())) + (() '()))))) (match uri ((uri tail ...) (or (fetch uri file) (try tail))) (() - (format (current-error-port) "failed to download ~s from ~s~%" - file url) - - ;; Remove FILE in case we made an incomplete download, for example due - ;; to ENOSPC. - (catch 'system-error - (lambda () - (delete-file file)) - (const #f)) - #f)))) + ;; If we are looking for a software archive, one last thing we + ;; can try is to use Disarchive to assemble it. + (or (disarchive-fetch/any disarchive-uris file + #:verify-certificate? verify-certificate? + #:timeout timeout) + (begin + (format (current-error-port) "failed to download ~s from ~s~%" + file url) + ;; Remove FILE in case we made an incomplete download, for + ;; example due to ENOSPC. + (catch 'system-error + (lambda () + (delete-file file)) + (const #f)) + #f)))))) ;;; download.scm ends here diff --git a/guix/build/dune-build-system.scm b/guix/build/dune-build-system.scm index 7e2ec1e3e1..e9ccc71057 100644 --- a/guix/build/dune-build-system.scm +++ b/guix/build/dune-build-system.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2019 Gabriel Hondet <gabrielhondet@gmail.com> +;;; Copyright © 2021 pukkamustard <pukkamustard@posteo.net> ;;; ;;; This file is part of GNU Guix. ;;; @@ -31,20 +32,26 @@ ;; Code: (define* (build #:key (build-flags '()) (jbuild? #f) - (use-make? #f) (package #f) #:allow-other-keys) + (use-make? #f) (package #f) (dune-release-flags '()) + #:allow-other-keys) "Build the given package." (let ((program (if jbuild? "jbuilder" "dune"))) (apply invoke program "build" "@install" - (append (if package (list "-p" package) '()) build-flags))) + (append (if package (list "-p" package) + dune-release-flags) + build-flags))) #t) (define* (check #:key (test-flags '()) (test-target "test") tests? - (jbuild? #f) (package #f) #:allow-other-keys) + (jbuild? #f) (package #f) (dune-release-flags '()) + #:allow-other-keys) "Test the given package." (when tests? (let ((program (if jbuild? "jbuilder" "dune"))) (apply invoke program "runtest" test-target - (append (if package (list "-p" package) '()) test-flags)))) + (append (if package (list "-p" package) + dune-release-flags) + test-flags)))) #t) (define* (install #:key outputs (install-target "install") (jbuild? #f) diff --git a/guix/build/emacs-build-system.scm b/guix/build/emacs-build-system.scm index 26ea59bc25..e41e9a6595 100644 --- a/guix/build/emacs-build-system.scm +++ b/guix/build/emacs-build-system.scm @@ -26,13 +26,16 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) + #:use-module (ice-9 format) + #:use-module (ice-9 ftw) #:use-module (ice-9 rdelim) #:use-module (ice-9 regex) #:use-module (ice-9 match) #:export (%standard-phases %default-include %default-exclude - emacs-build)) + emacs-build + elpa-directory)) ;; Commentary: ;; @@ -40,9 +43,12 @@ ;; ;; Code: -;;; All the packages are installed directly under site-lisp, which means that -;;; having that directory in the EMACSLOADPATH is enough to have them found by -;;; Emacs. +;;; The location in which Emacs looks for packages. Emacs Lisp code that is +;;; installed there directly will be found when that directory is added to +;;; EMACSLOADPATH. To avoid clashes between packages (particularly considering +;;; auxiliary files), we install them one directory level below, however. +;;; This indirection is handled by ‘expand-load-path’ during build and a +;;; profile hook otherwise. (define %install-dir "/share/emacs/site-lisp") ;; These are the default inclusion/exclusion regexps for the install phase. @@ -73,33 +79,43 @@ archive, a directory, or an Emacs Lisp file." #t) (gnu:unpack #:source source))) -(define* (add-source-to-load-path #:key dummy #:allow-other-keys) - "Augment the EMACSLOADPATH environment variable with the source directory." +(define* (expand-load-path #:key (prepend-source? #t) #:allow-other-keys) + "Expand EMACSLOADPATH, so that inputs, whose code resides in subdirectories, +are properly found. +If @var{prepend-source?} is @code{#t} (the default), also add the current +directory to EMACSLOADPATH in front of any other directories." (let* ((source-directory (getcwd)) (emacs-load-path (string-split (getenv "EMACSLOADPATH") #\:)) - ;; XXX: Make sure the Emacs core libraries appear at the end of - ;; EMACSLOADPATH, to avoid shadowing any other libraries depended - ;; upon. - (emacs-load-path-non-core (filter (cut string-contains <> - "/share/emacs/site-lisp") - emacs-load-path)) + (emacs-load-path* + (map + (lambda (dir) + (match (scandir dir (negate (cute member <> '("." "..")))) + ((sub) (string-append dir "/" sub)) + (_ dir))) + emacs-load-path)) (emacs-load-path-value (string-append - (string-join (cons source-directory - emacs-load-path-non-core) - ":") + (string-join + (if prepend-source? + (cons source-directory emacs-load-path*) + emacs-load-path*) + ":") ":"))) (setenv "EMACSLOADPATH" emacs-load-path-value) - (format #t "source directory ~s prepended to the `EMACSLOADPATH' \ -environment variable\n" source-directory))) + (when prepend-source? + (format #t "source directory ~s prepended to the `EMACSLOADPATH' \ +environment variable\n" source-directory)) + (let ((diff (lset-difference string=? emacs-load-path* emacs-load-path))) + (unless (null? diff) + (format #t "expanded load paths for ~{~a~^, ~}\n" + (map basename diff)))))) (define* (build #:key outputs inputs #:allow-other-keys) "Compile .el files." (let* ((emacs (string-append (assoc-ref inputs "emacs") "/bin/emacs")) - (out (assoc-ref outputs "out")) - (site-lisp (string-append out %install-dir))) + (out (assoc-ref outputs "out"))) (setenv "SHELL" "sh") (parameterize ((%emacs emacs)) - (emacs-byte-compile-directory site-lisp)))) + (emacs-byte-compile-directory (elpa-directory out))))) (define* (patch-el-files #:key outputs #:allow-other-keys) "Substitute the absolute \"/bin/\" directory with the right location in the @@ -116,7 +132,8 @@ store in '.el' files." #:binary #t)) (let* ((out (assoc-ref outputs "out")) - (site-lisp (string-append out %install-dir)) + (elpa-name-ver (store-directory->elpa-name-version out)) + (el-dir (string-append out %install-dir "/" elpa-name-ver)) ;; (ice-9 regex) uses libc's regexp routines, which cannot deal with ;; strings containing NULs. Filter out such files. TODO: Remove ;; this workaround when <https://bugs.gnu.org/30116> is fixed. @@ -130,7 +147,7 @@ store in '.el' files." (error "patch-el-files: unable to locate " cmd-name)) (string-append "\"" cmd "\""))))) - (with-directory-excursion site-lisp + (with-directory-excursion el-dir ;; Some old '.el' files (e.g., tex-buf.el in AUCTeX) are still ;; ISO-8859-1-encoded. (unless (false-if-exception (substitute-program-names)) @@ -181,14 +198,14 @@ parallel. PARALLEL-TESTS? is ignored when using a non-make TEST-COMMAND." (not (any (cut match-stripped-file "excluded" <>) exclude))))) (let* ((out (assoc-ref outputs "out")) - (site-lisp (string-append out %install-dir)) + (el-dir (elpa-directory out)) (files-to-install (find-files source install-file?))) (cond ((not (null? files-to-install)) (for-each (lambda (file) (let* ((stripped-file (string-drop file (string-length source))) - (target-file (string-append site-lisp stripped-file))) + (target-file (string-append el-dir stripped-file))) (format #t "`~a' -> `~a'~%" file target-file) (install-file file (dirname target-file)))) files-to-install) @@ -219,11 +236,11 @@ parallel. PARALLEL-TESTS? is ignored when using a non-make TEST-COMMAND." "Generate the autoloads file." (let* ((emacs (string-append (assoc-ref inputs "emacs") "/bin/emacs")) (out (assoc-ref outputs "out")) - (site-lisp (string-append out %install-dir)) (elpa-name-ver (store-directory->elpa-name-version out)) - (elpa-name (package-name->name+version elpa-name-ver))) + (elpa-name (package-name->name+version elpa-name-ver)) + (el-dir (elpa-directory out))) (parameterize ((%emacs emacs)) - (emacs-generate-autoloads elpa-name site-lisp)))) + (emacs-generate-autoloads elpa-name el-dir)))) (define* (enable-autoloads-compilation #:key outputs #:allow-other-keys) "Remove the NO-BYTE-COMPILATION local variable embedded in the generated @@ -258,10 +275,16 @@ second hyphen. This corresponds to 'name-version' as used in ELPA packages." strip-store-file-name) store-dir)) +(define (elpa-directory store-dir) + "Given the store directory STORE-DIR return the absolute install directory +for libraries following the ELPA convention." + (string-append store-dir %install-dir "/" + (store-directory->elpa-name-version store-dir))) + (define %standard-phases (modify-phases gnu:%standard-phases (replace 'unpack unpack) - (add-after 'unpack 'add-source-to-load-path add-source-to-load-path) + (add-after 'unpack 'expand-load-path expand-load-path) (delete 'bootstrap) (delete 'configure) (delete 'build) diff --git a/guix/build/emacs-utils.scm b/guix/build/emacs-utils.scm index 5f7ba71244..64ef40e25a 100644 --- a/guix/build/emacs-utils.scm +++ b/guix/build/emacs-utils.scm @@ -2,7 +2,7 @@ ;;; Copyright © 2014, 2018 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2014 Alex Kost <alezost@gmail.com> ;;; Copyright © 2018, 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> -;;; Copyright © 2019 Leo Prikler <leo.prikler@student.tugraz.at> +;;; Copyright © 2019 Liliana Marie Prikler <liliana.prikler@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; diff --git a/guix/build/go-build-system.scm b/guix/build/go-build-system.scm index 227df820db..645d2fe680 100644 --- a/guix/build/go-build-system.scm +++ b/guix/build/go-build-system.scm @@ -4,7 +4,7 @@ ;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; Copyright © 2020 Jack Hill <jackhill@jackhill.us> ;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net> -;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> +;;; Copyright © 2020, 2021 Efraim Flashner <efraim@flashner.co.il> ;;; ;;; This file is part of GNU Guix. ;;; @@ -131,7 +131,7 @@ ;; ;; Code: -(define* (setup-go-environment #:key inputs outputs #:allow-other-keys) +(define* (setup-go-environment #:key inputs outputs goos goarch #:allow-other-keys) "Prepare a Go build environment for INPUTS and OUTPUTS. Build a file system union of INPUTS. Export GOPATH, which helps the compiler find the source code of the package being built and its dependencies, and GOBIN, which determines @@ -149,6 +149,22 @@ dependencies, so it should be self-contained." ;; GOPATH behavior. (setenv "GO111MODULE" "off") (setenv "GOBIN" (string-append (assoc-ref outputs "out") "/bin")) + + ;; Make sure we're building for the correct architecture and OS targets + ;; that Guix targets. + (setenv "GOARCH" goarch) + (setenv "GOOS" goos) + (match goarch + ("arm" + (setenv "GOARM" "7")) + ((or "mips" "mipsel") + (setenv "GOMIPS" "hardfloat")) + ((or "mips64" "mips64le") + (setenv "GOMIPS64" "hardfloat")) + ((or "ppc64" "ppc64le") + (setenv "GOPPC64" "power8")) + (_ #t)) + (let ((tmpdir (tmpnam))) (match (go-inputs inputs) (((names . directories) ...) diff --git a/guix/build/graft.scm b/guix/build/graft.scm index c119ee71d1..f04c35fa74 100644 --- a/guix/build/graft.scm +++ b/guix/build/graft.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014, 2015, 2016, 2018 Ludovic Courtès <ludo@gnu.org> -;;; Copyright © 2016 Mark H Weaver <mhw@netris.org> +;;; Copyright © 2016, 2021 Mark H Weaver <mhw@netris.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -55,6 +55,52 @@ (string->char-set "0123456789abcdfghijklmnpqrsvwxyz") <>)) +(define (nix-base32-char-or-nul? c) + "Return true if C is a nix-base32 character or NUL, otherwise return false." + (or (nix-base32-char? c) + (char=? c #\nul))) + +(define (possible-utf16-hash? buffer i w) + "Return true if (I - W) is large enough to hold a UTF-16 encoded +nix-base32 hash and if BUFFER contains NULs in all positions where NULs +are to be expected in a UTF-16 encoded hash+dash pattern whose dash is +found at position I. Otherwise, return false." + (and (<= (* 2 hash-length) (- i w)) + (let loop ((j (+ 1 (- i (* 2 hash-length))))) + (or (>= j i) + (and (zero? (bytevector-u8-ref buffer j)) + (loop (+ j 2))))))) + +(define (possible-utf32-hash? buffer i w) + "Return true if (I - W) is large enough to hold a UTF-32 encoded +nix-base32 hash and if BUFFER contains NULs in all positions where NULs +are to be expected in a UTF-32 encoded hash+dash pattern whose dash is +found at position I. Otherwise, return false." + (and (<= (* 4 hash-length) (- i w)) + (let loop ((j (+ 1 (- i (* 4 hash-length))))) + (or (>= j i) + (and (zero? (bytevector-u8-ref buffer j)) + (zero? (bytevector-u8-ref buffer (+ j 1))) + (zero? (bytevector-u8-ref buffer (+ j 2))) + (loop (+ j 4))))))) + +(define (insert-nuls char-size bv) + "Given a bytevector BV, return a bytevector containing the same bytes but +with (CHAR-SIZE - 1) NULs inserted between every two adjacent bytes from BV. +For example, (insert-nuls 4 #u8(1 2 3)) => #u8(1 0 0 0 2 0 0 0 3)." + (if (= char-size 1) + bv + (let* ((len (bytevector-length bv)) + (bv* (make-bytevector (+ 1 (* char-size + (- len 1))) + 0))) + (let loop ((i 0)) + (when (< i len) + (bytevector-u8-set! bv* (* i char-size) + (bytevector-u8-ref bv i)) + (loop (+ i 1)))) + bv*))) + (define* (replace-store-references input output replacement-table #:optional (store (%store-directory))) "Read data from INPUT, replacing store references according to @@ -76,9 +122,9 @@ bytevectors to the same value." (list->vector (map pred (iota 256))) <>)) - (define nix-base32-byte? + (define nix-base32-byte-or-nul? (optimize-u8-predicate - (compose nix-base32-char? + (compose nix-base32-char-or-nul? integer->char))) (define (dash? byte) (= byte 45)) @@ -86,100 +132,153 @@ bytevectors to the same value." (define request-size (expt 2 20)) ; 1 MiB ;; We scan the file for the following 33-byte pattern: 32 bytes of - ;; nix-base32 characters followed by a dash. To accommodate large files, - ;; we do not read the entire file, but instead work on buffers of up to - ;; 'request-size' bytes. To ensure that every 33-byte sequence appears - ;; entirely within exactly one buffer, adjacent buffers must overlap, - ;; i.e. they must share 32 byte positions. We accomplish this by - ;; "ungetting" the last 32 bytes of each buffer before reading the next - ;; buffer, unless we know that we've reached the end-of-file. + ;; nix-base32 characters followed by a dash. When we find such a pattern + ;; whose hash is in REPLACEMENT-TABLE, we perform the required rewrite and + ;; continue scanning. + ;; + ;; To support UTF-16 and UTF-32 store references, the 33 bytes comprising + ;; this hash+dash pattern may optionally be interspersed by extra NUL bytes. + ;; This simple approach works because the characters we are looking for are + ;; restricted to ASCII. UTF-16 hashes are interspersed with single NUL + ;; bytes ("\0"), and UTF-32 hashes are interspersed with triplets of NULs + ;; ("\0\0\0"). Note that we require NULs to be present only *between* the + ;; other bytes, and not at either end, in order to be insensitive to byte + ;; order. + ;; + ;; To accommodate large files, we do not read the entire file at once, but + ;; instead work on buffers of up to REQUEST-SIZE bytes. To ensure that + ;; every hash+dash pattern appears in its entirety in at least one buffer, + ;; adjacent buffers must overlap by one byte less than the maximum size of a + ;; hash+dash pattern. We accomplish this by "ungetting" a suffix of each + ;; buffer before reading the next buffer, unless we know that we've reached + ;; the end-of-file. (let ((buffer (make-bytevector request-size))) - (let loop () - ;; Note: We avoid 'get-bytevector-n' to work around - ;; <http://bugs.gnu.org/17466>. + (define-syntax-rule (byte-at i) + (bytevector-u8-ref buffer i)) + (let outer-loop () (match (get-bytevector-n! input buffer 0 request-size) ((? eof-object?) 'done) (end - ;; We scan the buffer for dashes that might be preceded by a - ;; nix-base32 hash. The key optimization here is that whenever we - ;; find a NON-nix-base32 character at position 'i', we know that it - ;; cannot be part of a hash, so the earliest position where the next - ;; hash could start is i+1 with the following dash at position i+33. - ;; - ;; Since nix-base32 characters comprise only 1/8 of the 256 possible - ;; byte values, and exclude some of the most common letters in - ;; English text (e t o u), in practice we can advance by 33 positions - ;; most of the time. - (let scan-from ((i hash-length) (written 0)) - ;; 'i' is the first position where we look for a dash. 'written' - ;; is the number of bytes in the buffer that have already been - ;; written. + (define (scan-from i w) + ;; Scan the buffer for dashes that might be preceded by nix hashes, + ;; where I is the minimum position where such a dash might be + ;; found, and W is the number of bytes in the buffer that have been + ;; written so far. We assume that I - W >= HASH-LENGTH. + ;; + ;; The key optimization here is that whenever we find a byte at + ;; position I that cannot occur within a nix hash (because it's + ;; neither a nix-base32 character nor NUL), we can infer that the + ;; earliest position where the next hash could start is at I + 1, + ;; and therefore the earliest position for the following dash is + ;; (+ I 1 HASH-LENGTH), which is I + 33. + ;; + ;; Since nix-base32-or-nul characters comprise only about 1/8 of + ;; the 256 possible byte values, and exclude some of the most + ;; common letters in English text (e t o u), we can advance 33 + ;; positions much of the time. (if (< i end) - (let ((byte (bytevector-u8-ref buffer i))) - (cond ((and (dash? byte) - ;; We've found a dash. Note that we do not know - ;; whether the preceeding 32 bytes are nix-base32 - ;; characters, but we do not need to know. If - ;; they are not, the following lookup will fail. - (lookup-replacement - (string-tabulate (lambda (j) - (integer->char - (bytevector-u8-ref buffer - (+ j (- i hash-length))))) - hash-length))) - => (lambda (replacement) - ;; We've found a hash that needs to be replaced. - ;; First, write out all bytes preceding the hash - ;; that have not yet been written. - (put-bytevector output buffer written - (- i hash-length written)) - ;; Now write the replacement string. - (put-bytevector output replacement) - ;; Since the byte at position 'i' is a dash, - ;; which is not a nix-base32 char, the earliest - ;; position where the next hash might start is - ;; i+1, and the earliest position where the - ;; following dash might start is (+ i 1 - ;; hash-length). Also, increase the write - ;; position to account for REPLACEMENT. - (let ((len (bytevector-length replacement))) - (scan-from (+ i 1 len) - (+ i (- len hash-length)))))) - ;; If the byte at position 'i' is a nix-base32 char, - ;; then the dash we're looking for might be as early as - ;; the following byte, so we can only advance by 1. - ((nix-base32-byte? byte) - (scan-from (+ i 1) written)) - ;; If the byte at position 'i' is NOT a nix-base32 - ;; char, then the earliest position where the next hash - ;; might start is i+1, with the following dash at - ;; position (+ i 1 hash-length). + (let ((byte (byte-at i))) + (cond ((dash? byte) + (found-dash i w)) + ((nix-base32-byte-or-nul? byte) + (scan-from (+ i 1) w)) (else - (scan-from (+ i 1 hash-length) written)))) + (not-part-of-hash i w)))) + (finish-buffer i w))) + + (define (not-part-of-hash i w) + ;; Position I is known to not be within a nix hash that we must + ;; rewrite. Therefore, the earliest position where the next hash + ;; might start is I + 1, and therefore the earliest position of + ;; the following dash is (+ I 1 HASH-LENGTH). + (scan-from (+ i 1 hash-length) w)) + + (define (found-dash i w) + ;; We know that there is a dash '-' at position I, and that + ;; I - W >= HASH-LENGTH. The immediately preceding bytes *might* + ;; contain a nix-base32 hash, but that is not yet known. Here, + ;; we rule out all but one possible encoding (ASCII, UTF-16, + ;; UTF-32) by counting how many NULs precede the dash. + (cond ((not (zero? (byte-at (- i 1)))) + ;; The dash is *not* preceded by a NUL, therefore it + ;; cannot possibly be a UTF-16 or UTF-32 hash. Proceed + ;; to check for an ASCII hash. + (found-possible-hash 1 i w)) + + ((not (zero? (byte-at (- i 2)))) + ;; The dash is preceded by exactly one NUL, therefore it + ;; cannot be an ASCII or UTF-32 hash. Proceed to check + ;; for a UTF-16 hash. + (if (possible-utf16-hash? buffer i w) + (found-possible-hash 2 i w) + (not-part-of-hash i w))) + + (else + ;; The dash is preceded by at least two NULs, therefore + ;; it cannot be an ASCII or UTF-16 hash. Proceed to + ;; check for a UTF-32 hash. + (if (possible-utf32-hash? buffer i w) + (found-possible-hash 4 i w) + (not-part-of-hash i w))))) + + (define (found-possible-hash char-size i w) + ;; We know that there is a dash '-' at position I, that + ;; I - W >= CHAR-SIZE * HASH-LENGTH, and that the only + ;; possible encoding for the preceding hash is as indicated by + ;; CHAR-SIZE. Here we check to see if the given hash is in + ;; REPLACEMENT-TABLE, and if so, we perform the required + ;; rewrite. + (let* ((hash (string-tabulate + (lambda (j) + (integer->char + (byte-at (- i (* char-size + (- hash-length j)))))) + hash-length)) + (replacement* (lookup-replacement hash)) + (replacement (and replacement* + (insert-nuls char-size replacement*)))) + (cond + ((not replacement) + (not-part-of-hash i w)) + (else + ;; We've found a hash that needs to be replaced. + ;; First, write out all bytes preceding the hash + ;; that have not yet been written. + (put-bytevector output buffer w + (- i (* char-size hash-length) w)) + ;; Now write the replacement string. + (put-bytevector output replacement) + ;; Now compute the new values of W and I and continue. + (let ((w (+ (- i (* char-size hash-length)) + (bytevector-length replacement)))) + (scan-from (+ w hash-length) w)))))) + + (define (finish-buffer i w) + ;; We have finished scanning the buffer. Now we determine how many + ;; bytes have not yet been written, and how many bytes to "unget". + ;; If END is less than REQUEST-SIZE then we read less than we asked + ;; for, which indicates that we are at EOF, so we needn't unget + ;; anything. Otherwise, we unget up to (* 4 HASH-LENGTH) bytes. + ;; However, we must be careful not to unget bytes that have already + ;; been written, because that would cause them to be written again + ;; from the next buffer. In practice, this case occurs when a + ;; replacement is made near or beyond the end of the buffer. When + ;; REPLACEMENT went beyond END, we consume the extra bytes from + ;; INPUT. + (if (> w end) + (get-bytevector-n! input buffer 0 (- w end)) + (let* ((unwritten (- end w)) + (unget-size (if (= end request-size) + (min (* 4 hash-length) + unwritten) + 0)) + (write-size (- unwritten unget-size))) + (put-bytevector output buffer w write-size) + (unget-bytevector input buffer (+ w write-size) + unget-size))) + (outer-loop)) - ;; We have finished scanning the buffer. Now we determine how - ;; many bytes have not yet been written, and how many bytes to - ;; "unget". If 'end' is less than 'request-size' then we read - ;; less than we asked for, which indicates that we are at EOF, - ;; so we needn't unget anything. Otherwise, we unget up to - ;; 'hash-length' bytes (32 bytes). However, we must be careful - ;; not to unget bytes that have already been written, because - ;; that would cause them to be written again from the next - ;; buffer. In practice, this case occurs when a replacement is - ;; made near or beyond the end of the buffer. When REPLACEMENT - ;; went beyond END, we consume the extra bytes from INPUT. - (begin - (if (> written end) - (get-bytevector-n! input buffer 0 (- written end)) - (let* ((unwritten (- end written)) - (unget-size (if (= end request-size) - (min hash-length unwritten) - 0)) - (write-size (- unwritten unget-size))) - (put-bytevector output buffer written write-size) - (unget-bytevector input buffer (+ written write-size) - unget-size))) - (loop))))))))) + (scan-from hash-length 0)))))) (define (rename-matching-files directory mapping) "Apply MAPPING to the names of all the files in DIRECTORY, where MAPPING is diff --git a/guix/build/java-utils.scm b/guix/build/java-utils.scm index a868e4d52c..87c3ac43c9 100644 --- a/guix/build/java-utils.scm +++ b/guix/build/java-utils.scm @@ -2,7 +2,7 @@ ;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Alex Vong <alexvong1995@gmail.com> -;;; Copyright © 2020 Julien Lepiller <julien@lepiller.eu> +;;; Copyright © 2020, 2021 Julien Lepiller <julien@lepiller.eu> ;;; ;;; This file is part of GNU Guix. ;;; @@ -28,6 +28,7 @@ #:use-module (sxml simple) #:export (ant-build-javadoc generate-plugin.xml + generate-pom.xml install-jars install-javadoc install-pom-file @@ -68,9 +69,9 @@ fetched." (let* ((out (assoc-ref outputs "out")) (java-inputs (append (map cdr inputs) (map cdr outputs))) (pom-content (get-pom pom-file)) - (version (pom-version pom-content java-inputs)) + (version (pom-version pom-content)) (artifact (pom-artifactid pom-content)) - (group (group->dir (pom-groupid pom-content java-inputs))) + (group (group->dir (pom-groupid pom-content))) (repository (string-append out "/lib/m2/" group "/" artifact "/" version "/")) (pom-name (string-append repository artifact "-" version ".pom"))) @@ -86,8 +87,8 @@ to ensure that maven can find dependencies." (manifest (string-append dir "/META-INF/MANIFEST.MF")) (pom (get-pom pom-file)) (artifact (pom-artifactid pom)) - (group (pom-groupid pom inputs)) - (version (pom-version pom inputs)) + (group (pom-groupid pom)) + (version (pom-version pom)) (pom-dir (string-append "META-INF/maven/" group "/" artifact))) (mkdir-p (string-append dir "/" pom-dir)) (copy-file pom-file (string-append dir "/" pom-dir "/pom.xml")) @@ -112,9 +113,9 @@ the phase fails." (let* ((out (assoc-ref outputs "out")) (java-inputs (append (map cdr inputs) (map cdr outputs))) (pom-content (get-pom pom-file)) - (version (pom-version pom-content java-inputs)) + (version (pom-version pom-content)) (artifact (pom-artifactid pom-content)) - (group (group->dir (pom-groupid pom-content java-inputs))) + (group (group->dir (pom-groupid pom-content))) (repository (string-append out "/lib/m2/" group "/" artifact "/" version "/")) ;; We try to find the file that was built. If it was built from our @@ -124,7 +125,7 @@ the phase fails." version ".jar")))) ;; Otherwise, we try to find any jar file. (jars (if (null? jars) - (find-files "." ".*.jar") + (find-files "." "\\.jar$") jars)) (jar-name (string-append repository artifact "-" version ".jar")) (pom-name (string-append repository artifact "-" version ".pom"))) @@ -179,9 +180,9 @@ recognize the package as a plugin, and find the entry points in the plugin." (name (pom-name pom-content)) (description (pom-description pom-content)) (dependencies (pom-dependencies pom-content)) - (version (pom-version pom-content java-inputs)) + (version (pom-version pom-content)) (artifact (pom-artifactid pom-content)) - (groupid (pom-groupid pom-content java-inputs)) + (groupid (pom-groupid pom-content)) (mojos `(mojos ,@(with-directory-excursion directory @@ -206,3 +207,31 @@ recognize the package as a plugin, and find the entry points in the plugin." ,mojos (dependencies ,@dependencies))))))))) + +(define* (generate-pom.xml pom-file groupid artifactid version + #:key (dependencies '()) + (name artifactid)) + "Generates the @file{pom.xml} for a project. It is required by Maven to find +a package, and by the java build system to know where to install a package, when +a pom.xml doesn't already exist and installing to the maven repository." + (lambda _ + (mkdir-p (dirname pom-file)) + (with-output-to-file pom-file + (lambda _ + (sxml->xml + (sxml-indent + `(project + (modelVersion "4.0.0") + (name ,name) + (groupId ,groupid) + (artifactId ,artifactid) + (version ,version) + (dependencies + ,@(map + (match-lambda + ((groupid artifactid version) + `(dependency + (groupId ,groupid) + (artifactId ,artifactid) + (version ,version)))) + dependencies))))))))) diff --git a/guix/build/linux-module-build-system.scm b/guix/build/linux-module-build-system.scm index d51d76f94b..18ccf7cd8b 100644 --- a/guix/build/linux-module-build-system.scm +++ b/guix/build/linux-module-build-system.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019 Danny Milosavljevic <dannym@scratchpost.org> ;;; Copyright © 2020 Mathieu Othacehe <m.othacehe@gmail.com> +;;; Copyright © 2021 Tobias Geerinckx-Rice <me@tobias.gr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -33,7 +34,7 @@ ;; ;; Code: -;; Copied from make-linux-libre's "configure" phase. +;; Similar to make-linux-libre's "configure" phase. (define* (configure #:key inputs target arch #:allow-other-keys) (setenv "KCONFIG_NOTIMESTAMP" "1") (setenv "KBUILD_BUILD_TIMESTAMP" (getenv "SOURCE_DATE_EPOCH")) @@ -42,23 +43,29 @@ (format #t "`ARCH' set to `~a'~%" (getenv "ARCH")) (when target + ;; TODO? (setenv "EXTRA_VERSION" ,extra-version) + ;; TODO? kernel ".config". (setenv "CROSS_COMPILE" (string-append target "-")) (format #t "`CROSS_COMPILE' set to `~a'~%" - (getenv "CROSS_COMPILE"))) - ; TODO: (setenv "EXTRA_VERSION" ,extra-version) - ; TODO: kernel ".config". - #t) + (getenv "CROSS_COMPILE")))) -(define* (build #:key inputs make-flags #:allow-other-keys) +(define* (build #:key (make-flags '()) (parallel-build? #t) + (source-directory ".") + inputs + #:allow-other-keys) (apply invoke "make" "-C" (string-append (assoc-ref inputs "linux-module-builder") "/lib/modules/build") - (string-append "M=" (getcwd)) - (or make-flags '()))) + (string-append "M=" (canonicalize-path source-directory)) + `(,@(if parallel-build? + `("-j" ,(number->string (parallel-job-count))) + '()) + ,@make-flags))) -;; This block was copied from make-linux-libre--only took the "modules_install" -;; part. -(define* (install #:key make-flags inputs native-inputs outputs +;; Similar to the "modules_install" part of make-linux-libre. +(define* (install #:key (make-flags '()) (parallel-build? #t) + (source-directory ".") + inputs native-inputs outputs #:allow-other-keys) (let* ((out (assoc-ref outputs "out")) (moddir (string-append out "/lib/modules"))) @@ -67,7 +74,7 @@ (apply invoke "make" "-C" (string-append (assoc-ref inputs "linux-module-builder") "/lib/modules/build") - (string-append "M=" (getcwd)) + (string-append "M=" (canonicalize-path source-directory)) ;; Disable depmod because the Guix system's module directory ;; is an union of potentially multiple packages. It is not ;; possible to use depmod to usefully calculate a dependency @@ -78,7 +85,10 @@ (string-append "INSTALL_MOD_PATH=" out) "INSTALL_MOD_STRIP=1" "modules_install" - (or make-flags '())))) + `(,@(if parallel-build? + `("-j" ,(number->string (parallel-job-count))) + '()) + ,@make-flags)))) (define %standard-phases (modify-phases gnu:%standard-phases diff --git a/guix/build/maven-build-system.scm b/guix/build/maven-build-system.scm index 534b4ebcee..b3d97c81ea 100644 --- a/guix/build/maven-build-system.scm +++ b/guix/build/maven-build-system.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2020 Julien Lepiller <julien@lepiller.eu> +;;; Copyright © 2020, 2021 Julien Lepiller <julien@lepiller.eu> ;;; ;;; This file is part of GNU Guix. ;;; @@ -60,47 +60,23 @@ (invoke "mvn" "-v") #t) -(define (add-local-package local-packages group artifact version) - (define (alist-set lst key val) - (match lst - ('() (list (cons key val))) - (((k . v) lst ...) - (if (equal? k key) - (cons (cons key val) lst) - (cons (cons k v) (alist-set lst key val)))))) - (alist-set local-packages group - (alist-set (or (assoc-ref local-packages group) '()) artifact - version))) - (define (fix-pom pom-file inputs local-packages excludes) (chmod pom-file #o644) (format #t "fixing ~a~%" pom-file) (fix-pom-dependencies pom-file (map cdr inputs) #:with-plugins? #t #:with-build-dependencies? #t + #:with-modules? #t #:local-packages local-packages - #:excludes excludes) - (let* ((pom (get-pom pom-file)) - (java-inputs (map cdr inputs)) - (artifact (pom-artifactid pom)) - (group (pom-groupid pom java-inputs local-packages)) - (version (pom-version pom java-inputs local-packages))) - (let loop ((modules (pom-ref pom "modules")) - (local-packages - (add-local-package local-packages group artifact version))) - (pk 'local-packages local-packages) - (match modules - (#f local-packages) - ('() local-packages) - (((? string? _) modules ...) - (loop modules local-packages)) - (((_ module) modules ...) - (loop - modules - (fix-pom (string-append (dirname pom-file) "/" module "/pom.xml") - inputs local-packages excludes))))))) + #:excludes excludes)) (define* (fix-pom-files #:key inputs local-packages exclude #:allow-other-keys) - (fix-pom "pom.xml" inputs local-packages exclude) + (let ((local-packages (pom-local-packages "pom.xml" #:local-packages local-packages))) + (format (current-error-port) "Fix pom files with local packages: ~a~%" local-packages) + (for-each + (lambda (pom) + (when (file-exists? pom) + (fix-pom pom inputs local-packages exclude))) + (pom-and-submodules "pom.xml"))) #t) (define* (build #:key outputs #:allow-other-keys) diff --git a/guix/build/maven/pom.scm b/guix/build/maven/pom.scm index c92d409d2b..9e35e47a7f 100644 --- a/guix/build/maven/pom.scm +++ b/guix/build/maven/pom.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2019, 2020 Julien Lepiller <julien@lepiller.eu> +;;; Copyright © 2019-2021 Julien Lepiller <julien@lepiller.eu> ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,7 +21,8 @@ #:use-module (system foreign) #:use-module (ice-9 match) #:use-module (srfi srfi-1) - #:export (get-pom + #:export (add-local-package + get-pom pom-ref pom-description pom-name @@ -30,8 +31,24 @@ pom-groupid pom-dependencies group->dir + pom-and-submodules + pom-local-packages fix-pom-dependencies)) +(define (add-local-package local-packages group artifact version) + "Takes @var{local-packages}, a list of local packages, and adds a new one +for @var{group}:@var{artifact} at @var{version}." + (define (alist-set lst key val) + (match lst + ('() (list (cons key val))) + (((k . v) lst ...) + (if (equal? k key) + (cons (cons key val) lst) + (cons (cons k v) (alist-set lst key val)))))) + (alist-set local-packages group + (alist-set (or (assoc-ref local-packages group) '()) artifact + version))) + (define (get-pom file) "Return the content of a @file{.pom} file." (let ((pom-content (call-with-input-file file xml->sxml))) @@ -93,13 +110,12 @@ If no result is found, the result is @code{#f}." (get-pom (car java-inputs)))) #f))) -(define* (pom-groupid content inputs #:optional local-packages) +(define* (pom-groupid content) "Find the groupID of a pom file, potentially looking at its parent pom file. See @code{find-parent} for the meaning of the arguments." (if content (let ((res (or (pom-ref content "groupId") - (pom-groupid (find-parent content inputs local-packages) - inputs)))) + (pom-ref (pom-ref content "parent") "groupId")))) (cond ((string? res) res) ((null? res) #f) @@ -114,13 +130,12 @@ See @code{find-parent} for the meaning of the arguments." (car res) #f))) -(define* (pom-version content inputs #:optional local-packages) +(define* (pom-version content) "Find the version of a pom file, potentially looking at its parent pom file. See @code{find-parent} for the meaning of the arguments." (if content (let ((res (or (pom-ref content "version") - (pom-version (find-parent content inputs local-packages) - inputs)))) + (pom-ref (pom-ref content "parent") "version")))) (cond ((string? res) res) ((null? res) #f) @@ -236,13 +251,48 @@ to re-declare the namespaces in the top-level element." http://maven.apache.org/xsd/maven-4.0.0.xsd")) ,(map fix-xml sxml))))) +(define (pom-and-submodules pom-file) + "Given @var{pom-file}, the file name of a pom, return the list of pom file +names that correspond to itself and its submodules, recursively." + (define (get-modules modules) + (match modules + (#f '()) + ('() '()) + (((? string? _) rest ...) (get-modules rest)) + ((('http://maven.apache.org/POM/4.0.0:module mod) rest ...) + (let ((pom (string-append (dirname pom-file) "/" mod "/pom.xml"))) + (if (file-exists? pom) + (cons pom (get-modules rest)) + (get-modules rest)))))) + + (let* ((pom (get-pom pom-file)) + (modules (get-modules (pom-ref pom "modules")))) + (cons pom-file + (apply append (map pom-and-submodules modules))))) + +(define* (pom-local-packages pom-file #:key (local-packages '())) + "Given @var{pom-file}, a pom file name, return a list of local packages that +this repository contains." + (let loop ((modules (pom-and-submodules pom-file)) + (local-packages local-packages)) + (match modules + (() local-packages) + ((module modules ...) + (let* ((pom (get-pom module)) + (version (pom-version pom)) + (artifactid (pom-artifactid pom)) + (groupid (pom-groupid pom))) + (loop modules + (add-local-package local-packages groupid artifactid version))))))) + (define (group->dir group) "Convert a group ID to a directory path." (string-join (string-split group #\.) "/")) (define* (fix-pom-dependencies pom-file inputs #:key with-plugins? with-build-dependencies? - (excludes '()) (local-packages '())) + with-modules? (excludes '()) + (local-packages '())) "Open @var{pom-file}, and override its content, rewritting its dependencies to set their version to the latest version available in the @var{inputs}. @@ -290,8 +340,24 @@ Returns nothing, but overrides the @var{pom-file} as a side-effect." `((http://maven.apache.org/POM/4.0.0:build ,(fix-build build)) ,@(fix-pom rest)) (cons tag (fix-pom rest)))) + (('http://maven.apache.org/POM/4.0.0:modules modules ...) + (if with-modules? + `((http://maven.apache.org/POM/4.0.0:modules ,(fix-modules modules)) + ,@(fix-pom rest)) + (cons tag (fix-pom rest)))) (tag (cons tag (fix-pom rest))))))) + (define fix-modules + (match-lambda + ('() '()) + ((tag rest ...) + (match tag + (('http://maven.apache.org/POM/4.0.0:module module) + (if (file-exists? (string-append (dirname pom-file) "/" module "/pom.xml")) + `((http://maven.apache.org/POM/4.0.0:module ,module) ,@(fix-modules rest)) + (fix-modules rest))) + (tag (cons tag (fix-modules rest))))))) + (define fix-dep-management (match-lambda ('() '()) @@ -325,8 +391,27 @@ Returns nothing, but overrides the @var{pom-file} as a side-effect." `((http://maven.apache.org/POM/4.0.0:plugins ,(fix-plugins plugins)) ,@(fix-build rest))) + (('http://maven.apache.org/POM/4.0.0:extensions extensions ...) + `((http://maven.apache.org/POM/4.0.0:extensions + ,(fix-extensions extensions)) + ,@(fix-build rest))) (tag (cons tag (fix-build rest))))))) + (define* (fix-extensions extensions #:optional optional?) + (match extensions + ('() '()) + ((tag rest ...) + (match tag + (('http://maven.apache.org/POM/4.0.0:extension extension ...) + (let ((group (or (pom-groupid extension) "org.apache.maven.plugins")) + (artifact (pom-artifactid extension))) + (if (member artifact (or (assoc-ref excludes group) '())) + (fix-extensions rest optional?) + `((http://maven.apache.org/POM/4.0.0:extension + ,(fix-plugin extension optional?)); extensions are similar to plugins + ,@(fix-extensions rest optional?))))) + (tag (cons tag (fix-extensions rest optional?))))))) + (define fix-management (match-lambda ('() '()) @@ -344,7 +429,7 @@ Returns nothing, but overrides the @var{pom-file} as a side-effect." ((tag rest ...) (match tag (('http://maven.apache.org/POM/4.0.0:plugin plugin ...) - (let ((group (or (pom-groupid plugin inputs) "org.apache.maven.plugins")) + (let ((group (or (pom-groupid plugin) "org.apache.maven.plugins")) (artifact (pom-artifactid plugin))) (if (member artifact (or (assoc-ref excludes group) '())) (fix-plugins rest optional?) @@ -355,11 +440,11 @@ Returns nothing, but overrides the @var{pom-file} as a side-effect." (define* (fix-plugin plugin #:optional optional?) (let* ((artifact (pom-artifactid plugin)) - (group (or (pom-groupid plugin inputs) "org.apache.maven.plugins")) + (group (or (pom-groupid plugin) "org.apache.maven.plugins")) (version (or (assoc-ref (assoc-ref local-packages group) artifact) (find-version inputs group artifact optional?) - (pom-version plugin inputs)))) - (if (pom-version plugin inputs) + (pom-version plugin)))) + (if (pom-version plugin) (map (lambda (tag) (match tag @@ -373,7 +458,7 @@ Returns nothing, but overrides the @var{pom-file} as a side-effect." (define* (fix-dep dep #:optional optional?) (let* ((artifact (pom-artifactid dep)) - (group (or (pom-groupid dep inputs) (pom-groupid pom inputs))) + (group (or (pom-groupid dep) (pom-groupid pom))) (scope (pom-ref dep "scope")) (is-optional? (equal? (pom-ref dep "optional") '("true")))) (format (current-error-port) "maven: ~a:~a :: ~a (optional: ~a)~%" @@ -382,8 +467,8 @@ Returns nothing, but overrides the @var{pom-file} as a side-effect." with-build-dependencies?) (let ((version (or (assoc-ref (assoc-ref local-packages group) artifact) (find-version inputs group artifact optional?) - (pom-version dep inputs)))) - (if (pom-version dep inputs) + (pom-version dep)))) + (if (pom-version dep) (map (lambda (tag) (match tag @@ -396,7 +481,7 @@ Returns nothing, but overrides the @var{pom-file} as a side-effect." (cons `(http://maven.apache.org/POM/4.0.0:version ,version) dep))) dep))) - (define* (find-version inputs group artifact #:optional optional?) + (define (find-packaged-version inputs group artifact) (let* ((directory (string-append "lib/m2/" (group->dir group) "/" artifact)) (java-inputs (filter @@ -408,15 +493,22 @@ Returns nothing, but overrides the @var{pom-file} as a side-effect." (versions (append-map ls java-inputs)) (versions (sort versions version>?))) (if (null? versions) - (if optional? #f - (begin - (format (current-error-port) "maven: ~a:~a is missing from inputs~%" - group artifact) - (throw 'no-such-input group artifact))) - (car versions)))) + (car versions)))) + + (define* (find-version inputs group artifact #:optional optional?) + (let ((packaged-version (find-packaged-version inputs group artifact)) + (local-version (assoc-ref (assoc-ref local-packages group) artifact))) + (or local-version packaged-version + (if optional? + #f + (begin + (format (current-error-port) "maven: ~a:~a is missing from inputs~%" + group artifact) + (throw 'no-such-input group artifact)))))) (let ((tmpfile (string-append pom-file ".tmp"))) - (with-output-to-file pom-file + (with-output-to-file tmpfile (lambda _ - (sxml->xml (fix-maven-xml (fix-pom pom))))))) + (sxml->xml (fix-maven-xml (fix-pom pom))))) + (rename-file tmpfile pom-file))) diff --git a/guix/build/minetest-build-system.scm b/guix/build/minetest-build-system.scm new file mode 100644 index 0000000000..477cc3d1d0 --- /dev/null +++ b/guix/build/minetest-build-system.scm @@ -0,0 +1,229 @@ +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build minetest-build-system) + #:use-module (guix build utils) + #:use-module (srfi srfi-1) + #:use-module (ice-9 format) + #:use-module (ice-9 match) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 receive) + #:use-module (ice-9 regex) + #:use-module ((guix build gnu-build-system) #:prefix gnu:) + #:use-module ((guix build copy-build-system) #:prefix copy:) + #:export (%standard-phases + mod-install-plan minimise-png read-mod-name check)) + +;; (guix build copy-build-system) does not export 'install'. +(define copy:install + (assoc-ref copy:%standard-phases 'install)) + +(define (mod-install-plan mod-name) + `(("." ,(string-append "share/minetest/mods/" mod-name) + ;; Only install files that will actually be used at run time. + ;; This can save a little disk space. + ;; + ;; See <https://github.com/minetest/minetest/blob/master/doc/lua_api.txt> + ;; for an incomple list of files that can be found in mods. + #:include ("mod.conf" "modpack.conf" "settingtypes.txt" "depends.txt" + "description.txt") + #:include-regexp (".lua$" ".png$" ".ogg$" ".obj$" ".b3d$" ".tr$" + ".mts$")))) + +(define* (guess-mod-name #:key inputs #:allow-other-keys) + "Try to determine the name of the mod or modpack that is being built. +If it is unknown, make an educated guess." + ;; Minetest doesn't care about the directory names in "share/minetest/mods" + ;; so there is no technical problem if the directory names don't match + ;; the mod names. The directory can appear in the GUI if the modpack + ;; doesn't have the 'name' set though, so try to make a guess. + (define (guess) + (let* ((source (assoc-ref inputs "source")) + ;; Don't retain a reference to the store. + (file-name (strip-store-file-name source)) + ;; The "minetest-" prefix is not informative, so strip it. + (file-name (if (string-prefix? "minetest-" file-name) + (substring file-name (string-length "minetest-")) + file-name)) + ;; Strip "-checkout" suffixes of git checkouts. + (file-name (if (string-suffix? "-checkout" file-name) + (substring file-name + 0 + (- (string-length file-name) + (string-length "-checkout"))) + file-name)) + (first-dot (string-index file-name #\.)) + ;; If the source code is in an archive (.tar.gz, .zip, ...), + ;; strip the extension. + (file-name (if first-dot + (substring file-name 0 first-dot) + file-name))) + (format (current-error-port) + "warning: the modpack ~a did not set 'name' in 'modpack.conf'~%" + file-name) + file-name)) + (cond ((file-exists? "mod.conf") + ;; Mods must have 'name' set in "mod.conf", so don't guess. + (read-mod-name "mod.conf")) + ((file-exists? "modpack.conf") + ;; While it is recommended to have 'name' set in 'modpack.conf', + ;; it is optional, so guess a name if necessary. + (read-mod-name "modpack.conf" guess)) + (#t (guess)))) + +(define* (install #:key inputs #:allow-other-keys #:rest arguments) + (apply copy:install + #:install-plan (mod-install-plan (apply guess-mod-name arguments)) + arguments)) + +(define %png-magic-bytes + ;; Magic bytes of PNG images, see ‘5.2 PNG signatures’ in + ;; ‘Portable Network Graphics (PNG) Specification (Second Edition)’ + ;; on <https://www.w3.org/TR/PNG/>. + #vu8(137 80 78 71 13 10 26 10)) + +(define png-file? + ((@@ (guix build utils) file-header-match) %png-magic-bytes)) + +(define* (minimise-png #:key inputs native-inputs #:allow-other-keys) + "Minimise PNG images found in the working directory." + (define optipng (which "optipng")) + (define (optimise image) + (format #t "Optimising ~a~%" image) + (make-file-writable (dirname image)) + (make-file-writable image) + (define old-size (stat:size (stat image))) + ;; The mod "technic" has a file "technic_music_player_top.png" that + ;; actually is a JPEG file, see + ;; <https://github.com/minetest-mods/technic/issues/590>. + (if (png-file? image) + (invoke optipng "-o4" "-quiet" image) + (format #t "warning: skipping ~a because it's not actually a PNG image~%" + image)) + (define new-size (stat:size (stat image))) + (values old-size new-size)) + (define files (find-files "." ".png$")) + (let loop ((total-old-size 0) + (total-new-size 0) + (images (find-files "." ".png$"))) + (cond ((pair? images) + (receive (old-size new-size) + (optimise (car images)) + (loop (+ total-old-size old-size) + (+ total-new-size new-size) + (cdr images)))) + ((= total-old-size 0) + (format #t "There were no PNG images to minimise.")) + (#t + (format #t "Minimisation reduced size of images by ~,2f% (~,2f MiB to ~,2f MiB)~%" + (* 100.0 (- 1 (/ total-new-size total-old-size))) + (/ total-old-size (expt 1024 2)) + (/ total-new-size (expt 1024 2))))))) + +(define name-regexp (make-regexp "^name[ ]*=(.+)$")) + +(define* (read-mod-name mod.conf #:optional not-found) + "Read the name of a mod from MOD.CONF. If MOD.CONF +does not have a name field and NOT-FOUND is #false, raise an +error. If NOT-FOUND is TRUE, call NOT-FOUND instead." + (call-with-input-file mod.conf + (lambda (port) + (let loop () + (define line (read-line port)) + (if (eof-object? line) + (if not-found + (not-found) + (error "~a does not have a 'name' field" mod.conf)) + (let ((match (regexp-exec name-regexp line))) + (if (regexp-match? match) + (string-trim-both (match:substring match 1) #\ ) + (loop)))))))) + +(define* (check #:key outputs tests? #:allow-other-keys) + "Test whether the mod loads. The mod must first be installed first." + (define (all-mod-names directories) + (append-map + (lambda (directory) + (map read-mod-name (find-files directory "mod.conf"))) + directories)) + (when tests? + (mkdir "guix_testworld") + ;; Add the mod to the mod search path, such that Minetest can find it. + (setenv "MINETEST_MOD_PATH" + (list->search-path-as-string + (cons + (string-append (assoc-ref outputs "out") "/share/minetest/mods") + (search-path-as-string->list + (or (getenv "MINETEST_MOD_PATH") ""))) + ":")) + (with-directory-excursion "guix_testworld" + (setenv "HOME" (getcwd)) + ;; Create a world in which all mods are loaded. + (call-with-output-file "world.mt" + (lambda (port) + (display + "gameid = minetest +world_name = guix_testworld +backend = sqlite3 +player_backend = sqlite3 +auth_backend = sqlite3 +" port) + (for-each + (lambda (mod) + (format port "load_mod_~a = true~%" mod)) + (all-mod-names (search-path-as-string->list + (getenv "MINETEST_MOD_PATH")))))) + (receive (port pid) + ((@@ (guix build utils) open-pipe-with-stderr) + "xvfb-run" "--" "minetest" "--info" "--world" "." "--go") + (format #t "Started Minetest with all mods loaded for testing~%") + ;; Scan the output for error messages. + ;; When the player has joined the server, stop minetest. + (define (error? line) + (and (string? line) + (string-contains line ": ERROR["))) + (define (stop? line) + (and (string? line) + (string-contains line "ACTION[Server]: singleplayer [127.0.0.1] joins game."))) + (let loop () + (match (read-line port) + ((? error? line) + (error "minetest raised an error: ~a" line)) + ((? stop?) + (kill pid SIGINT) + (close-port port) + (waitpid pid)) + ((? string? line) + (display line) + (newline) + (loop)) + ((? eof-object?) + (error "minetest didn't start")))))))) + +(define %standard-phases + (modify-phases gnu:%standard-phases + (delete 'bootstrap) + (delete 'configure) + (add-before 'build 'minimise-png minimise-png) + (delete 'build) + (delete 'check) + (replace 'install install) + ;; The 'check' phase requires the mod to be installed, + ;; so move the 'check' phase after the 'install' phase. + (add-after 'install 'check check))) + +;;; minetest-build-system.scm ends here diff --git a/guix/build/minify-build-system.scm b/guix/build/minify-build-system.scm index 92158a033f..c5a876726f 100644 --- a/guix/build/minify-build-system.scm +++ b/guix/build/minify-build-system.scm @@ -39,7 +39,7 @@ (format #t "minifying ~a\n" file) (let* ((base (basename file ".js")) (installed (or target (string-append directory base ".min.js"))) - (minified (open-pipe* OPEN_READ "uglify-js" file))) + (minified (open-pipe* OPEN_READ "uglifyjs" file))) (call-with-output-file installed (cut dump-port minified <>)) #t)) diff --git a/guix/build/node-build-system.scm b/guix/build/node-build-system.scm index a55cab237c..70a367618e 100644 --- a/guix/build/node-build-system.scm +++ b/guix/build/node-build-system.scm @@ -120,7 +120,14 @@ #t) (define* (repack #:key inputs #:allow-other-keys) - (invoke "tar" "-czf" "../package.tgz" ".") + (invoke "tar" + ;; Add options suggested by https://reproducible-builds.org/docs/archives/ + "--sort=name" + (string-append "--mtime=@" (getenv "SOURCE_DATE_EPOCH")) + "--owner=0" + "--group=0" + "--numeric-owner" + "-czf" "../package.tgz" ".") #t) (define* (install #:key outputs inputs #:allow-other-keys) diff --git a/guix/build/pack.scm b/guix/build/pack.scm new file mode 100644 index 0000000000..3b73d1b227 --- /dev/null +++ b/guix/build/pack.scm @@ -0,0 +1,54 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix build pack) + #:use-module (guix build utils) + #:export (tar-base-options)) + +(define* (tar-base-options #:key tar compressor) + "Return the base GNU tar options required to produce deterministic archives +deterministically. When TAR, a GNU tar command file name, is provided, the +`--sort' option is used only if supported. When COMPRESSOR, a command such as +'(\"gzip\" \"-9n\"), is provided, the compressor is explicitly specified via +the `-I' option." + (define (tar-supports-sort? tar) + (with-error-to-port (%make-void-port "w") + (lambda () + (zero? (system* tar "cf" "/dev/null" "--files-from=/dev/null" + "--sort=name"))))) + + `(,@(if compressor + (list "-I" (string-join compressor)) + '()) + ;; The --sort option was added to GNU tar in version 1.28, released + ;; 2014-07-28. For testing, we use the bootstrap tar, which is older + ;; and doesn't support it. + ,@(if (and=> tar tar-supports-sort?) + '("--sort=name") + '()) + ;; Use GNU format so there's no file name length limitation. + "--format=gnu" + "--mtime=@1" + "--owner=root:0" + "--group=root:0" + ;; The 'nlink' of the store item files leads tar to store hard links + ;; instead of actual copies. However, the 'nlink' count depends on + ;; deduplication in the store; it's an "implicit input" to the build + ;; process. Use '--hard-dereference' to eliminate it. + "--hard-dereference" + "--check-links")) diff --git a/guix/build/profiles.scm b/guix/build/profiles.scm index a40c3f96de..f9875ca92e 100644 --- a/guix/build/profiles.scm +++ b/guix/build/profiles.scm @@ -20,6 +20,8 @@ #:use-module (guix build union) #:use-module (guix build utils) #:use-module (guix search-paths) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:use-module (ice-9 ftw) #:use-module (ice-9 match) @@ -143,45 +145,71 @@ instead make DIRECTORY a \"real\" directory containing symlinks." directory)))) (apply throw args)))))) -(define* (build-profile output inputs - #:key manifest search-paths - (symlink symlink)) - "Build a user profile from INPUTS in directory OUTPUT, using SYMLINK to -create symlinks. Write MANIFEST, an sexp, to OUTPUT/manifest. Create -OUTPUT/etc/profile with Bash definitions for -all the variables listed in -SEARCH-PATHS." +(define (manifest-sexp->inputs+search-paths manifest) + "Parse MANIFEST, an sexp as produced by 'manifest->gexp', and return two +values: the list of store items of its manifest entries, and the list of +search path specifications." + (match manifest ;this must match 'manifest->gexp' + (('manifest ('version 3) + ('packages (entries ...))) + (let loop ((entries entries) + (inputs '()) + (search-paths '())) + (match entries + (((name version output item + ('propagated-inputs deps) + ('search-paths paths) _ ...) . rest) + (loop (append rest deps) ;breadth-first traversal + (cons item inputs) + (append paths search-paths))) + (() + (values (reverse inputs) + (delete-duplicates + (cons $PATH + (map sexp->search-path-specification + (reverse search-paths))))))))))) + +(define* (build-profile output manifest + #:key (extra-inputs '()) (symlink symlink)) + "Build a user profile from MANIFEST, an sexp, and EXTRA-INPUTS, a list of +store items, in directory OUTPUT, using SYMLINK to create symlinks. Create +OUTPUT/etc/profile with Bash definitions for all the variables listed in the +search paths of MANIFEST's entries." (define manifest-file (string-append output "/manifest")) - ;; Make the symlinks. - (union-build output inputs - #:symlink symlink - #:log-port (%make-void-port "w")) + (let-values (((inputs search-paths) + (manifest-sexp->inputs+search-paths manifest))) - ;; If one of the INPUTS provides a '/manifest' file, delete it. That can - ;; happen if MANIFEST contains something such as a Guix instance, which is - ;; ultimately built as a profile. - (when (file-exists? manifest-file) - (delete-file manifest-file)) + ;; Make the symlinks. + (union-build output (append extra-inputs inputs) + #:symlink symlink + #:log-port (%make-void-port "w")) - ;; Store meta-data. - (call-with-output-file manifest-file - (lambda (p) - (display "\ + ;; If one of the INPUTS provides a '/manifest' file, delete it. That can + ;; happen if MANIFEST contains something such as a Guix instance, which is + ;; ultimately built as a profile. + (when (file-exists? manifest-file) + (delete-file manifest-file)) + + ;; Store meta-data. + (call-with-output-file manifest-file + (lambda (p) + (display "\ ;; This file was automatically generated and is for internal use only. ;; It cannot be passed to the '--manifest' option. ;; Run 'guix package --export-manifest' if you want to export a file ;; suitable for '--manifest'.\n\n" - p) - (pretty-print manifest p))) + p) + (pretty-print manifest p))) - ;; Make sure we can write to 'OUTPUT/etc'. 'union-build' above could have - ;; made 'etc' a symlink to a read-only sub-directory in the store so we need - ;; to work around that. - (ensure-writable-directory (string-append output "/etc") - #:symlink symlink) + ;; Make sure we can write to 'OUTPUT/etc'. 'union-build' above could have + ;; made 'etc' a symlink to a read-only sub-directory in the store so we + ;; need to work around that. + (ensure-writable-directory (string-append output "/etc") + #:symlink symlink) - ;; Write 'OUTPUT/etc/profile'. - (build-etc/profile output search-paths)) + ;; Write 'OUTPUT/etc/profile'. + (build-etc/profile output search-paths))) ;;; profile.scm ends here diff --git a/guix/build/qt-build-system.scm b/guix/build/qt-build-system.scm index b0d6ddafac..a6955ce4c2 100644 --- a/guix/build/qt-build-system.scm +++ b/guix/build/qt-build-system.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 Federico Beffa <beffa@fbengineering.ch> -;;; Copyright © 2014, 2015, 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2019, 2020, 2021 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; @@ -48,97 +48,6 @@ (setenv "CTEST_OUTPUT_ON_FAILURE" "1") #t) -(define (variables-for-wrapping base-directories) - - (define (collect-sub-dirs base-directories file-type subdirectory - selectors) - ;; Append SUBDIRECTORY and each of BASE-DIRECTORIES, and return the subset - ;; that exists and has at least one of the SELECTORS sub-directories, - ;; unless SELECTORS is the empty list. FILE-TYPE should by 'directory or - ;; 'regular file. For the later, it allows searching for plain files - ;; rather than directories. - (define exists? (match file-type - ('directory directory-exists?) - ('regular file-exists?))) - - (filter-map (lambda (dir) - (let ((directory (string-append dir subdirectory))) - (and (exists? directory) - (or (null? selectors) - (any (lambda (selector) - (exists? - (string-append directory selector))) - selectors)) - directory))) - base-directories)) - - (filter-map - (match-lambda - ((variable file-type directory selectors ...) - (match (collect-sub-dirs base-directories file-type directory - selectors) - (() - #f) - (directories - `(,variable = ,directories))))) - - ;; These shall match the search-path-specification for Qt and KDE - ;; libraries. - (list '("XDG_DATA_DIRS" directory "/share" - - ;; These are "selectors": consider /share if and only if at least - ;; one of these sub-directories exist. This avoids adding - ;; irrelevant packages to XDG_DATA_DIRS just because they have a - ;; /share sub-directory. - "/glib-2.0/schemas" "/sounds" "/themes" - "/cursors" "/wallpapers" "/icons" "/mime") - '("XDG_CONFIG_DIRS" directory "/etc/xdg") - '("QT_PLUGIN_PATH" directory "/lib/qt5/plugins") - '("QML2_IMPORT_PATH" directory "/lib/qt5/qml") - '("QTWEBENGINEPROCESS_PATH" regular - "/lib/qt5/libexec/QtWebEngineProcess")))) - -(define* (wrap-all-programs #:key inputs outputs - (qt-wrap-excluded-outputs '()) - #:allow-other-keys) - "Implement phase \"qt-wrap\": look for GSettings schemas and -gtk+-v.0 libraries and create wrappers with suitably set environment variables -if found. - -Wrapping is not applied to outputs whose name is listed in -QT-WRAP-EXCLUDED-OUTPUTS. This is useful when an output is known not -to contain any Qt binaries, and where wrapping would gratuitously -add a dependency of that output on Qt." - (define (find-files-to-wrap directory) - (append-map - (lambda (dir) - (if (directory-exists? dir) (find-files dir ".*") (list))) - (list (string-append directory "/bin") - (string-append directory "/sbin") - (string-append directory "/libexec") - (string-append directory "/lib/libexec")))) - - (define input-directories - ;; FIXME: Filter out unwanted inputs, e.g. cmake - (match inputs - (((_ . dir) ...) - dir))) - - (define handle-output - (match-lambda - ((output . directory) - (unless (member output qt-wrap-excluded-outputs) - (let ((bin-list (find-files-to-wrap directory)) - (vars-to-wrap (variables-for-wrapping - (append (list directory) - input-directories)))) - (when (not (null? vars-to-wrap)) - (for-each (cut apply wrap-program <> vars-to-wrap) - bin-list))))))) - - (for-each handle-output outputs) - #t) - (define %standard-phases (modify-phases cmake:%standard-phases (add-before 'check 'check-setup check-setup) diff --git a/guix/build/qt-utils.scm b/guix/build/qt-utils.scm index 8e6db10ca1..c2b80cab7d 100644 --- a/guix/build/qt-utils.scm +++ b/guix/build/qt-utils.scm @@ -2,6 +2,8 @@ ;;; Copyright © 2016 David Craven <david@craven.ch> ;;; Copyright © 2019, 2020, 2021 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net> +;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -36,40 +38,57 @@ ;; contain any of the standard subdirectories. (define (variables-for-wrapping base-directories output-directory) - (define (collect-sub-dirs base-directories subdirectory-spec) - (filter-map - (lambda (dir) - (match - subdirectory-spec - ((subdir) - (and (directory-exists? (string-append dir subdir)) - (string-append dir (car subdirectory-spec)))) - ((subdir children) - (and - (or - (and (string=? dir output-directory) - (directory-exists? (string-append dir subdir))) - (or-map - (lambda (kid) (directory-exists? (string-append dir subdir kid))) - children)) - (string-append dir subdir))))) - base-directories)) + (define (collect-sub-dirs base-directories file-type subdirectory selectors) + ;; Append SUBDIRECTORY and each of BASE-DIRECTORIES, and return the subset + ;; that exists and has at least one of the SELECTORS sub-directories, + ;; unless SELECTORS is the empty list. FILE-TYPE should by 'directory or + ;; 'regular file. For the later, it allows searching for plain files + ;; rather than directories. + (define exists? (match file-type + ('directory directory-exists?) + ('regular file-exists?))) - (filter - (lambda (var-to-wrap) (not (null? (last var-to-wrap)))) - (map - (match-lambda - ((var kind . subdir-spec) - `(,var ,kind ,(collect-sub-dirs base-directories subdir-spec)))) - (list - ;; these shall match the search-path-specification for Qt and KDE - ;; libraries - '("XDG_DATA_DIRS" suffix "/share" ("/applications" "/fonts" - "/icons" "/mime")) - '("XDG_CONFIG_DIRS" suffix "/etc/xdg") - '("QT_PLUGIN_PATH" prefix "/lib/qt5/plugins") - '("QML2_IMPORT_PATH" prefix "/lib/qt5/qml"))))) + (filter-map (lambda (dir) + (let ((directory (string-append dir subdirectory))) + (and (exists? directory) + (or (null? selectors) + (any (lambda (selector) + (exists? + (string-append directory selector))) + selectors)) + directory))) + base-directories)) + (filter-map + (match-lambda + ((variable type file-type directory selectors ...) + (match (collect-sub-dirs base-directories file-type directory selectors) + (() + #f) + (directories + `(,variable ,type ,directories))))) + ;; These shall match the search-path-specification for Qt and KDE + ;; libraries. + (list + ;; The XDG environment variables are defined with the 'suffix type, which + ;; allows the users to override or extend their value, so that custom icon + ;; themes can be honored, for example. + '("XDG_DATA_DIRS" suffix directory "/share" + ;; These are "selectors": consider /share if and only if at least + ;; one of these sub-directories exist. This avoids adding + ;; irrelevant packages to XDG_DATA_DIRS just because they have a + ;; /share sub-directory. + "/applications" "/cursors" "/fonts" "/icons" "/glib-2.0/schemas" + "/mime" "/sounds" "/themes" "/wallpapers") + '("XDG_CONFIG_DIRS" suffix directory "/etc/xdg") + ;; The following variables can be extended by the user, but not + ;; overridden, to ensure proper operation. + '("QT_PLUGIN_PATH" prefix directory "/lib/qt5/plugins") + '("QML2_IMPORT_PATH" prefix directory "/lib/qt5/qml") + ;; QTWEBENGINEPROCESS_PATH accepts a single value, which makes 'exact the + ;; most suitable environment variable type for it. + '("QTWEBENGINEPROCESS_PATH" = regular + "/lib/qt5/libexec/QtWebEngineProcess")))) (define* (wrap-qt-program* program #:key inputs output-dir qt-wrap-excluded-inputs) @@ -88,7 +107,6 @@ (when (not (null? vars-to-wrap)) (apply wrap-program program vars-to-wrap)))) - (define* (wrap-qt-program program-name #:key inputs output (qt-wrap-excluded-inputs %qt-wrap-excluded-inputs)) "Wrap the specified programm (which must reside in the OUTPUT's \"/bin\" @@ -100,7 +118,6 @@ is wrapped." #:output-dir output #:inputs inputs #:qt-wrap-excluded-inputs qt-wrap-excluded-inputs)) - (define* (wrap-all-qt-programs #:key inputs outputs (qt-wrap-excluded-outputs '()) (qt-wrap-excluded-inputs %qt-wrap-excluded-inputs) @@ -131,5 +148,4 @@ add a dependency of that output on Qt." #:qt-wrap-excluded-inputs qt-wrap-excluded-inputs) (find-files-to-wrap output-dir)))))) - (for-each handle-output outputs) - #t) + (for-each handle-output outputs)) diff --git a/guix/build/renpy-build-system.scm b/guix/build/renpy-build-system.scm index 66683971c5..e4a88456be 100644 --- a/guix/build/renpy-build-system.scm +++ b/guix/build/renpy-build-system.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2021 Leo Prikler <leo.prikler@student.tugraz.at> +;;; Copyright © 2021 Liliana Marie Prikler <liliana.prikler@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; diff --git a/guix/build/syscalls.scm b/guix/build/syscalls.scm index 8886fc0fb9..99a3b45004 100644 --- a/guix/build/syscalls.scm +++ b/guix/build/syscalls.scm @@ -7,6 +7,7 @@ ;;; Copyright © 2020 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2020 Jan (janneke) Nieuwenhuizen <janneke@gnu.org> ;;; Copyright © 2021 Chris Marusich <cmmarusich@gmail.com> +;;; Copyright © 2021 Tobias Geerinckx-Rice <me@tobias.gr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -56,6 +57,9 @@ restart-on-EINTR + device-number + device-number->major+minor + mount? mount-device-number mount-source @@ -450,6 +454,29 @@ the returned procedure is called." ;;; +;;; Block devices. +;;; + +;; Convert between major:minor pairs and packed ‘device number’ representation. +;; XXX These aren't syscalls, but if you squint very hard they are part of the +;; FFI or however you want to justify me not finding a better fit… :-) +(define (device-number major minor) ; see glibc's <sys/sysmacros.h> + "Return the device number for the device with MAJOR and MINOR, for use as +the last argument of `mknod'." + (logior (ash (logand #x00000fff major) 8) + (ash (logand #xfffff000 major) 32) + (logand #x000000ff minor) + (ash (logand #xffffff00 minor) 12))) + +(define (device-number->major+minor device) ; see glibc's <sys/sysmacros.h> + "Return two values: the major and minor device numbers that make up DEVICE." + (values (logior (ash (logand #x00000000000fff00 device) -8) + (ash (logand #xfffff00000000000 device) -32)) + (logior (logand #x00000000000000ff device) + (ash (logand #x00000ffffff00000 device) -12)))) + + +;;; ;;; File systems. ;;; @@ -628,7 +655,7 @@ current process." (define (string->device-number str) (match (string-split str #\:) (((= string->number major) (= string->number minor)) - (+ (* major 256) minor)))) + (device-number major minor)))) (call-with-input-file "/proc/self/mountinfo" (lambda (port) @@ -2236,8 +2263,8 @@ correspond to a terminal, return the value returned by FALL-BACK." ;; would return EINVAL instead in some cases: ;; <https://bugs.ruby-lang.org/issues/10494>. ;; Furthermore, some FUSE file systems like unionfs return ENOSYS for - ;; that ioctl, and bcachefs returns EPERM. - (if (memv errno (list ENOTTY EINVAL ENOSYS EPERM)) + ;; that ioctl. + (if (memv errno (list ENOTTY EINVAL ENOSYS)) (fall-back) (apply throw args)))))) diff --git a/guix/channels.scm b/guix/channels.scm index c40fc0c507..e4e0428eb5 100644 --- a/guix/channels.scm +++ b/guix/channels.scm @@ -657,10 +657,11 @@ that unconditionally resumes the continuation." store)))) (define* (build-from-source instance - #:key core verbose? (dependencies '())) + #:key core verbose? (dependencies '()) system) "Return a derivation to build Guix from INSTANCE, using the self-build script contained therein. When CORE is true, build package modules under -SOURCE using CORE, an instance of Guix." +SOURCE using CORE, an instance of Guix. By default, build for the current +system, or SYSTEM if specified." (define name (symbol->string (channel-name (channel-instance-channel instance)))) @@ -700,20 +701,22 @@ SOURCE using CORE, an instance of Guix." (with-trivial-build-handler (build source #:verbose? verbose? #:version commit + #:system system #:channel-metadata (channel-instance->sexp instance) #:pull-version %pull-version)))) ;; Build a set of modules that extend Guix using the standard method. (standard-module-derivation name source core dependencies))) -(define* (build-channel-instance instance +(define* (build-channel-instance instance system #:optional core (dependencies '())) "Return, as a monadic value, the derivation for INSTANCE, a channel -instance. DEPENDENCIES is a list of extensions providing Guile modules that -INSTANCE depends on." +instance, for SYSTEM. DEPENDENCIES is a list of extensions providing Guile +modules that INSTANCE depends on." (build-from-source instance #:core core - #:dependencies dependencies)) + #:dependencies dependencies + #:system system)) (define (resolve-dependencies instances) "Return a procedure that, given one of the elements of INSTANCES, returns @@ -743,9 +746,9 @@ list of instances it depends on." (lambda (instance) (vhash-foldq* cons '() instance edges))) -(define (channel-instance-derivations instances) +(define* (channel-instance-derivations instances #:key system) "Return the list of derivations to build INSTANCES, in the same order as -INSTANCES." +INSTANCES. Build for the current system by default, or SYSTEM if specified." (define core-instance ;; The 'guix' channel is treated specially: it's an implicit dependency of ;; all the other channels. @@ -757,13 +760,13 @@ INSTANCES." (resolve-dependencies instances)) (define (instance->derivation instance) - (mlet %store-monad ((system (current-system))) + (mlet %store-monad ((system (if system (return system) (current-system)))) (mcached (if (eq? instance core-instance) - (build-channel-instance instance) + (build-channel-instance instance system) (mlet %store-monad ((core (instance->derivation core-instance)) (deps (mapm %store-monad instance->derivation (edges instance)))) - (build-channel-instance instance core deps))) + (build-channel-instance instance system core deps))) instance system))) @@ -865,9 +868,10 @@ derivation." intro)))))) '())))) -(define (channel-instances->manifest instances) +(define* (channel-instances->manifest instances #:key system) "Return a profile manifest with entries for all of INSTANCES, a list of -channel instances." +channel instances. By default, build for the current system, or SYSTEM if +specified." (define (instance->entry instance drv) (let ((commit (channel-instance-commit instance)) (channel (channel-instance-channel instance))) @@ -883,7 +887,8 @@ channel instances." (properties `((source ,(channel-instance->sexp instance))))))) - (mlet* %store-monad ((derivations (channel-instance-derivations instances)) + (mlet* %store-monad ((derivations (channel-instance-derivations instances + #:system system)) (entries -> (map instance->entry instances derivations))) (return (manifest entries)))) @@ -1084,8 +1089,13 @@ cannot be found." (if (channel-news-entry-commit entry) entry (let* ((tag (channel-news-entry-tag entry)) - (reference (string-append "refs/tags/" tag)) - (oid (reference-name->oid repository reference))) + (reference (reference-lookup repository + (string-append "refs/tags/" tag))) + (target (reference-target reference)) + (oid (let ((obj (object-lookup repository target))) + (if (= OBJ-TAG (object-type obj)) ;annotated tag? + (tag-target-id (tag-lookup repository target)) + target)))) (channel-news-entry (oid->string oid) tag (channel-news-entry-title entry) (channel-news-entry-body entry))))) diff --git a/guix/ci.scm b/guix/ci.scm index f04109112c..01b493b3af 100644 --- a/guix/ci.scm +++ b/guix/ci.scm @@ -20,9 +20,12 @@ (define-module (guix ci) #:use-module (guix http-client) #:use-module (guix utils) + #:use-module ((guix build download) + #:select (resolve-uri-reference)) #:use-module (json) #:use-module (srfi srfi-1) #:use-module (ice-9 match) + #:use-module (web uri) #:use-module (guix i18n) #:use-module (guix diagnostics) #:autoload (guix channels) (channel) @@ -43,7 +46,7 @@ checkout? checkout-commit - checkout-input + checkout-channel evaluation? evaluation-id @@ -51,10 +54,24 @@ evaluation-complete? evaluation-checkouts + job? + job-build-id + job-status + job-name + + history? + history-evaluation + history-checkouts + history-jobs + %query-limit queued-builds latest-builds evaluation + evaluation-jobs + build + job-build + jobs-history latest-evaluations evaluations-for-commit @@ -75,13 +92,31 @@ (file-size build-product-file-size) ;integer (path build-product-path)) ;string +(define-syntax-rule (define-enumeration-mapping proc + (names integers) ...) + (define (proc value) + (match value + (integers 'names) ...))) + +(define-enumeration-mapping integer->build-status + ;; Copied from 'build-status' in Cuirass. + (submitted -3) + (scheduled -2) + (started -1) + (succeeded 0) + (failed 1) + (failed-dependency 2) + (failed-other 3) + (canceled 4)) + (define-json-mapping <build> make-build build? json->build (id build-id "id") ;integer (derivation build-derivation) ;string | #f (evaluation build-evaluation) ;integer (system build-system) ;string - (status build-status "buildstatus" ) ;integer + (status build-status "buildstatus" ;symbol + integer->build-status) (timestamp build-timestamp) ;integer (products build-products "buildproducts" ;<build-product>* (lambda (products) @@ -91,16 +126,35 @@ (vector->list products) '()))))) +(define-json-mapping <job> make-job job? + json->job + (build-id job-build-id "build") ;integer + (status job-status "status" ;symbol + integer->build-status) + (name job-name)) ;string + +(define-json-mapping <history> make-history history? + json->history + (evaluation history-evaluation) ;integer + (checkouts history-checkouts "checkouts" ;<checkout>* + (lambda (checkouts) + (map json->checkout + (vector->list checkouts)))) + (jobs history-jobs "jobs" + (lambda (jobs) + (map json->job + (vector->list jobs))))) + (define-json-mapping <checkout> make-checkout checkout? json->checkout (commit checkout-commit) ;string (SHA1) - (input checkout-input)) ;string (name) + (channel checkout-channel)) ;string (name) (define-json-mapping <evaluation> make-evaluation evaluation? json->evaluation (id evaluation-id) ;integer (spec evaluation-spec "specification") ;string - (complete? evaluation-complete? "in-progress" + (complete? evaluation-complete? "status" (match-lambda (0 #t) (_ #f))) ;Boolean @@ -113,16 +167,44 @@ ;; Max number of builds requested in queries. 1000) +(define* (api-url base-url path #:rest query) + "Build a proper API url, taking into account BASE-URL's trailing slashes. +QUERY takes any number of '(\"name\" value) 2-element lists, with VALUE being +either a string or a number (which will be converted to a string). If VALUE +is #f, the respective element will not be added to the query parameters. +Other types of VALUE will raise an error since this low-level function is +api-agnostic." + + (define (build-query-string query) + (let lp ((query (or (reverse query) '())) (acc '())) + (match query + (() (string-concatenate acc)) + (((_ #f) . rest) (lp rest acc)) + (((name val) . rest) + (lp rest (cons* + name "=" + (if (string? val) (uri-encode val) (number->string val)) + (if (null? acc) "" "&") + acc)))))) + + (let* ((query-string (build-query-string query)) + (base (string->uri base-url)) + (ref (build-relative-ref #:path path #:query query-string))) + (resolve-uri-reference ref base))) + (define (json-fetch url) (let* ((port (http-fetch url)) (json (json->scm port))) (close-port port) json)) +(define* (json-api-fetch base-url path #:rest query) + (json-fetch (apply api-url base-url path query))) + (define* (queued-builds url #:optional (limit %query-limit)) "Return the list of queued derivations on URL." - (let ((queue (json-fetch (string-append url "/api/queue?nr=" - (number->string limit))))) + (let ((queue + (json-api-fetch url "/api/queue" `("nr" ,limit)))) (map json->build (vector->list queue)))) (define* (latest-builds url #:optional (limit %query-limit) @@ -130,38 +212,32 @@ "Return the latest builds performed by the CI server at URL. If EVALUATION is an integer, restrict to builds of EVALUATION. If SYSTEM is true (a system string such as \"x86_64-linux\"), restrict to builds for SYSTEM." - (define* (option name value #:optional (->string identity)) - (if value - (string-append "&" name "=" (->string value)) - "")) - - (let ((latest (json-fetch (string-append url "/api/latestbuilds?nr=" - (number->string limit) - (option "evaluation" evaluation - number->string) - (option "system" system) - (option "job" job) - (option "status" status - number->string))))) + (let ((latest (json-api-fetch + url "/api/latestbuilds" + `("nr" ,limit) + `("evaluation" ,evaluation) + `("system" ,system) + `("job" ,job) + `("status" ,status)))) ;; Note: Hydra does not provide a "derivation" field for entries in ;; 'latestbuilds', but Cuirass does. (map json->build (vector->list latest)))) (define (evaluation url evaluation) "Return the given EVALUATION performed by the CI server at URL." - (let ((evaluation (json-fetch - (string-append url "/api/evaluation?id=" - (number->string evaluation))))) + (let ((evaluation + (json-api-fetch url "/api/evaluation" `("id" ,evaluation)))) (json->evaluation evaluation))) -(define* (latest-evaluations url #:optional (limit %query-limit)) - "Return the latest evaluations performed by the CI server at URL." +(define* (latest-evaluations url + #:optional (limit %query-limit) + #:key spec) + "Return the latest evaluations performed by the CI server at URL. If SPEC +is passed, only consider the evaluations for the given SPEC specification." (map json->evaluation (vector->list - (json->scm - (http-fetch (string-append url "/api/evaluations?nr=" - (number->string limit))))))) - + (json-api-fetch + url "/api/evaluations" `("nr" ,limit) `("spec" ,spec))))) (define* (evaluations-for-commit url commit #:optional (limit %query-limit)) "Return the evaluations among the latest LIMIT evaluations that have COMMIT @@ -172,6 +248,38 @@ as one of their inputs." (evaluation-checkouts evaluation))) (latest-evaluations url limit))) +(define (evaluation-jobs url evaluation-id) + "Return the list of jobs of evaluation EVALUATION-ID." + (map json->job + (vector->list + (json-api-fetch url "/api/jobs" `("evaluation" ,evaluation-id))))) + +(define (build url id) + "Look up build ID at URL and return it. Raise &http-get-error if it is not +found (404)." + (json->build + (http-fetch (api-url url (string-append "/build/" ;note: no "/api" here + (number->string id)))))) + +(define (job-build url job) + "Return the build associated with JOB." + (build url (job-build-id job))) + +(define* (jobs-history url jobs + #:key + (specification "master") + (limit 20)) + "Return the job history for the SPECIFICATION jobs which names are part of +the JOBS list, from the CI server at URL. Limit the history to the latest +LIMIT evaluations. " + (let ((names (string-join jobs ","))) + (map json->history + (vector->list + (json->scm + (http-fetch + (format #f "~a/api/jobs/history?spec=~a&names=~a&nr=~a" + url specification names (number->string limit)))))))) + (define (find-latest-commit-with-substitutes url) "Return the latest commit with available substitutes for the Guix package definitions at URL. Return false if no commit were found." diff --git a/guix/cpio.scm b/guix/cpio.scm index c9932f5bf9..d4a7d5f1e0 100644 --- a/guix/cpio.scm +++ b/guix/cpio.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Tobias Geerinckx-Rice <me@tobias.gr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -17,6 +18,8 @@ ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. (define-module (guix cpio) + #:use-module ((guix build syscalls) #:select (device-number + device-number->major+minor)) #:use-module ((guix build utils) #:select (dump-port)) #:use-module (srfi srfi-9) #:use-module (srfi srfi-11) @@ -128,8 +131,8 @@ (nlink 1) (mtime 0) (size 0) (dev 0) (rdev 0) (name-size 0)) "Return a new cpio file header." - (let-values (((major minor) (device->major+minor dev)) - ((rmajor rminor) (device->major+minor rdev))) + (let-values (((major minor) (device-number->major+minor dev)) + ((rmajor rminor) (device-number->major+minor rdev))) (%make-cpio-header MAGIC inode mode uid gid nlink mtime @@ -153,16 +156,6 @@ denotes, similar to 'stat:type'." (else (error "unsupported file type" mode))))) -(define (device-number major minor) ;see <sys/sysmacros.h> - "Return the device number for the device with MAJOR and MINOR, for use as -the last argument of `mknod'." - (+ (* major 256) minor)) - -(define (device->major+minor device) - "Return two values: the major and minor device numbers that make up DEVICE." - (values (ash device -8) - (logand device #xff))) - (define* (file->cpio-header file #:optional (file-name file) #:key (stat lstat)) "Return a cpio header corresponding to the info returned by STAT for FILE, diff --git a/guix/cve.scm b/guix/cve.scm index b3a8b13a06..9e1cf5b587 100644 --- a/guix/cve.scm +++ b/guix/cve.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -99,7 +99,9 @@ (define (reference-data->cve-references alist) (map json->cve-reference - (vector->list (assoc-ref alist "reference_data")))) + ;; Normally "reference_data" is always present but rejected CVEs such + ;; as CVE-2020-10020 can lack it. + (vector->list (or (assoc-ref alist "reference_data") '#())))) (define %cpe-package-rx ;; For applications: "cpe:2.3:a:VENDOR:PACKAGE:VERSION", or sometimes @@ -137,17 +139,20 @@ package." (starte (assoc-ref alist "versionStartExcluding")) (endi (assoc-ref alist "versionEndIncluding")) (ende (assoc-ref alist "versionEndExcluding"))) - (let-values (((package version) (cpe->package-name cpe))) - (and package - `(,package - ,(cond ((and (or starti starte) (or endi ende)) - `(and ,(if starti `(>= ,starti) `(> ,starte)) - ,(if endi `(<= ,endi) `(< ,ende)))) - (starti `(>= ,starti)) - (starte `(> ,starte)) - (endi `(<= ,endi)) - (ende `(< ,ende)) - (else version))))))) + ;; Normally "cpe23Uri" is here in each "cpe_match" item, but CVE-2020-0534 + ;; has a configuration that lacks it. + (and cpe + (let-values (((package version) (cpe->package-name cpe))) + (and package + `(,package + ,(cond ((and (or starti starte) (or endi ende)) + `(and ,(if starti `(>= ,starti) `(> ,starte)) + ,(if endi `(<= ,endi) `(< ,ende)))) + (starti `(>= ,starti)) + (starte `(> ,starte)) + (endi `(<= ,endi)) + (ende `(< ,ende)) + (else version)))))))) (define (configuration-data->cve-configurations alist) "Given ALIST, a JSON dictionary for the baroque \"configurations\" diff --git a/guix/cvs-download.scm b/guix/cvs-download.scm index 76b3eac739..943d971622 100644 --- a/guix/cvs-download.scm +++ b/guix/cvs-download.scm @@ -28,7 +28,8 @@ #:use-module (ice-9 match) #:export (cvs-reference cvs-reference? - cvs-reference-url + cvs-reference-root-directory + cvs-reference-module cvs-reference-revision cvs-fetch)) @@ -63,13 +64,20 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." (define guile-zlib (module-ref (resolve-interface '(gnu packages guile)) 'guile-zlib)) + (define guile-json + (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + + (define gnutls + (module-ref (resolve-interface '(gnu packages tls)) 'gnutls)) + (define modules (delete '(guix config) (source-module-closure '((guix build cvs) (guix build download-nar))))) (define build (with-imported-modules modules - (with-extensions (list guile-zlib) + (with-extensions (list guile-json gnutls ;for (guix swh) + guile-zlib) #~(begin (use-modules (guix build cvs) (guix build download-nar)) diff --git a/guix/derivations.scm b/guix/derivations.scm index 2fe684cc18..33f4dc5d9d 100644 --- a/guix/derivations.scm +++ b/guix/derivations.scm @@ -241,32 +241,29 @@ the store." "Return a list of inputs, such that when INPUTS contains the same DRV twice, they are coalesced, with their sub-derivations merged. This is needed because Nix itself keeps only one of them." - (define (find pred lst) ;inlinable copy of 'find' - (let loop ((lst lst)) - (match lst - (() #f) - ((head . tail) - (if (pred head) head (loop tail)))))) + (define table + (make-hash-table 25)) - (fold (lambda (input result) - (match input - (($ <derivation-input> (= derivation-file-name path) sub-drvs) - ;; XXX: quadratic - (match (find (match-lambda - (($ <derivation-input> (= derivation-file-name p) - s) - (string=? p path))) - result) - (#f - (cons input result)) - ((and dup ($ <derivation-input> drv sub-drvs2)) - ;; Merge DUP with INPUT. - (let ((sub-drvs (delete-duplicates - (append sub-drvs sub-drvs2)))) - (cons (make-derivation-input drv (sort sub-drvs string<?)) - (delq dup result)))))))) - '() - inputs)) + (for-each (lambda (input) + (let* ((drv (derivation-input-path input)) + (sub-drvs (derivation-input-sub-derivations input))) + (match (hash-get-handle table drv) + (#f + (hash-set! table drv input)) + ((and handle (key . ($ <derivation-input> drv sub-drvs2))) + ;; Merge DUP with INPUT. + (let* ((sub-drvs (delete-duplicates + (append sub-drvs sub-drvs2))) + (input + (make-derivation-input drv + (sort sub-drvs string<?)))) + (set-cdr! handle input)))))) + inputs) + + (hash-fold (lambda (key input lst) + (cons input lst)) + '() + table)) (define* (derivation-prerequisites drv #:optional (cut? (const #f))) "Return the list of derivation-inputs required to build DRV, recursively. diff --git a/guix/describe.scm b/guix/describe.scm index 0683ad8a27..65cd79094b 100644 --- a/guix/describe.scm +++ b/guix/describe.scm @@ -115,22 +115,35 @@ lives in, or the empty list if this is not applicable." "Return manifest entries corresponding to extra channels--i.e., not the 'guix' channel." (remove (lambda (entry) - (string=? (manifest-entry-name entry) "guix")) + (or (string=? (manifest-entry-name entry) "guix") + + ;; If ENTRY lacks the 'source' property, it's not an entry + ;; from 'guix pull'. See <https://bugs.gnu.org/48778>. + (not (assq 'source (manifest-entry-properties entry))))) (current-profile-entries)))) (define current-channels (mlambda () "Return the list of channels currently available, including the 'guix' channel. Return the empty list if this information is missing." + (define (build-time-metadata) + (match (channel-metadata) + (#f '()) + (sexp (or (and=> (sexp->channel sexp 'guix) list) '())))) + (match (current-profile-entries) (() ;; As a fallback, if we're not running from a profile, use 'guix' ;; channel metadata from (guix config). - (match (channel-metadata) - (#f '()) - (sexp (or (and=> (sexp->channel sexp 'guix) list) '())))) + (build-time-metadata)) (entries - (filter-map manifest-entry-channel entries))))) + (match (filter-map manifest-entry-channel entries) + (() + ;; This profile lacks provenance metadata, so fall back to + ;; build-time metadata as returned by 'channel-metadata'. + (build-time-metadata)) + (lst + lst)))))) (define (package-path-entries) "Return two values: the list of package path entries to be added to the diff --git a/guix/diagnostics.scm b/guix/diagnostics.scm index 7b9ffc61b5..6a792febd4 100644 --- a/guix/diagnostics.scm +++ b/guix/diagnostics.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -233,6 +233,10 @@ etc." (make-location file (+ line 1) col))) (#f #f) + (#(file line column) + ;; Guile >= 3.0.6 uses vectors instead of alists internally, which can be + ;; seen in the arguments to 'syntax-error' exceptions. + (location file (+ 1 line) column)) (_ (let ((file (assq-ref loc 'filename)) (line (assq-ref loc 'line)) diff --git a/guix/discovery.scm b/guix/discovery.scm index b84b9ff370..81d4ca600f 100644 --- a/guix/discovery.scm +++ b/guix/discovery.scm @@ -107,19 +107,25 @@ name and the exception key and arguments." (define prefix-len (string-length directory)) - (filter-map (lambda (file) - (let* ((relative (string-drop file prefix-len)) - (module (file-name->module-name relative))) - (catch #t - (lambda () - (resolve-interface module)) - (lambda args - ;; Report the error, but keep going. - (warn file module args) - #f)))) - (scheme-files (if sub-directory - (string-append directory "/" sub-directory) - directory)))) + ;; Hide Guile warnings such as "source file [...] newer than compiled" when + ;; loading user code, unless we're hacking on Guix proper. See + ;; <https://issues.guix.gnu.org/43747>. + (parameterize ((current-warning-port (if (getenv "GUIX_UNINSTALLED") + (current-warning-port) + (%make-void-port "w")))) + (filter-map (lambda (file) + (let* ((relative (string-drop file prefix-len)) + (module (file-name->module-name relative))) + (catch #t + (lambda () + (resolve-interface module)) + (lambda args + ;; Report the error, but keep going. + (warn file module args) + #f)))) + (scheme-files (if sub-directory + (string-append directory "/" sub-directory) + directory))))) (define* (scheme-modules* directory #:optional sub-directory) "Return the list of module names found under SUB-DIRECTORY in DIRECTORY. diff --git a/guix/docker.scm b/guix/docker.scm index 889aaeacb5..a6f73d423c 100644 --- a/guix/docker.scm +++ b/guix/docker.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2017, 2018, 2019, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,6 +22,7 @@ (define-module (guix docker) #:use-module (gcrypt hash) #:use-module (guix base16) + #:use-module (guix build pack) #:use-module ((guix build utils) #:select (mkdir-p delete-file-recursively @@ -58,8 +60,13 @@ (container_config . #nil))) (define (canonicalize-repository-name name) - "\"Repository\" names are restricted to roughtl [a-z0-9_.-]. + "\"Repository\" names are restricted to roughly [a-z0-9_.-]. Return a version of TAG that follows these rules." + ;; Refer to https://docs.docker.com/docker-hub/repos/. + (define min-length 2) + (define padding-character #\a) + (define max-length 255) + (define ascii-letters (string->char-set "abcdefghijklmnopqrstuvwxyz")) @@ -69,11 +76,21 @@ Return a version of TAG that follows these rules." (define repo-char-set (char-set-union char-set:digit ascii-letters separators)) - (string-map (lambda (chr) - (if (char-set-contains? repo-char-set chr) - chr - #\.)) - (string-trim (string-downcase name) separators))) + (define normalized-name + (string-map (lambda (chr) + (if (char-set-contains? repo-char-set chr) + chr + #\.)) + (string-trim (string-downcase name) separators))) + + (let ((l (string-length normalized-name))) + (match l + ((? (cut > <> max-length)) + (string-take normalized-name max-length)) + ((? (cut < <> min-length)) + (string-append normalized-name + (make-string (- min-length l) padding-character))) + (_ normalized-name)))) (define* (manifest path id #:optional (tag "guix")) "Generate a simple image manifest." @@ -110,18 +127,6 @@ Return a version of TAG that follows these rules." (rootfs . ((type . "layers") (diff_ids . #(,(layer-diff-id layer))))))) -(define %tar-determinism-options - ;; GNU tar options to produce archives deterministically. - '("--sort=name" "--mtime=@1" - "--owner=root:0" "--group=root:0" - - ;; When 'build-docker-image' is passed store items, the 'nlink' of the - ;; files therein leads tar to store hard links instead of actual copies. - ;; However, the 'nlink' count depends on deduplication in the store; it's - ;; an "implicit input" to the build process. '--hard-dereference' - ;; eliminates it. - "--hard-dereference")) - (define directive-file ;; Return the file or directory created by a 'evaluate-populate-directive' ;; directive. @@ -238,7 +243,7 @@ SRFI-19 time-utc object, as the creation time in metadata." (apply invoke "tar" "-cf" "../layer.tar" `(,@transformation-options - ,@%tar-determinism-options + ,@(tar-base-options) ,@paths ,@(scandir "." (lambda (file) @@ -273,9 +278,6 @@ SRFI-19 time-utc object, as the creation time in metadata." (scm->json (repositories prefix id repository))))) (apply invoke "tar" "-cf" image "-C" directory - `(,@%tar-determinism-options - ,@(if compressor - (list "-I" (string-join compressor)) - '()) + `(,@(tar-base-options #:compressor compressor) ".")) (delete-file-recursively directory))) diff --git a/guix/download.scm b/guix/download.scm index 30f69c0325..85b97a4766 100644 --- a/guix/download.scm +++ b/guix/download.scm @@ -35,6 +35,7 @@ #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:export (%mirrors + %disarchive-mirrors (url-fetch* . url-fetch) url-fetch/executable url-fetch/tarbomb @@ -87,10 +88,8 @@ "ftp://ftp.ring.gr.jp/pub/net/gnupg/" "ftp://ftp.gnupg.org/gcrypt/") (gnome - "http://ftp.belnet.be/ftp.gnome.org/" - "http://ftp.linux.org.uk/mirrors/ftp.gnome.org/" - "http://ftp.gnome.org/pub/GNOME/" "https://download.gnome.org/" + "http://ftp.gnome.org/pub/GNOME/" "http://mirror.yandex.ru/mirrors/ftp.gnome.org/") (hackage "http://hackage.haskell.org/") @@ -370,7 +369,7 @@ ;; procedure that takes a file name, an algorithm (symbol) and a hash ;; (bytevector), and returns a URL or #f. '(begin - (use-modules (guix base32)) + (use-modules (guix base16) (guix base32)) (define (guix-publish host) (lambda (file algo hash) @@ -380,12 +379,6 @@ file "/" (symbol->string algo) "/" (bytevector->nix-base32-string hash)))) - ;; XXX: (guix base16) appeared in March 2017 (and thus 0.13.0) so old - ;; installations of the daemon might lack it. Thus, load it lazily to - ;; avoid gratuitous errors. See <https://bugs.gnu.org/33542>. - (module-autoload! (current-module) - '(guix base16) '(bytevector->base16-string)) - (list (guix-publish "ci.guix.gnu.org") (lambda (file algo hash) ;; 'tarballs.nixos.org' supports several algorithms. @@ -406,12 +399,21 @@ (plain-file "content-addressed-mirrors" (object->string %content-addressed-mirrors))) +(define %disarchive-mirrors + ;; TODO: Eventually turn into a procedure that takes a hash algorithm + ;; (symbol) and hash (bytevector). + '("https://disarchive.ngyro.com/")) + +(define %disarchive-mirror-file + (plain-file "disarchive-mirrors" (object->string %disarchive-mirrors))) + (define built-in-builders* (store-lift built-in-builders)) (define* (built-in-download file-name url #:key system hash-algo hash mirrors content-addressed-mirrors + disarchive-mirrors executable? (guile 'unused)) "Download FILE-NAME from URL using the built-in 'download' builder. When @@ -422,13 +424,16 @@ explicitly depend on Guile, GnuTLS, etc. Instead, the daemon performs the download by itself using its own dependencies." (mlet %store-monad ((mirrors (lower-object mirrors)) (content-addressed-mirrors - (lower-object content-addressed-mirrors))) + (lower-object content-addressed-mirrors)) + (disarchive-mirrors (lower-object disarchive-mirrors))) (raw-derivation file-name "builtin:download" '() #:system system #:hash-algo hash-algo #:hash hash #:recursive? executable? - #:sources (list mirrors content-addressed-mirrors) + #:sources (list mirrors + content-addressed-mirrors + disarchive-mirrors) ;; Honor the user's proxy and locale settings. #:leaked-env-vars '("http_proxy" "https_proxy" @@ -439,6 +444,7 @@ download by itself using its own dependencies." ("mirrors" . ,mirrors) ("content-addressed-mirrors" . ,content-addressed-mirrors) + ("disarchive-mirrors" . ,disarchive-mirrors) ,@(if executable? '(("executable" . "1")) '())) @@ -492,7 +498,9 @@ name in the store." #:executable? executable? #:mirrors %mirror-file #:content-addressed-mirrors - %content-addressed-mirror-file))))) + %content-addressed-mirror-file + #:disarchive-mirrors + %disarchive-mirror-file))))) (define* (url-fetch/executable url hash-algo hash #:optional name diff --git a/guix/gexp.scm b/guix/gexp.scm index afb935761e..f3d278b3e6 100644 --- a/guix/gexp.scm +++ b/guix/gexp.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org> ;;; Copyright © 2019, 2020 Mathieu Othacehe <m.othacehe@gmail.com> ;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> ;;; ;;; This file is part of GNU Guix. ;;; @@ -42,6 +43,7 @@ with-imported-modules with-extensions let-system + gexp->approximate-sexp gexp-input gexp-input? @@ -157,6 +159,23 @@ "Return the source code location of GEXP." (and=> (%gexp-location gexp) source-properties->location)) +(define* (gexp->approximate-sexp gexp) + "Return the S-expression corresponding to GEXP, but do not lower anything. +As a result, the S-expression will be approximate if GEXP has references." + (define (gexp-like? thing) + (or (gexp? thing) (gexp-input? thing))) + (apply (gexp-proc gexp) + (map (lambda (reference) + (match reference + (($ <gexp-input> thing output native) + (if (gexp-like? thing) + (gexp->approximate-sexp thing) + ;; Simply returning 'thing' won't work in some + ;; situations; see 'write-gexp' below. + '(*approximate*))) + (_ '(*approximate*)))) + (gexp-references gexp)))) + (define (write-gexp gexp port) "Write GEXP on PORT." (display "#<gexp " port) @@ -1921,6 +1940,7 @@ This is the declarative counterpart of 'text-file*'." (define build (gexp (call-with-output-file (ungexp output "out") (lambda (port) + (set-port-encoding! port "UTF-8") (display (string-append (ungexp-splicing text)) port))))) (computed-file name build)) diff --git a/guix/git-download.scm b/guix/git-download.scm index 425184717a..5e624b9ae9 100644 --- a/guix/git-download.scm +++ b/guix/git-download.scm @@ -27,11 +27,15 @@ #:use-module (guix packages) #:use-module (guix modules) #:autoload (guix build-system gnu) (standard-packages) + #:autoload (git bindings) (libgit2-init!) #:autoload (git repository) (repository-open repository-close! repository-discover repository-head repository-working-directory) + #:autoload (git submodule) (repository-submodules + submodule-lookup + submodule-path) #:autoload (git commit) (commit-lookup commit-tree) #:autoload (git reference) (reference-target) #:autoload (git tree) (tree-list) @@ -193,11 +197,17 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." ;;; 'git-predicate'. ;;; -(define (git-file-list directory) +(define* (git-file-list directory #:optional prefix #:key (recursive? #t)) "Return the list of files checked in in the Git repository at DIRECTORY. The result is similar to that of the 'git ls-files' command, except that it -also includes directories, not just regular files. The returned file names -are relative to DIRECTORY, which is not necessarily the root of the checkout." +also includes directories, not just regular files. + +When RECURSIVE? is true, also list files in submodules, similar to the 'git +ls-files --recurse-submodules' command. This is enabled by default. + +The returned file names are relative to DIRECTORY, which is not necessarily +the root of the checkout. If a PREFIX is provided, it is prepended to each +file name." (let* (;; 'repository-working-directory' always returns a trailing "/", ;; so add one here to ease the comparisons below. (directory (string-append (canonicalize-path directory) "/")) @@ -208,32 +218,65 @@ are relative to DIRECTORY, which is not necessarily the root of the checkout." (oid (reference-target head)) (commit (commit-lookup repository oid)) (tree (commit-tree commit)) - (files (tree-list tree))) + (files (tree-list tree)) + (submodules (if recursive? + (map (lambda (name) + (submodule-path + (submodule-lookup repository name))) + (repository-submodules repository)) + '())) + (relative (and (not (string=? workdir directory)) + (string-drop directory (string-length workdir)))) + (included? (lambda (path) + (or (not relative) + (string-prefix? relative path)))) + (make-relative (lambda (path) + (if relative + (string-drop path (string-length relative)) + path))) + (add-prefix (lambda (path) + (if prefix + (string-append prefix "/" path) + path))) + (rectify (compose add-prefix make-relative))) (repository-close! repository) - (if (string=? workdir directory) - files - (let ((relative (string-drop directory (string-length workdir)))) - (filter-map (lambda (file) - (and (string-prefix? relative file) - (string-drop file (string-length relative)))) - files))))) + (append + (if (or relative prefix) + (filter-map (lambda (file) + (and (included? file) + (rectify file))) + files) + files) + (append-map (lambda (submodule) + (if (included? submodule) + (git-file-list + (string-append workdir submodule) + (rectify submodule)) + '())) + submodules)))) -(define (git-predicate directory) +(define* (git-predicate directory #:key (recursive? #t)) "Return a predicate that returns true if a file is part of the Git checkout living at DIRECTORY. If DIRECTORY does not lie within a Git checkout, and upon Git errors, return #f instead of a predicate. +When RECURSIVE? is true, the predicate also returns true if a file is part of +any Git submodule under DIRECTORY. This is enabled by default. + The returned predicate takes two arguments FILE and STAT where FILE is an absolute file name and STAT is the result of 'lstat'." + (libgit2-init!) (catch 'git-error (lambda () - (let* ((files (git-file-list directory)) + (let* ((files (git-file-list directory #:recursive? recursive?)) (inodes (fold (lambda (file result) - (let ((stat - (lstat (string-append directory "/" - file)))) - (vhash-consv (stat:ino stat) (stat:dev stat) - result))) + (let* ((file (string-append directory "/" file)) + (stat (false-if-exception (lstat file)))) + ;; Ignore FILE if it has been deleted. + (if stat + (vhash-consv (stat:ino stat) (stat:dev stat) + result) + result))) vlist-null files))) (lambda (file stat) diff --git a/guix/git.scm b/guix/git.scm index 776b03f33a..dc2ca1be84 100644 --- a/guix/git.scm +++ b/guix/git.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2017, 2020 Mathieu Othacehe <m.othacehe@gmail.com> ;;; Copyright © 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2021 Kyle Meyer <kyle@kyleam.com> +;;; Copyright © 2021 Marius Bakke <marius@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -33,8 +34,9 @@ #:use-module (guix records) #:use-module (guix gexp) #:use-module (guix sets) - #:use-module ((guix diagnostics) #:select (leave)) + #:use-module ((guix diagnostics) #:select (leave warning)) #:use-module (guix progress) + #:autoload (guix swh) (swh-download commit-id?) #:use-module (rnrs bytevectors) #:use-module (ice-9 format) #:use-module (ice-9 match) @@ -56,6 +58,8 @@ commit-difference commit-relation + remote-refs + git-checkout git-checkout? git-checkout-url @@ -179,6 +183,13 @@ the 'SSL_CERT_FILE' and 'SSL_CERT_DIR' environment variables." (lambda args (make-fetch-options auth-method))))) +(define GITERR_HTTP + ;; Guile-Git <= 0.5.2 lacks this constant. + (let ((errors (resolve-interface '(git errors)))) + (if (module-defined? errors 'GITERR_HTTP) + (module-ref errors 'GITERR_HTTP) + 34))) + (define (clone* url directory) "Clone git repository at URL into DIRECTORY. Upon failure, make sure no empty directory is left behind." @@ -223,15 +234,29 @@ corresponding Git object." (object-lookup-prefix repository (string->oid commit) len) (object-lookup repository (string->oid commit))))) (('tag-or-commit . str) - (if (or (> (string-length str) 40) - (not (string-every char-set:hex-digit str))) - (resolve `(tag . ,str)) ;definitely a tag - (catch 'git-error - (lambda () - (resolve `(tag . ,str))) - (lambda _ - ;; There's no such tag, so it must be a commit ID. - (resolve `(commit . ,str)))))) + (cond ((and (string-contains str "-g") + (match (string-split str #\-) + ((version ... revision g+commit) + (if (and (> (string-length g+commit) 4) + (string-every char-set:digit revision) + (string-every char-set:hex-digit + (string-drop g+commit 1))) + ;; Looks like a 'git describe' style ID, like + ;; v1.3.0-7-gaa34d4d28d. + (string-drop g+commit 1) + #f)) + (_ #f))) + => (lambda (commit) (resolve `(commit . ,commit)))) + ((or (> (string-length str) 40) + (not (string-every char-set:hex-digit str))) + (resolve `(tag . ,str))) ;definitely a tag + (else + (catch 'git-error + (lambda () + (resolve `(tag . ,str))) + (lambda _ + ;; There's no such tag, so it must be a commit ID. + (resolve `(commit . ,str))))))) (('tag . tag) (let ((oid (reference-name->oid repository (string-append "refs/tags/" tag)))) @@ -283,13 +308,15 @@ dynamic extent of EXP." (report-git-error err)))) (define* (update-submodules repository - #:key (log-port (current-error-port))) + #:key (log-port (current-error-port)) + (fetch-options #f)) "Update the submodules of REPOSITORY, a Git repository object." (for-each (lambda (name) (let ((submodule (submodule-lookup repository name))) (format log-port (G_ "updating submodule '~a'...~%") name) - (submodule-update submodule) + (submodule-update submodule + #:fetch-options fetch-options) ;; Recurse in SUBMODULE. (let ((directory (string-append @@ -297,6 +324,7 @@ dynamic extent of EXP." "/" (submodule-path submodule)))) (with-repository directory repository (update-submodules repository + #:fetch-options fetch-options #:log-port log-port))))) (repository-submodules repository))) @@ -314,7 +342,8 @@ dynamic extent of EXP." "Return true if REF, a reference such as '(commit . \"cabba9e\"), is definitely available in REPOSITORY, false otherwise." (match ref - (('commit . commit) + ((or ('commit . commit) + ('tag-or-commit . (? commit-id? commit))) (let ((len (string-length commit)) (oid (string->oid commit))) (false-if-git-not-found @@ -324,6 +353,42 @@ definitely available in REPOSITORY, false otherwise." (_ #f))) +(define (clone-from-swh url tag-or-commit output) + "Attempt to clone TAG-OR-COMMIT (a string), which originates from URL, using +a copy archived at Software Heritage." + (call-with-temporary-directory + (lambda (bare) + (and (swh-download url tag-or-commit bare + #:archive-type 'git-bare) + (let ((repository (clone* bare output))) + (remote-set-url! repository "origin" url) + repository))))) + +(define (clone/swh-fallback url ref cache-directory) + "Like 'clone', but fallback to Software Heritage if the repository cannot be +found at URL." + (define (inaccessible-url-error? err) + (let ((class (git-error-class err)) + (code (git-error-code err))) + (or (= class GITERR_HTTP) ;404 or similar + (= class GITERR_NET)))) ;unknown host, etc. + + (catch 'git-error + (lambda () + (clone* url cache-directory)) + (lambda (key err) + (match ref + (((or 'commit 'tag-or-commit) . commit) + (if (inaccessible-url-error? err) + (or (clone-from-swh url commit cache-directory) + (begin + (warning (G_ "revision ~a of ~a \ +could not be fetched from Software Heritage~%") + commit url) + (throw key err))) + (throw key err))) + (_ (throw key err)))))) + (define cached-checkout-expiration ;; Return the expiration time procedure for a cached checkout. ;; TODO: Honor $GUIX_GIT_CACHE_EXPIRATION. @@ -390,14 +455,15 @@ it unchanged." (let* ((cache-exists? (openable-repository? cache-directory)) (repository (if cache-exists? (repository-open cache-directory) - (clone* url cache-directory)))) + (clone/swh-fallback url ref cache-directory)))) ;; Only fetch remote if it has not been cloned just before. (when (and cache-exists? (not (reference-available? repository ref))) (remote-fetch (remote-lookup repository "origin") #:fetch-options (make-default-fetch-options))) (when recursive? - (update-submodules repository #:log-port log-port)) + (update-submodules repository #:log-port log-port + #:fetch-options (make-default-fetch-options))) ;; Note: call 'commit-relation' from here because it's more efficient ;; than letting users re-open the checkout later on. @@ -420,6 +486,14 @@ it unchanged." ;; REPOSITORY as soon as possible. (repository-close! repository) + ;; Update CACHE-DIRECTORY's mtime to so the cache logic sees it. + (match (gettimeofday) + ((seconds . microseconds) + (let ((nanoseconds (* 1000 microseconds))) + (utime cache-directory + seconds seconds + nanoseconds nanoseconds)))) + ;; When CACHE-DIRECTORY is a sub-directory of the default cache ;; directory, remove expired checkouts that are next to it. (let ((parent (dirname cache-directory))) @@ -544,6 +618,45 @@ objects: 'ancestor (meaning that OLD is an ancestor of NEW), 'descendant, or (if (set-contains? oldest new) 'descendant 'unrelated)))))) + +;; +;;; Remote operations. +;;; + +(define* (remote-refs url #:key tags?) + "Return the list of references advertised at Git repository URL. If TAGS? +is true, limit to only refs/tags." + (define (ref? ref) + ;; Like `git ls-remote --refs', only show actual references. + (and (string-prefix? "refs/" ref) + (not (string-suffix? "^{}" ref)))) + + (define (tag? ref) + (string-prefix? "refs/tags/" ref)) + + (define (include? ref) + (and (ref? ref) + (or (not tags?) (tag? ref)))) + + (define (remote-head->ref remote) + (let ((name (remote-head-name remote))) + (and (include? name) + name))) + + (with-libgit2 + (call-with-temporary-directory + (lambda (cache-directory) + (let* ((repository (repository-init cache-directory)) + ;; Create an in-memory remote so we don't touch disk. + (remote (remote-create-anonymous repository url))) + (remote-connect remote) + + (let* ((remote-heads (remote-ls remote)) + (refs (filter-map remote-head->ref remote-heads))) + ;; Wait until we're finished with the repository before closing it. + (remote-disconnect remote) + (repository-close! repository) + refs)))))) ;;; diff --git a/guix/gnu-maintenance.scm b/guix/gnu-maintenance.scm index fece84b341..e7edbf6656 100644 --- a/guix/gnu-maintenance.scm +++ b/guix/gnu-maintenance.scm @@ -243,7 +243,8 @@ network to check in GNU's database." ;; The "-src" pattern is for "TeXmacs-1.0.7.9-src.tar.gz". ;; The "-gnu[0-9]" pattern is for "icecat-38.4.0-gnu1.tar.bz2". ;; Accept underscores as in "PKG_1.2.tar.gz" for some non-GNU packages. - (make-regexp "^([^.]+)[-_]([0-9]|[^-])+(-(src|[sS]ource|gnu[0-9]))?\\.(tar\\.|zip$)")) + ;; Accept 'v' or 'V' prefix as in 'PKG-v2.3.tgz'. + (make-regexp "^([^.]+)[-_][vV]?([0-9]|[^-])+(-(src|[sS]ource|gnu[0-9]))?\\.(tar\\.|tgz|zip$)")) (define %alpha-tarball-rx (make-regexp "^.*-.*[0-9](-|~)?(alpha|beta|rc|RC|cvs|svn|git)-?[0-9\\.]*\\.tar\\.")) @@ -495,9 +496,30 @@ are unavailable." (define (url->release url) (let* ((base (basename url)) - (url (if (string=? base url) - (string-append base-url directory "/" url) - url))) + (base-url (string-append base-url directory)) + (url (cond ((and=> (string->uri url) uri-scheme) ;full URL? + url) + ((string-prefix? "/" url) ;absolute path? + (let ((uri (string->uri base-url))) + (uri->string + (build-uri (uri-scheme uri) + #:host (uri-host uri) + #:port (uri-port uri) + #:path url)))) + + ;; URL is a relative path and BASE-URL may or may not + ;; end in slash. + ((string-suffix? "/" base-url) + (string-append base-url url)) + (else + ;; If DIRECTORY is non-empty, assume BASE-URL + ;; denotes a directory; otherwise, assume BASE-URL + ;; denotes a file within a directory, and that URL + ;; is relative to that directory. + (string-append (if (string-null? directory) + (dirname base-url) + base-url) + "/" url))))) (and (release-file? package base) (let ((version (tarball->version base))) (upstream-source @@ -596,7 +618,7 @@ list available from %GNU-FILE-LIST-URI over HTTP(S)." (define %package-name-rx ;; Regexp for a package name, e.g., "foo-X.Y". Since TeXmacs uses ;; "TeXmacs-X.Y-src", the `-src' suffix is allowed. - (make-regexp "^(.*)[-_](([0-9]|\\.)+)(-src|\\.src|\\.orig)?")) + (make-regexp "^(.*)[-_][vV]?(([0-9]|\\.)+)(-src|\\.src|\\.orig)?")) (define (gnu-package-name->name+version name+version) "Return the package name and version number extracted from NAME+VERSION." diff --git a/guix/grafts.scm b/guix/grafts.scm index fd8a108092..4c69eb35a2 100644 --- a/guix/grafts.scm +++ b/guix/grafts.scm @@ -25,10 +25,10 @@ #:use-module ((guix utils) #:select (%current-system)) #:use-module (guix sets) #:use-module (srfi srfi-1) - #:use-module (srfi srfi-11) #:use-module (srfi srfi-9 gnu) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) + #:use-module (srfi srfi-71) #:use-module (ice-9 match) #:use-module (ice-9 vlist) #:export (graft? @@ -172,10 +172,20 @@ references." items)))) (remove (cut member <> self) refs))) +(define %graft-cache + ;; Cache that maps derivation/outputs/grafts tuples to lists of grafts. + (allocate-store-connection-cache 'grafts)) + +(define record-cache-lookup! + (cache-lookup-recorder "derivation-graft-cache" + "Derivation graft cache")) + (define-syntax-rule (with-cache key exp ...) "Cache the value of monadic expression EXP under KEY." - (mlet %state-monad ((cache (current-state))) - (match (vhash-assoc key cache) + (mlet* %state-monad ((cache (current-state)) + (result -> (vhash-assoc key cache))) + (record-cache-lookup! result cache) + (match result ((_ . result) ;cache hit (return result)) (#f ;cache miss @@ -217,10 +227,10 @@ have no corresponding element in the resulting list." ((set-contains? visited drv) (loop rest items result visited)) (else - (let*-values (((inputs) - (map derivation-input-derivation - (derivation-inputs drv))) - ((result items) + (let* ((inputs + (map derivation-input-derivation + (derivation-inputs drv))) + (result items (fold2 lookup-derivers result items inputs))) (loop (append rest inputs) @@ -265,7 +275,7 @@ derivations to the corresponding set of grafts." #:system system))))) (reference-origins drv items))) - (with-cache (cons (derivation-file-name drv) outputs) + (with-cache (list (derivation-file-name drv) outputs grafts) (match (non-self-references store drv outputs) (() ;no dependencies (return grafts)) @@ -303,17 +313,25 @@ derivations to the corresponding set of grafts." "Apply GRAFTS to the OUTPUTS of DRV and all their dependencies, recursively. That is, if GRAFTS apply only indirectly to DRV, graft the dependencies of DRV, and graft DRV itself to refer to those grafted dependencies." - (match (run-with-state - (cumulative-grafts store drv grafts - #:outputs outputs - #:guile guile #:system system) - vlist-null) ;the initial cache - ((first . rest) - ;; If FIRST is not a graft for DRV, it means that GRAFTS are not - ;; applicable to DRV and nothing needs to be done. - (if (equal? drv (graft-origin first)) - (graft-replacement first) - drv)))) + (let ((grafts cache + (run-with-state + (cumulative-grafts store drv grafts + #:outputs outputs + #:guile guile #:system system) + (store-connection-cache store %graft-cache)))) + + ;; Save CACHE in STORE to benefit from it on the next call. + ;; XXX: Ideally we'd use %STORE-MONAD and 'mcached' and avoid mutating + ;; STORE. + (set-store-connection-cache! store %graft-cache cache) + + (match grafts + ((first . rest) + ;; If FIRST is not a graft for DRV, it means that GRAFTS are not + ;; applicable to DRV and nothing needs to be done. + (if (equal? drv (graft-origin first)) + (graft-replacement first) + drv))))) ;; The following might feel more at home in (guix packages) but since (guix diff --git a/guix/graph.scm b/guix/graph.scm index 0d4cd83667..3a1cab244b 100644 --- a/guix/graph.scm +++ b/guix/graph.scm @@ -337,11 +337,12 @@ nodeArray.push(nodes[\"~a\"]);~%" (define* (export-graph sinks port #:key - reverse-edges? node-type + reverse-edges? node-type (max-depth +inf.0) (backend %graphviz-backend)) "Write to PORT the representation of the DAG with the given SINKS, using the given BACKEND. Use NODE-TYPE to traverse the DAG. When REVERSE-EDGES? is -true, draw reverse arrows." +true, draw reverse arrows. Do not represent nodes whose distance to one of +the SINKS is greater than MAX-DEPTH." (match backend (($ <graph-backend> _ _ emit-prologue emit-epilogue emit-node emit-edge) (emit-prologue (node-type-name node-type) port) @@ -349,6 +350,7 @@ true, draw reverse arrows." (match node-type (($ <node-type> node-identifier node-label node-edges) (let loop ((nodes sinks) + (depths (make-list (length sinks) 0)) (visited (set))) (match nodes (() @@ -356,20 +358,29 @@ true, draw reverse arrows." (emit-epilogue port) (store-return #t))) ((head . tail) - (mlet %store-monad ((id (node-identifier head))) - (if (set-contains? visited id) - (loop tail visited) - (mlet* %store-monad ((dependencies (node-edges head)) - (ids (mapm %store-monad - node-identifier - dependencies))) - (emit-node id (node-label head) port) - (for-each (lambda (dependency dependency-id) - (if reverse-edges? - (emit-edge dependency-id id port) - (emit-edge id dependency-id port))) - dependencies ids) - (loop (append dependencies tail) - (set-insert id visited))))))))))))) + (match depths + ((depth . depths) + (mlet %store-monad ((id (node-identifier head))) + (if (set-contains? visited id) + (loop tail depths visited) + (mlet* %store-monad ((dependencies + (if (= depth max-depth) + (return '()) + (node-edges head))) + (ids + (mapm %store-monad + node-identifier + dependencies))) + (emit-node id (node-label head) port) + (for-each (lambda (dependency dependency-id) + (if reverse-edges? + (emit-edge dependency-id id port) + (emit-edge id dependency-id port))) + dependencies ids) + (loop (append dependencies tail) + (append (make-list (length dependencies) + (+ 1 depth)) + depths) + (set-insert id visited))))))))))))))) ;;; graph.scm ends here diff --git a/guix/hg-download.scm b/guix/hg-download.scm index bd55946523..382c34922a 100644 --- a/guix/hg-download.scm +++ b/guix/hg-download.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014, 2015, 2016, 2017, 2019 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,6 +26,8 @@ #:use-module (guix modules) #:use-module (guix packages) #:autoload (guix build-system gnu) (standard-packages) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) #:use-module (ice-9 match) #:use-module (ice-9 popen) #:use-module (ice-9 rdelim) @@ -32,9 +35,10 @@ hg-reference? hg-reference-url hg-reference-changeset - hg-reference-recursive? hg-predicate - hg-fetch)) + hg-fetch + hg-version + hg-file-name)) ;;; Commentary: ;;; @@ -62,26 +66,61 @@ "Return a fixed-output derivation that fetches REF, a <hg-reference> object. The output is expected to have recursive hash HASH of type HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." + (define inputs + ;; The 'swh-download' procedure requires tar and gzip. + `(("gzip" ,(module-ref (resolve-interface '(gnu packages compression)) + 'gzip)) + ("tar" ,(module-ref (resolve-interface '(gnu packages base)) + 'tar)))) + (define guile-zlib (module-ref (resolve-interface '(gnu packages guile)) 'guile-zlib)) + (define guile-json + (module-ref (resolve-interface '(gnu packages guile)) 'guile-json-4)) + + (define gnutls + (module-ref (resolve-interface '(gnu packages tls)) 'gnutls)) + (define modules (delete '(guix config) (source-module-closure '((guix build hg) - (guix build download-nar))))) + (guix build download-nar) + (guix swh))))) (define build (with-imported-modules modules - (with-extensions (list guile-zlib) + (with-extensions (list guile-json gnutls ;for (guix swh) + guile-zlib) #~(begin (use-modules (guix build hg) - (guix build download-nar)) + (guix build utils) ;for `set-path-environment-variable' + (guix build download-nar) + (guix swh) + (ice-9 match)) + + (set-path-environment-variable "PATH" '("bin") + (match '#+inputs + (((names dirs outputs ...) ...) + dirs))) + + (setvbuf (current-output-port) 'line) + (setvbuf (current-error-port) 'line) (or (hg-fetch '#$(hg-reference-url ref) '#$(hg-reference-changeset ref) #$output #:hg-command (string-append #+hg "/bin/hg")) - (download-nar #$output)))))) + (download-nar #$output) + ;; As a last resort, attempt to download from Software Heritage. + ;; Disable X.509 certificate verification to avoid depending + ;; on nss-certs--we're authenticating the checkout anyway. + (parameterize ((%verify-swh-certificate? #f)) + (format (current-error-port) + "Trying to download from Software Heritage...~%") + (swh-download #$(hg-reference-url ref) + #$(hg-reference-changeset ref) + #$output))))))) (mlet %store-monad ((guile (package->derivation guile system))) (gexp->derivation (or name "hg-checkout") build @@ -95,6 +134,23 @@ HASH-ALGO (a symbol). Use NAME as the file name, or a generic name if #f." #:recursive? #t #:guile-for-build guile))) +(define (hg-version version revision changeset) + "Return the version string for packages using hg-download." + ;; hg-version is almost exclusively executed while modules are being loaded. + ;; This makes any errors hide their backtrace. Avoid the mysterious error + ;; "Value out of range 0 to N: 7" when the commit ID is too short, which + ;; can happen, for example, when the user swapped the revision and commit + ;; arguments by mistake. + (when (< (string-length changeset) 7) + (raise + (condition + (&message (message "hg-version: changeset ID unexpectedly short"))))) + (string-append version "-" revision "." (string-take changeset 7))) + +(define (hg-file-name name version) + "Return the file-name for packages using hg-download." + (string-append name "-" version "-checkout")) + (define (hg-file-list directory) "Evaluates to a list of files contained in the repository at path @var{directory}" diff --git a/guix/http-client.scm b/guix/http-client.scm index a2e11a1b73..10bc278023 100644 --- a/guix/http-client.scm +++ b/guix/http-client.scm @@ -38,7 +38,7 @@ #:use-module (guix utils) #:use-module (guix base64) #:autoload (gcrypt hash) (sha256) - #:autoload (gnutls) (error/invalid-session) + #:autoload (gnutls) (error/invalid-session error/again error/interrupted) #:use-module ((guix build utils) #:select (mkdir-p dump-port)) #:use-module ((guix build download) @@ -163,7 +163,14 @@ reusing stale cached connections." (if (or (and (eq? key 'system-error) (= EPIPE (system-error-errno `(,key ,@args)))) (and (eq? key 'gnutls-error) - (eq? (first args) error/invalid-session)) + (memq (first args) + (list error/invalid-session + + ;; XXX: These two are not properly handled in + ;; GnuTLS < 3.7.2, in + ;; 'write_to_session_record_port'; see + ;; <https://bugs.gnu.org/47867>. + error/again error/interrupted))) (memq key '(bad-response bad-header bad-header-component))) #f @@ -207,15 +214,14 @@ returning." ;; Inherit the HTTP proxying property from P. (set-http-proxy-port?! buffer (http-proxy-port? p)) - (unless (false-if-networking-error - (begin - (for-each (cut write-request <> buffer) batch) - (put-bytevector p (get)) - (force-output p) - #t)) - ;; If PORT becomes unusable, open a fresh connection and retry. - (close-port p) ; close the broken port - (connect #f requests result))) + ;; Swallow networking errors that could occur due to connection reuse + ;; and the like; they will be handled down the road when trying to + ;; read responses. + (false-if-networking-error + (begin + (for-each (cut write-request <> buffer) batch) + (put-bytevector p (get)) + (force-output p)))) ;; Now start processing responses. (let loop ((sent batch) diff --git a/guix/import/cabal.scm b/guix/import/cabal.scm index da00019297..98d7234098 100644 --- a/guix/import/cabal.scm +++ b/guix/import/cabal.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 Federico Beffa <beffa@fbengineering.ch> ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -145,7 +146,7 @@ to the stack." (lalr-parser ;; --- token definitions (CCURLY VCCURLY OPAREN CPAREN TEST ID VERSION RELATION TRUE FALSE -ANY -NONE - (right: IF FLAG EXEC TEST-SUITE CUSTOM-SETUP SOURCE-REPO BENCHMARK LIB OCURLY) + (right: IF FLAG EXEC TEST-SUITE CUSTOM-SETUP SOURCE-REPO BENCHMARK LIB COMMON OCURLY) (left: OR) (left: PROPERTY AND) (right: ELSE NOT)) @@ -155,6 +156,7 @@ to the stack." (sections source-repo) : (append $1 (list $2)) (sections executables) : (append $1 $2) (sections test-suites) : (append $1 $2) + (sections common) : (append $1 $2) (sections custom-setup) : (append $1 $2) (sections benchmarks) : (append $1 $2) (sections lib-sec) : (append $1 (list $2)) @@ -178,6 +180,10 @@ to the stack." (ts-sec) : (list $1)) (ts-sec (TEST-SUITE OCURLY exprs CCURLY) : `(section test-suite ,$1 ,$3) (TEST-SUITE open exprs close) : `(section test-suite ,$1 ,$3)) + (common (common common-sec) : (append $1 (list $2)) + (common-sec) : (list $1)) + (common-sec (COMMON OCURLY exprs CCURLY) : `(section common ,$1 ,$3) + (COMMON open exprs close) : `(section common ,$1 ,$3)) (custom-setup (CUSTOM-SETUP exprs) : (list `(section custom-setup ,$1 ,$2))) (benchmarks (benchmarks bm-sec) : (append $1 (list $2)) (bm-sec) : (list $1)) @@ -367,6 +373,9 @@ matching a string against the created regexp." (define is-test-suite (make-rx-matcher "^test-suite +([a-z0-9_-]+)" regexp/icase)) +(define is-common (make-rx-matcher "^common +([a-z0-9_-]+)" + regexp/icase)) + (define is-custom-setup (make-rx-matcher "^(custom-setup)" regexp/icase)) @@ -391,14 +400,20 @@ matching a string against the created regexp." (define (is-or s) (string=? s "||")) -(define (is-id s port) +(define (is-id s port loc) (let ((cabal-reserved-words '("if" "else" "library" "flag" "executable" "test-suite" "custom-setup" - "source-repository" "benchmark")) + "source-repository" "benchmark" "common")) (spaces (read-while (cut char-set-contains? char-set:blank <>) port)) (c (peek-char port))) (unread-string spaces port) - (and (every (cut string-ci<> s <>) cabal-reserved-words) + ;; Sometimes the name of an identifier is the same as one of the reserved + ;; words, which would normally lead to an error, see + ;; <https://debbugs.gnu.org/cgi/bugreport.cgi?bug=25138>. Unless the word + ;; is at the beginning of a line (excluding whitespace), treat is as just + ;; another identifier instead of a reserved word. + (and (or (not (= (source-location-column loc) (current-indentation))) + (every (cut string-ci<> s <>) cabal-reserved-words)) (and (not (char=? (last (string->list s)) #\:)) (not (char=? #\: c)))))) @@ -469,6 +484,8 @@ string with the read characters." (define (lex-test-suite ts-rx-res loc) (lex-rx-res ts-rx-res 'TEST-SUITE loc)) +(define (lex-common common-rx-res loc) (lex-rx-res common-rx-res 'COMMON loc)) + (define (lex-custom-setup ts-rx-res loc) (lex-rx-res ts-rx-res 'CUSTOM-SETUP loc)) (define (lex-benchmark bm-rx-res loc) (lex-rx-res bm-rx-res 'BENCHMARK loc)) @@ -558,7 +575,7 @@ LOC is the current port location." ((is-none w) (lex-none loc)) ((is-and w) (lex-and loc)) ((is-or w) (lex-or loc)) - ((is-id w port) (lex-id w loc)) + ((is-id w port loc) (lex-id w loc)) (else (unread-string w port) #f)))) (define (lex-line port loc) @@ -570,6 +587,7 @@ the current port location." ((is-src-repo s) => (cut lex-src-repo <> loc)) ((is-exec s) => (cut lex-exec <> loc)) ((is-test-suite s) => (cut lex-test-suite <> loc)) + ((is-common s) => (cut lex-common <> loc)) ((is-custom-setup s) => (cut lex-custom-setup <> loc)) ((is-benchmark s) => (cut lex-benchmark <> loc)) ((is-lib s) (lex-lib loc)) @@ -796,7 +814,16 @@ the ordering operation and the version." (let ((value (or (assoc-ref env name) (assoc-ref (cabal-flags->alist (cabal-flags)) name)))) (if (eq? value 'false) #f #t))) + + (define common-stanzas + (filter-map (match-lambda + (('section 'common common-name common) + (cons common-name common)) + (_ #f)) + cabal-sexp)) + (define (eval sexp) + "Given an SEXP and an ENV, return the evaluated (SEXP . ENV)." (match sexp (() '()) ;; nested 'if' @@ -831,6 +858,9 @@ the ordering operation and the version." (list 'section type name (eval parameters))) (((? string? name) values) (list name values)) + ((("import" imports) rest ...) + (eval (append (append-map (cut assoc-ref common-stanzas <>) imports) + rest))) ((element rest ...) (cons (eval element) (eval rest))) (_ (raise (condition diff --git a/guix/import/cran.scm b/guix/import/cran.scm index dbc858cb84..f649928c5a 100644 --- a/guix/import/cran.scm +++ b/guix/import/cran.scm @@ -153,9 +153,9 @@ package definition." (define %cran-canonical-url "https://cran.r-project.org/package=") (define %bioconductor-url "https://bioconductor.org/packages/") -;; The latest Bioconductor release is 3.12. Bioconductor packages should be +;; The latest Bioconductor release is 3.13. Bioconductor packages should be ;; updated together. -(define %bioconductor-version "3.12") +(define %bioconductor-version "3.13") (define* (bioconductor-packages-list-url #:optional type) (string-append "https://bioconductor.org/packages/" diff --git a/guix/import/egg.scm b/guix/import/egg.scm new file mode 100644 index 0000000000..89e7a9160d --- /dev/null +++ b/guix/import/egg.scm @@ -0,0 +1,357 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; Copyright © 2021 Tobias Geerinckx-Rice <me@tobias.gr> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix import egg) + #:use-module (ice-9 ftw) + #:use-module (ice-9 match) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-71) + #:use-module (gcrypt hash) + #:use-module (guix git) + #:use-module (guix i18n) + #:use-module (guix base32) + #:use-module (guix diagnostics) + #:use-module (guix memoization) + #:use-module (guix packages) + #:use-module (guix upstream) + #:use-module (guix build-system) + #:use-module (guix build-system chicken) + #:use-module (guix store) + #:use-module ((guix download) #:select (download-to-store url-fetch)) + #:use-module (guix import utils) + #:use-module ((guix licenses) #:prefix license:) + #:export (egg->guix-package + egg-recursive-import + %egg-updater + + ;; For tests. + guix-package->egg-name)) + +;;; Commentary: +;;; +;;; (guix import egg) provides package importer for CHICKEN eggs. See the +;;; official specification format for eggs +;;; <https://wiki.call-cc.org/man/5/Egg%20specification%20format>. +;;; +;;; The following happens under the hood: +;;; +;;; * <git://code.call-cc.org/eggs-5-latest> is a Git repository that contains +;;; the latest version of all CHICKEN eggs. We look clone this repository +;;; and retrieve the latest version number, and the PACKAGE.egg file, which +;;; contains a list of lists containing metadata about the egg. +;;; +;;; * All the eggs are stored as tarballs at +;;; <https://code.call-cc.org/egg-tarballs/5>, so we grab the tarball for +;;; the egg from there. +;;; +;;; * The rest of the package fields will be parsed from the PACKAGE.egg file. +;;; +;;; Todos: +;;; +;;; * Support for CHICKEN 4? +;;; +;;; * Some packages will specify a specific version of a depencency in the +;;; PACKAGE.egg file, how should we handle this? +;;; +;;; Code: + + +;;; +;;; Egg metadata fetcher and helper functions. +;;; + +(define package-name-prefix "chicken-") + +(define %eggs-url + (make-parameter "https://code.call-cc.org/egg-tarballs/5")) + +(define %eggs-home-page + (make-parameter "https://wiki.call-cc.org/egg")) + +(define (egg-source-url name version) + "Return the URL to the source tarball for version VERSION of the CHICKEN egg +NAME." + `(egg-uri ,name version)) + +(define (egg-name->guix-name name) + "Return the package name for CHICKEN egg NAME." + (string-append package-name-prefix name)) + +(define (eggs-repository) + "Update or fetch the latest version of the eggs repository and return the path +to the repository." + (let* ((url "git://code.call-cc.org/eggs-5-latest") + (directory commit _ (update-cached-checkout url))) + directory)) + +(define (egg-directory name) + "Return the directory containing the source code for the egg NAME." + (let ((eggs-directory (eggs-repository))) + (string-append eggs-directory "/" name))) + +(define (find-latest-version name) + "Get the latest version of the egg NAME." + (let ((directory (scandir (egg-directory name)))) + (if directory + (last directory) + #f))) + +(define* (egg-metadata name #:optional file) + "Return the package metadata file for the egg NAME, or if FILE is specified, +return the package metadata in FILE." + (call-with-input-file (or file + (string-append (egg-directory name) "/" + (find-latest-version name) + "/" name ".egg")) + read)) + +(define (guix-name->egg-name name) + "Return the CHICKEN egg name corresponding to the Guix package NAME." + (if (string-prefix? package-name-prefix name) + (string-drop name (string-length package-name-prefix)) + name)) + +(define (guix-package->egg-name package) + "Return the CHICKEN egg name of the Guix CHICKEN PACKAGE." + (or (assq-ref (package-properties package) 'upstream-name) + (guix-name->egg-name (package-name package)))) + +(define (egg-package? package) + "Check if PACKAGE is an CHICKEN egg package." + (and (eq? (package-build-system package) chicken-build-system) + (string-prefix? package-name-prefix (package-name package)))) + +(define string->license + ;; Doesn't seem to use a specific format. + ;; <https://wiki.call-cc.org/eggs-licensing> + (match-lambda + ("GPL-2" 'license:gpl2) + ("GPL-2+" 'license:gpl2+) + ("GPL-3" 'license:gpl3) + ("GPL-3+" 'license:gpl3+) + ("GPL" 'license:gpl?) + ("AGPL-3" 'license:agpl3) + ("AGPL" 'license:agpl?) + ("LGPL-2.0" 'license:lgpl2.0) + ("LGPL-2.0+" 'license:lgpl2.0+) + ("LGPL-2.1" 'license:lgpl2.1) + ("LGPL-2.1+" 'license:lgpl2.1+) + ("LGPL-3" 'license:lgpl3) + ("LGPL-3" 'license:lgpl3+) + ("LGPL" 'license:lgpl?) + ("BSD-1-Clause" 'license:bsd-1) + ("BSD-2-Clause" 'license:bsd-2) + ("BSD-3-Clause" 'license:bsd-3) + ("BSD" 'license:bsd?) + ("MIT" 'license:expat) + ("ISC" 'license:isc) + ("Artistic-2" 'license:artistic2.0) + ("Apache-2.0" 'license:asl2.0) + ("Public Domain" 'license:public-domain) + ((x) (string->license x)) + ((lst ...) `(list ,@(map string->license lst))) + (_ #f))) + + +;;; +;;; Egg importer. +;;; + +(define* (egg->guix-package name #:key (file #f) (source #f)) + "Import a CHICKEN egg called NAME from either the given .egg FILE, or from +the latest NAME metadata downloaded from the official repository if FILE is #f. +Return a <package> record or #f on failure. + +SOURCE is a ``file-like'' object containing the source code corresponding to +the egg. If SOURCE is not specified, the latest tarball for egg NAME will be +downloaded. + +Specifying the SOURCE argument is mainly useful for developing a CHICKEN egg +locally. Note that if FILE and SOURCE are specified, recursive import will +not work." + (define egg-content (if file + (egg-metadata name file) + (egg-metadata name))) + (if (not egg-content) + (values #f '()) ; egg doesn't exist + (let* ((version* (or (assoc-ref egg-content 'version) + (find-latest-version name))) + (version (if (list? version*) (first version*) version*)) + (source-url (if source #f (egg-source-url name version))) + (tarball (if source + #f + (with-store store + (download-to-store + store (egg-uri name version)))))) + + (define egg-home-page + (string-append (%eggs-home-page) "/" name)) + + (define egg-synopsis + (match (assoc-ref egg-content 'synopsis) + ((synopsis) synopsis) + (_ #f))) + + (define egg-licenses + (let ((licenses* + (match (assoc-ref egg-content 'license) + ((license) + (map string->license (string-split license #\/))) + (#f + '())))) + (match licenses* + ((license) license) + ((license1 license2 ...) `(list ,@licenses*))))) + + (define (maybe-symbol->string sym) + (if (symbol? sym) (symbol->string sym) sym)) + + (define (prettify-system-dependency name) + ;; System dependencies sometimes have spaces and/or upper case + ;; letters in them. + ;; + ;; There will probably still be some weird edge cases. + (string-map (lambda (char) + (case char + ((#\space) #\-) + (else char))) + (maybe-symbol->string name))) + + (define* (egg-parse-dependency name #:key (system? #f)) + (define extract-name + (match-lambda + ((name version) name) + (name name))) + + (define (prettify-name name) + (if system? + (prettify-system-dependency name) + (maybe-symbol->string name))) + + (let ((name (prettify-name (extract-name name)))) + ;; Dependencies are sometimes specified as symbols and sometimes + ;; as strings + (list (string-append (if system? "" package-name-prefix) + name) + (list 'unquote + (string->symbol (string-append + (if system? "" package-name-prefix) + name)))))) + + (define egg-propagated-inputs + (let ((dependencies (assoc-ref egg-content 'dependencies))) + (if (list? dependencies) + (map egg-parse-dependency + dependencies) + '()))) + + ;; TODO: Or should these be propagated? + (define egg-inputs + (let ((dependencies (assoc-ref egg-content 'foreign-dependencies))) + (if (list? dependencies) + (map (lambda (name) + (egg-parse-dependency name #:system? #t)) + dependencies) + '()))) + + (define egg-native-inputs + (let* ((test-dependencies (or (assoc-ref egg-content + 'test-dependencies) + '())) + (build-dependencies (or (assoc-ref egg-content + 'build-dependencies) + '())) + (test+build-dependencies (append test-dependencies + build-dependencies))) + (match test+build-dependencies + ((_ _ ...) (map egg-parse-dependency + test+build-dependencies)) + (() '())))) + + ;; Copied from (guix import hackage). + (define (maybe-inputs input-type inputs) + (match inputs + (() + '()) + ((inputs ...) + (list (list input-type + (list 'quasiquote inputs)))))) + + (values + `(package + (name ,(egg-name->guix-name name)) + (version ,version) + (source + ,(if source + source + `(origin + (method url-fetch) + (uri ,source-url) + (sha256 + (base32 ,(if tarball + (bytevector->nix-base32-string + (file-sha256 tarball)) + "failed to download tar archive")))))) + (build-system chicken-build-system) + (arguments ,(list 'quasiquote (list #:egg-name name))) + ,@(maybe-inputs 'native-inputs egg-native-inputs) + ,@(maybe-inputs 'inputs egg-inputs) + ,@(maybe-inputs 'propagated-inputs egg-propagated-inputs) + (home-page ,egg-home-page) + (synopsis ,egg-synopsis) + (description #f) + (license ,egg-licenses)) + (filter (lambda (name) + (not (member name '("srfi-4")))) + (map (compose guix-name->egg-name first) + (append egg-propagated-inputs + egg-native-inputs))))))) + +(define egg->guix-package/m ;memoized variant + (memoize egg->guix-package)) + +(define (egg-recursive-import package-name) + (recursive-import package-name + #:repo->guix-package (lambda* (name #:key version repo) + (egg->guix-package/m name)) + #:guix-name egg-name->guix-name)) + + +;;; +;;; Updater. +;;; + +(define (latest-release package) + "Return an @code{<upstream-source>} for the latest release of PACKAGE." + (let* ((egg-name (guix-package->egg-name package)) + (version (find-latest-version egg-name)) + (source-url (egg-source-url egg-name version))) + (upstream-source + (package (package-name package)) + (version version) + (urls (list source-url))))) + +(define %egg-updater + (upstream-updater + (name 'egg) + (description "Updater for CHICKEN egg packages") + (pred egg-package?) + (latest latest-release))) + +;;; egg.scm ends here diff --git a/guix/import/elpa.scm b/guix/import/elpa.scm index c0dc5acf51..96ebc17af1 100644 --- a/guix/import/elpa.scm +++ b/guix/import/elpa.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> ;;; Copyright © 2020 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -80,6 +81,7 @@ NAMES (strings)." (let ((elpa-archives '((gnu . "https://elpa.gnu.org/packages") (gnu/http . "http://elpa.gnu.org/packages") ;for testing + (nongnu . "https://elpa.nongnu.org/nongnu") (melpa-stable . "https://stable.melpa.org/packages") (melpa . "https://melpa.org/packages")))) (assq-ref elpa-archives repo))) @@ -257,7 +259,7 @@ RECIPE." ((assoc-ref recipe #:commit) => (lambda (commit) (cons 'commit commit))) (else - '(branch . "master")))) + '()))) (let-values (((directory commit) (download-git-repository url ref))) `(origin diff --git a/guix/import/git.scm b/guix/import/git.scm new file mode 100644 index 0000000000..1eb219f3fe --- /dev/null +++ b/guix/import/git.scm @@ -0,0 +1,225 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix import git) + #:use-module (guix build utils) + #:use-module (guix diagnostics) + #:use-module (guix git) + #:use-module (guix git-download) + #:use-module (guix i18n) + #:use-module (guix packages) + #:use-module (guix upstream) + #:use-module (guix utils) + #:use-module (ice-9 format) + #:use-module (ice-9 match) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 regex) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) + #:export (%generic-git-updater + + ;; For tests. + latest-git-tag-version)) + +;;; Commentary: +;;; +;;; This module provides a generic package updater for packages hosted on Git +;;; repositories. +;;; +;;; It tries to be smart about tag names, but if it is not automatically able +;;; to parse the tag names correctly, users can set the `release-tag-prefix', +;;; `release-tag-suffix' and `release-tag-version-delimiter' properties of the +;;; package to make the updater parse the Git tag name correctly. +;;; +;;; Possible improvements: +;;; +;;; * More robust method for trying to guess the delimiter. Maybe look at the +;;; previous version/tag combo to determine the delimiter. +;;; +;;; * Differentiate between "normal" versions, e.g., 1.2.3, and dates, e.g., +;;; 2021.12.31. Honor a `release-tag-date-scheme?' property? +;;; +;;; Code: + +;;; Errors & warnings + +(define-condition-type &git-no-valid-tags-error &error + git-no-valid-tags-error?) + +(define (git-no-valid-tags-error) + (raise (condition (&message (message "no valid tags found")) + (&git-no-valid-tags-error)))) + +(define-condition-type &git-no-tags-error &error + git-no-tags-error?) + +(define (git-no-tags-error) + (raise (condition (&message (message "no tags were found")) + (&git-no-tags-error)))) + + +;;; Updater + +(define %pre-release-words + '("alpha" "beta" "rc" "dev" "test" "pre")) + +(define %pre-release-rx + (map (lambda (word) + (make-regexp (string-append ".+" word) regexp/icase)) + %pre-release-words)) + +(define* (version-mapping tags #:key prefix suffix delim pre-releases?) + "Given a list of Git TAGS, return an association list where the car is the +version corresponding to the tag, and the cdr is the name of the tag." + (define (guess-delimiter) + (let ((total (length tags)) + (dots (reduce + 0 (map (cut string-count <> #\.) tags))) + (dashes (reduce + 0 (map (cut string-count <> #\-) tags))) + (underscores (reduce + 0 (map (cut string-count <> #\_) tags)))) + (cond + ((>= dots (* total 0.35)) ".") + ((>= dashes (* total 0.8)) "-") + ((>= underscores (* total 0.8)) "_") + (else "")))) + + (define delim-rx (regexp-quote (or delim (guess-delimiter)))) + (define suffix-rx (string-append (or suffix "") "$")) + (define prefix-rx (string-append "^" (or prefix "[^[:digit:]]*"))) + (define pre-release-rx + (if pre-releases? + (string-append "(.*(" (string-join %pre-release-words "|") ").*)") + "")) + + (define tag-rx + (string-append prefix-rx "([[:digit:]][^" delim-rx "[:punct:]]*" + "(" delim-rx "[^[:punct:]" delim-rx "]+)" + ;; If there are no delimiters, it could mean that the + ;; version just contains one number (e.g., "2"), thus, use + ;; "*" instead of "+" to match zero or more numbers. + (if (string=? delim-rx "") "*" "+") ")" + ;; We don't want the pre-release stuff (e.g., "-alpha") be + ;; part of the first group; otherwise, the "-" in "-alpha" + ;; might be interpreted as a delimiter, and thus replaced + ;; with "." + pre-release-rx suffix-rx)) + + + + (define (get-version tag) + (let ((tag-match (regexp-exec (make-regexp tag-rx) tag))) + (and=> (and tag-match + (regexp-substitute/global + #f delim-rx (match:substring tag-match 1) + ;; If there were no delimiters, don't insert ".". + 'pre (if (string=? delim-rx "") "" ".") 'post)) + (lambda (version) + (if pre-releases? + (string-append version (match:substring tag-match 3)) + version))))) + + (define (entry<? a b) + (eq? (version-compare (car a) (car b)) '<)) + + (stable-sort (filter-map (lambda (tag) + (let ((version (get-version tag))) + (and version (cons version tag)))) + tags) + entry<?)) + +(define* (latest-tag url #:key prefix suffix delim pre-releases?) + "Return the latest version and corresponding tag available from the Git +repository at URL." + (define (pre-release? tag) + (any (cut regexp-exec <> tag) + %pre-release-rx)) + + (let* ((tags (map (cut string-drop <> (string-length "refs/tags/")) + (remote-refs url #:tags? #t))) + (versions->tags + (version-mapping (if pre-releases? + tags + (filter (negate pre-release?) tags)) + #:prefix prefix + #:suffix suffix + #:delim delim + #:pre-releases? pre-releases?))) + (cond + ((null? tags) + (git-no-tags-error)) + ((null? versions->tags) + (git-no-valid-tags-error)) + (else + (match (last versions->tags) + ((version . tag) + (values version tag))))))) + +(define (latest-git-tag-version package) + "Given a PACKAGE, return the latest version of it, or #f if the latest version +could not be determined." + (guard (c ((or (git-no-tags-error? c) (git-no-valid-tags-error? c)) + (warning (or (package-field-location package 'source) + (package-location package)) + (G_ "~a for ~a~%") + (condition-message c) + (package-name package)) + #f) + ((eq? (exception-kind c) 'git-error) + (warning (or (package-field-location package 'source) + (package-location package)) + (G_ "failed to fetch Git repository for ~a~%") + (package-name package)) + #f)) + (let* ((source (package-source package)) + (url (git-reference-url (origin-uri source))) + (property (cute assq-ref (package-properties package) <>))) + (latest-tag url + #:prefix (property 'release-tag-prefix) + #:suffix (property 'release-tag-suffix) + #:delim (property 'release-tag-version-delimiter) + #:pre-releases? (property 'accept-pre-releases?))))) + +(define (git-package? package) + "Return true if PACKAGE is hosted on a Git repository." + (match (package-source package) + ((? origin? origin) + (and (eq? (origin-method origin) git-fetch) + (git-reference? (origin-uri origin)))) + (_ #f))) + +(define (latest-git-release package) + "Return an <upstream-source> for the latest release of PACKAGE." + (let* ((name (package-name package)) + (old-version (package-version package)) + (url (git-reference-url (origin-uri (package-source package)))) + (new-version (latest-git-tag-version package))) + + (and new-version + (upstream-source + (package name) + (version new-version) + (urls (list url)))))) + +(define %generic-git-updater + (upstream-updater + (name 'generic-git) + (description "Updater for packages hosted on Git repositories") + (pred git-package?) + (latest latest-git-release))) diff --git a/guix/import/go.scm b/guix/import/go.scm index bc53f8f558..9769b557ae 100644 --- a/guix/import/go.scm +++ b/guix/import/go.scm @@ -5,6 +5,8 @@ ;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; Copyright © 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -33,13 +35,14 @@ #:use-module (guix http-client) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix memoization) - #:use-module (htmlprag) ;from Guile-Lib + #:autoload (htmlprag) (html->sxml) ;from Guile-Lib #:autoload (guix git) (update-cached-checkout) #:autoload (gcrypt hash) (open-hash-port hash-algorithm sha256) #:autoload (guix serialization) (write-file) #:autoload (guix base32) (bytevector->nix-base32-string) #:autoload (guix build utils) (mkdir-p) #:use-module (ice-9 match) + #:use-module (ice-9 peg) #:use-module (ice-9 rdelim) #:use-module (ice-9 receive) #:use-module (ice-9 regex) @@ -61,11 +64,9 @@ #:use-module (web uri) #:export (go-module->guix-package + go-module->guix-package* go-module-recursive-import)) -;;; Parameterize htmlprag to parse valid HTML more reliably. -(%strict-tokenizer? #t) - ;;; Commentary: ;;; ;;; (guix import go) attempts to make it easier to create Guix package @@ -148,26 +149,26 @@ name (e.g. \"github.com/golang/protobuf/proto\")." ;; Extract the text contained in a h2 child node of any ;; element marked with a "License" class attribute. (select (sxpath `(// (* (@ (equal? (class "License")))) - h2 // *text*)))) - (select (html->sxml body)))) + h2 // div // *text*)))) + (select (html->sxml body #:strict? #t)))) (define (sxml->texi sxml-node) "A very basic SXML to Texinfo converter which attempts to preserve HTML formatting and links as text." (sxml-match sxml-node - ((strong ,text) - (format #f "@strong{~a}" text)) - ((a (@ (href ,url)) ,text) - (format #f "@url{~a,~a}" url text)) - ((code ,text) - (format #f "@code{~a}" text)) - (,something-else something-else))) + ((strong ,text) + (format #f "@strong{~a}" text)) + ((a (@ (href ,url)) ,text) + (format #f "@url{~a,~a}" url text)) + ((code ,text) + (format #f "@code{~a}" text)) + (,something-else something-else))) (define (go-package-description name) "Retrieve a short description for NAME, a Go package name, e.g. \"google.golang.org/protobuf/proto\"." (let* ((body (go.pkg.dev-info name)) - (sxml (html->sxml body)) + (sxml (html->sxml body #:strict? #t)) (overview ((sxpath `(// (* (@ (equal? (class "Documentation-overview")))) @@ -189,8 +190,9 @@ e.g. \"google.golang.org/protobuf/proto\"." (description (if (not (null? overview)) overview (select-content sxml))) - (description* (and (not (null? description)) - (first description)))) + (description* (if (not (null? description)) + (first description) + description))) (match description* (() #f) ;nothing selected ((p elements ...) @@ -209,7 +211,7 @@ the https://pkg.go.dev/ web site." (select-title (sxpath `(// (div (@ (equal? (class "UnitReadme-content")))) // h3 *text*)))) - (match (select-title (html->sxml body)) + (match (select-title (html->sxml body #:strict? #t)) (() #f) ;nothing selected ((title more ...) ;title is the first string of the list (string-trim-both title))))) @@ -245,128 +247,138 @@ and VERSION and return an input port." (go-path-escape version)))) (http-fetch* url))) -(define %go.mod-require-directive-rx - ;; A line in a require directive is composed of a module path and - ;; a version separated by whitespace and an optionnal '//' comment at - ;; the end. - (make-regexp - (string-append - "^[[:blank:]]*([^[:blank:]]+)[[:blank:]]+" ;the module path - "([^[:blank:]]+)" ;the version - "([[:blank:]]+//.*)?"))) ;an optional comment -(define %go.mod-replace-directive-rx +(define (parse-go.mod content) + "Parse the go.mod file CONTENT, returning a list of directives, comments, +and unknown lines. Each sublist begins with a symbol (go, module, require, +replace, exclude, retract, comment, or unknown) and is followed by one or more +sublists. Each sublist begins with a symbol (module-path, version, file-path, +comment, or unknown) and is followed by the indicated data." + ;; https://golang.org/ref/mod#go-mod-file-grammar + (define-peg-pattern NL none "\n") + (define-peg-pattern WS none (or " " "\t" "\r")) + (define-peg-pattern => none (and (* WS) "=>")) + (define-peg-pattern punctuation none (or "," "=>" "[" "]" "(" ")")) + (define-peg-pattern comment all + (and (ignore "//") (* WS) (* (and (not-followed-by NL) peg-any)))) + (define-peg-pattern EOL body (and (* WS) (? comment) NL)) + (define-peg-pattern block-start none (and (* WS) "(" EOL)) + (define-peg-pattern block-end none (and (* WS) ")" EOL)) + (define-peg-pattern any-line body + (and (* WS) (* (and (not-followed-by NL) peg-any)) EOL)) + + ;; Strings and identifiers + (define-peg-pattern identifier body + (+ (and (not-followed-by (or NL WS punctuation)) peg-any))) + (define-peg-pattern string-raw body + (and (ignore "`") (+ (and (not-followed-by "`") peg-any)) (ignore "`"))) + (define-peg-pattern string-quoted body + (and (ignore "\"") + (+ (or (and (ignore "\\") peg-any) + (and (not-followed-by "\"") peg-any))) + (ignore "\""))) + (define-peg-pattern string-or-ident body + (and (* WS) (or string-raw string-quoted identifier))) + + (define-peg-pattern version all string-or-ident) + (define-peg-pattern module-path all string-or-ident) + (define-peg-pattern file-path all string-or-ident) + + ;; Non-directive lines + (define-peg-pattern unknown all any-line) + (define-peg-pattern block-line body + (or EOL (and (not-followed-by block-end) unknown))) + + ;; GoDirective = "go" GoVersion newline . + (define-peg-pattern go all (and (ignore "go") version EOL)) + + ;; ModuleDirective = "module" ( ModulePath | "(" newline ModulePath newline ")" ) newline . + (define-peg-pattern module all + (and (ignore "module") (or (and block-start module-path EOL block-end) + (and module-path EOL)))) + + ;; The following directives may all be used solo or in a block + ;; RequireSpec = ModulePath Version newline . + (define-peg-pattern require all (and module-path version EOL)) + (define-peg-pattern require-top body + (and (ignore "require") + (or (and block-start (* (or require block-line)) block-end) require))) + + ;; ExcludeSpec = ModulePath Version newline . + (define-peg-pattern exclude all (and module-path version EOL)) + (define-peg-pattern exclude-top body + (and (ignore "exclude") + (or (and block-start (* (or exclude block-line)) block-end) exclude))) + ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline ;; | ModulePath [ Version ] "=>" ModulePath Version newline . - (make-regexp - (string-append - "([^[:blank:]]+)" ;the module path - "([[:blank:]]+([^[:blank:]]+))?" ;optional version - "[[:blank:]]+=>[[:blank:]]+" - "([^[:blank:]]+)" ;the file or module path - "([[:blank:]]+([^[:blank:]]+))?"))) ;the version (if a module path) + (define-peg-pattern original all (or (and module-path version) module-path)) + (define-peg-pattern with all (or (and module-path version) file-path)) + (define-peg-pattern replace all (and original => with EOL)) + (define-peg-pattern replace-top body + (and (ignore "replace") + (or (and block-start (* (or replace block-line)) block-end) replace))) -(define (parse-go.mod content) - "Parse the go.mod file CONTENT, returning a list of requirements." - ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar - ;; which we think necessary for our use case. - (define (toplevel requirements replaced) - "This is the main parser. The results are accumulated in THE REQUIREMENTS -and REPLACED lists." - (let ((line (read-line))) - (cond - ((eof-object? line) - ;; parsing ended, give back the result - (values requirements replaced)) - ((string=? line "require (") - ;; a require block begins, delegate parsing to IN-REQUIRE - (in-require requirements replaced)) - ((string=? line "replace (") - ;; a replace block begins, delegate parsing to IN-REPLACE - (in-replace requirements replaced)) - ((string-prefix? "require " line) - ;; a require directive by itself - (let* ((stripped-line (string-drop line 8))) - (call-with-values - (lambda () - (require-directive requirements replaced stripped-line)) - toplevel))) - ((string-prefix? "replace " line) - ;; a replace directive by itself - (let* ((stripped-line (string-drop line 8))) - (call-with-values - (lambda () - (replace-directive requirements replaced stripped-line)) - toplevel))) - (#t - ;; unrecognised line, ignore silently - (toplevel requirements replaced))))) + ;; RetractSpec = ( Version | "[" Version "," Version "]" ) newline . + (define-peg-pattern range all + (and (* WS) (ignore "[") version + (* WS) (ignore ",") version (* WS) (ignore "]"))) + (define-peg-pattern retract all (and (or range version) EOL)) + (define-peg-pattern retract-top body + (and (ignore "retract") + (or (and block-start (* (or retract block-line)) block-end) retract))) - (define (in-require requirements replaced) - (let ((line (read-line))) - (cond - ((eof-object? line) - ;; this should never happen here but we ignore silently - (values requirements replaced)) - ((string=? line ")") - ;; end of block, coming back to toplevel - (toplevel requirements replaced)) - (#t - (call-with-values (lambda () - (require-directive requirements replaced line)) - in-require))))) + (define-peg-pattern go-mod body + (* (and (* WS) (or go module require-top exclude-top replace-top + retract-top EOL unknown)))) - (define (in-replace requirements replaced) - (let ((line (read-line))) - (cond - ((eof-object? line) - ;; this should never happen here but we ignore silently - (values requirements replaced)) - ((string=? line ")") - ;; end of block, coming back to toplevel - (toplevel requirements replaced)) - (#t - (call-with-values (lambda () - (replace-directive requirements replaced line)) - in-replace))))) + (let ((tree (peg:tree (match-pattern go-mod content))) + (keywords '(go module require replace exclude retract comment unknown))) + (keyword-flatten keywords tree))) - (define (replace-directive requirements replaced line) - "Extract replaced modules and new requirements from the replace directive -in LINE and add them to the REQUIREMENTS and REPLACED lists." - (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line)) - (module-path (match:substring rx-match 1)) - (version (match:substring rx-match 3)) - (new-module-path (match:substring rx-match 4)) - (new-version (match:substring rx-match 6)) - (new-replaced (cons (list module-path version) replaced)) - (new-requirements - (if (string-match "^\\.?\\./" new-module-path) - requirements - (cons (list new-module-path new-version) requirements)))) - (values new-requirements new-replaced))) +;; Prevent inlining of this procedure, which is accessed by unit tests. +(set! parse-go.mod parse-go.mod) + +(define (go.mod-directives go.mod directive) + "Return the list of top-level directive bodies in GO.MOD matching the symbol +DIRECTIVE." + (filter-map (match-lambda + (((? (cut eq? <> directive) head) . rest) rest) + (_ #f)) + go.mod)) + +(define (go.mod-requirements go.mod) + "Compute and return the list of requirements specified by GO.MOD." + (define (replace directive requirements) + (define (maybe-replace module-path new-requirement) + ;; Do not allow version updates for indirect dependencies (see: + ;; https://golang.org/ref/mod#go-mod-file-replace). + (if (and (equal? module-path (first new-requirement)) + (not (assoc-ref requirements module-path))) + requirements + (cons new-requirement (alist-delete module-path requirements)))) + + (match directive + ((('original ('module-path module-path) . _) with . _) + (match with + (('with ('file-path _) . _) + (alist-delete module-path requirements)) + (('with ('module-path new-module-path) ('version new-version) . _) + (maybe-replace module-path + (list new-module-path new-version))))))) - (define (require-directive requirements replaced line) - "Extract requirement from LINE and augment the REQUIREMENTS and REPLACED -lists." - (let* ((rx-match (regexp-exec %go.mod-require-directive-rx line)) - (module-path (match:substring rx-match 1)) - ;; Double-quoted strings were seen in the wild without escape - ;; sequences; trim the quotes to be on the safe side. - (module-path (string-trim-both module-path #\")) - (version (match:substring rx-match 2))) - (values (cons (list module-path version) requirements) replaced))) + (define (require directive requirements) + (match directive + ((('module-path module-path) ('version version) . _) + (cons (list module-path version) requirements)))) - (with-input-from-string content - (lambda () - (receive (requirements replaced) - (toplevel '() '()) - ;; At last remove the replaced modules from the requirements list. - (remove (lambda (r) - (assoc (car r) replaced)) - requirements))))) + (let* ((requires (go.mod-directives go.mod 'require)) + (replaces (go.mod-directives go.mod 'replace)) + (requirements (fold require '() requires))) + (fold replace requirements replaces))) ;; Prevent inlining of this procedure, which is accessed by unit tests. -(set! parse-go.mod parse-go.mod) +(set! go.mod-requirements go.mod-requirements) (define-record-type <vcs> (%make-vcs url-prefix root-regex type) @@ -381,28 +393,28 @@ lists." (define known-vcs ;; See the following URL for the official Go equivalent: ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087 - (list - (make-vcs - "github.com" - "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$" - 'git) - (make-vcs - "bitbucket.org" - "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$" - 'unknown) - (make-vcs - "hub.jazz.net/git/" - "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$" - 'git) - (make-vcs - "git.apache.org" - "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$" - 'git) - (make-vcs - "git.openstack.org" - "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?\ + (list + (make-vcs + "github.com" + "^(github\\.com/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$" + 'git) + (make-vcs + "bitbucket.org" + "^(bitbucket\\.org/([A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+))(/[A-Za-z0-9_.\\-]+)*$" + 'unknown) + (make-vcs + "hub.jazz.net/git/" + "^(hub\\.jazz\\.net/git/[a-z0-9]+/[A-Za-z0-9_.\\-]+)(/[A-Za-z0-9_.\\-]+)*$" + 'git) + (make-vcs + "git.apache.org" + "^(git\\.apache\\.org/[a-z0-9_.\\-]+\\.git)(/[A-Za-z0-9_.\\-]+)*$" + 'git) + (make-vcs + "git.openstack.org" + "^(git\\.openstack\\.org/[A-Za-z0-9_.\\-]+/[A-Za-z0-9_.\\-]+)(\\.git)?\ (/[A-Za-z0-9_.\\-]+)*$" - 'git))) + 'git))) (define (module-path->repository-root module-path) "Infer the repository root from a module path. Go modules can be @@ -431,9 +443,9 @@ hence the need to derive this information." (define* (go-module->guix-package-name module-path #:optional version) "Converts a module's path to the canonical Guix format for Go packages. Optionally include a VERSION string to append to the name." - ;; Map dot, slash and underscore characters to hyphens. + ;; Map dot, slash, underscore and tilde characters to hyphens. (let ((module-path* (string-map (lambda (c) - (if (member c '(#\. #\/ #\_)) + (if (member c '(#\. #\/ #\_ #\~)) #\- c)) module-path))) @@ -461,17 +473,24 @@ Optionally include a VERSION string to append to the name." "Retrieve the module meta-data from its landing page. This is necessary because goproxy servers don't currently provide all the information needed to build a package." + (define (go-import->module-meta content-text) + (match (string-split content-text #\space) + ((root-path vcs repo-url) + (make-module-meta root-path (string->symbol vcs) + (strip-.git-suffix/maybe repo-url))))) ;; <meta name="go-import" content="import-prefix vcs repo-root"> (let* ((meta-data (http-fetch* (format #f "https://~a?go-get=1" module-path))) - (select (sxpath `(// head (meta (@ (equal? (name "go-import")))) + (select (sxpath `(// (meta (@ (equal? (name "go-import")))) // content)))) - (match (select (html->sxml meta-data)) + (match (select (html->sxml meta-data #:strict? #t)) (() #f) ;nothing selected - (((content content-text)) - (match (string-split content-text #\space) - ((root-path vcs repo-url) - (make-module-meta root-path (string->symbol vcs) - (strip-.git-suffix/maybe repo-url)))))))) + ((('content content-text) ..1) + (or + (find (lambda (meta) + (string-prefix? (module-meta-import-prefix meta) module-path)) + (map go-import->module-meta content-text)) + ;; Fallback to the first meta if no import prefixes match. + (go-import->module-meta (first content-text))))))) (define (module-meta-data-repo-url meta-data goproxy) "Return the URL where the fetcher which will be used can download the @@ -589,7 +608,7 @@ When VERSION is unspecified, the latest version available is used." hint: use one of the following available versions ~a\n" version* available-versions)))) (content (fetch-go.mod goproxy module-path version*)) - (dependencies+versions (parse-go.mod content)) + (dependencies+versions (go.mod-requirements (parse-go.mod content))) (dependencies (if pin-versions? dependencies+versions (map car dependencies+versions))) @@ -600,7 +619,7 @@ hint: use one of the following available versions ~a\n" (meta-data (fetch-module-meta-data root-module-path)) (vcs-type (module-meta-vcs meta-data)) (vcs-repo-url (module-meta-data-repo-url meta-data goproxy)) - (synopsis (go-package-synopsis root-module-path)) + (synopsis (go-package-synopsis module-path)) (description (go-package-description module-path)) (licenses (go-package-licenses module-path))) (values @@ -611,7 +630,10 @@ hint: use one of the following available versions ~a\n" ,(vcs->origin vcs-type vcs-repo-url version*)) (build-system go-build-system) (arguments - '(#:import-path ,root-module-path)) + '(#:import-path ,module-path + ,@(if (string=? module-path root-module-path) + '() + `(#:unpack-path ,root-module-path)))) ,@(maybe-propagated-inputs (map (match-lambda ((name version) @@ -632,7 +654,28 @@ hint: use one of the following available versions ~a\n" dependencies+versions dependencies)))) -(define go-module->guix-package* (memoize go-module->guix-package)) +(define go-module->guix-package* + (lambda args + ;; Disable output buffering so that the following warning gets printed + ;; consistently. + (setvbuf (current-error-port) 'none) + (let ((package-name (match args ((name _ ...) name)))) + (guard (c ((http-get-error? c) + (warning (G_ "Failed to import package ~s. +reason: ~s could not be fetched: HTTP error ~a (~s). +This package and its dependencies won't be imported.~%") + package-name + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c)) + (values #f '())) + (else + (warning (G_ "Failed to import package ~s. +reason: ~s.~%") + package-name + (exception-args c)) + (values #f '()))) + (apply go-module->guix-package args))))) (define* (go-module-recursive-import package-name #:key (goproxy "https://proxy.golang.org") @@ -642,23 +685,12 @@ hint: use one of the following available versions ~a\n" (recursive-import package-name #:repo->guix-package - (lambda* (name #:key version repo) - ;; Disable output buffering so that the following warning gets printed - ;; consistently. - (setvbuf (current-error-port) 'none) - (guard (c ((http-get-error? c) - (warning (G_ "Failed to import package ~s. -reason: ~s could not be fetched: HTTP error ~a (~s). -This package and its dependencies won't be imported.~%") - name - (uri->string (http-get-error-uri c)) - (http-get-error-code c) - (http-get-error-reason c)) - (values '() '()))) - (receive (package-sexp dependencies) - (go-module->guix-package* name #:goproxy goproxy - #:version version - #:pin-versions? pin-versions?) - (values package-sexp dependencies)))) + (memoize + (lambda* (name #:key version repo) + (receive (package-sexp dependencies) + (go-module->guix-package* name #:goproxy goproxy + #:version version + #:pin-versions? pin-versions?) + (values package-sexp dependencies)))) #:guix-name go-module->guix-package-name #:version version)) diff --git a/guix/import/hackage.scm b/guix/import/hackage.scm index 9f992ffe8e..f94a1e7087 100644 --- a/guix/import/hackage.scm +++ b/guix/import/hackage.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2016 Nikita <nikita@n0.is> ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2019 Robert Vollmert <rob@vllmrt.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -163,22 +164,22 @@ version." ;; https://www.haskell.org ;; /cabal/release/cabal-latest/doc/API/Cabal/Distribution-License.html. (match-lambda - ("GPL-2" 'gpl2) - ("GPL-3" 'gpl3) + ("GPL-2" 'license:gpl2) + ("GPL-3" 'license:gpl3) ("GPL" "'gpl??") - ("AGPL-3" 'agpl3) + ("AGPL-3" 'license:agpl3) ("AGPL" "'agpl??") - ("LGPL-2.1" 'lgpl2.1) - ("LGPL-3" 'lgpl3) + ("LGPL-2.1" 'license:lgpl2.1) + ("LGPL-3" 'license:lgpl3) ("LGPL" "'lgpl??") - ("BSD2" 'bsd-2) - ("BSD3" 'bsd-3) - ("BSD-3-Clause" 'bsd-3) - ("MIT" 'expat) - ("ISC" 'isc) - ("MPL" 'mpl2.0) - ("Apache-2.0" 'asl2.0) - ("PublicDomain" 'public-domain) + ("BSD2" 'license:bsd-2) + ("BSD3" 'license:bsd-3) + ("BSD-3-Clause" 'license:bsd-3) + ("MIT" 'license:expat) + ("ISC" 'license:isc) + ("MPL" 'license:mpl2.0) + ("Apache-2.0" 'license:asl2.0) + ("PublicDomain" 'license:public-domain) ((x) (string->license x)) ((lst ...) `(list ,@(map string->license lst))) (_ #f))) diff --git a/guix/import/launchpad.scm b/guix/import/launchpad.scm index fd3cfa8444..aeb447b0a5 100644 --- a/guix/import/launchpad.scm +++ b/guix/import/launchpad.scm @@ -1,5 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019, 2020 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2021 Matthew James Kraai <kraai@ftbfs.org> +;;; Copyright © 2020 Brice Waegeneire <brice@waegenei.re> ;;; ;;; This file is part of GNU Guix. ;;; @@ -100,8 +102,8 @@ URL of the form (match (string-split (uri-path (string->uri url)) #\/) ((_ repo . rest) repo))) -(define (latest-released-version package-name) - "Return a string of the newest released version name given the PACKAGE-NAME, +(define (latest-released-version repository) + "Return a string of the newest released version name given the REPOSITORY, for example, 'linuxdcpp'. Return #f if there is no releases." (define (pre-release? x) ;; Versions containing anything other than digit characters and "." (for @@ -110,27 +112,27 @@ for example, 'linuxdcpp'. Return #f if there is no releases." char-set:digit) (assoc-ref x "version")))) - (assoc-ref - (last (remove - pre-release? - (vector->list - (assoc-ref (json-fetch - (string-append "https://api.launchpad.net/1.0/" - package-name "/releases")) - "entries")))) - "version")) + (match (json-fetch + (string-append "https://api.launchpad.net/1.0/" + repository "/releases")) + (#f #f) ;404 or similar + (json + (assoc-ref + (last (remove pre-release? (vector->list (assoc-ref json "entries")))) + "version")))) (define (latest-release pkg) "Return an <upstream-source> for the latest release of PKG." - (define (origin-github-uri origin) + (define (origin-launchpad-uri origin) (match (origin-uri origin) ((? string? url) url) ; surely a Launchpad URL ((urls ...) (find (cut string-contains <> "launchpad.net") urls)))) - (let* ((source-uri (origin-github-uri (package-source pkg))) + (let* ((source-uri (origin-launchpad-uri (package-source pkg))) (name (package-name pkg)) - (newest-version (latest-released-version name))) + (repository (launchpad-repository source-uri)) + (newest-version (latest-released-version repository))) (if newest-version (upstream-source (package name) diff --git a/guix/import/minetest.scm b/guix/import/minetest.scm new file mode 100644 index 0000000000..ba86c60bfd --- /dev/null +++ b/guix/import/minetest.scm @@ -0,0 +1,468 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix import minetest) + #:use-module (ice-9 match) + #:use-module (ice-9 receive) + #:use-module (ice-9 threads) + #:use-module (ice-9 hash-table) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-2) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-26) + #:use-module (guix utils) + #:use-module (guix ui) + #:use-module (guix i18n) + #:use-module (guix memoization) + #:use-module (guix serialization) + #:use-module (guix import utils) + #:use-module (guix import json) + #:use-module ((gcrypt hash) #:select (open-sha256-port port-sha256)) + #:use-module (json) + #:use-module (guix base32) + #:use-module (guix git) + #:use-module (guix store) + #:export (%default-sort-key + %contentdb-api + json->package + contentdb-fetch + elaborate-contentdb-name + minetest->guix-package + minetest-recursive-import + sort-packages)) + +;; The ContentDB API is documented at +;; <https://content.minetest.net>. + +(define %contentdb-api + (make-parameter "https://content.minetest.net/api/")) + +(define (string-or-false x) + (and (string? x) x)) + +(define (natural-or-false x) + (and (exact-integer? x) (>= x 0) x)) + +;; Descriptions on ContentDB use carriage returns, but Guix doesn't. +(define (delete-cr text) + (string-delete #\cr text)) + + + +;;; +;;; JSON mappings +;;; + +;; Minetest package. +;; +;; API endpoint: /packages/AUTHOR/NAME/ +(define-json-mapping <package> make-package package? + json->package + (author package-author) ; string + (creation-date package-creation-date ; string + "created_at") + (downloads package-downloads) ; integer + (forums package-forums "forums" natural-or-false) + (issue-tracker package-issue-tracker "issue_tracker") ; string + (license package-license) ; string + (long-description package-long-description "long_description") ; string + (maintainers package-maintainers ; list of strings + "maintainers" vector->list) + (media-license package-media-license "media_license") ; string + (name package-name) ; string + (provides package-provides ; list of strings + "provides" vector->list) + (release package-release) ; integer + (repository package-repository "repo" string-or-false) + (score package-score) ; flonum + (screenshots package-screenshots "screenshots" vector->list) ; list of strings + (short-description package-short-description "short_description") ; string + (state package-state) ; string + (tags package-tags "tags" vector->list) ; list of strings + (thumbnail package-thumbnail) ; string + (title package-title) ; string + (type package-type) ; string + (url package-url) ; string + (website package-website "website" string-or-false)) + +(define-json-mapping <release> make-release release? + json->release + ;; If present, a git commit identified by its hash + (commit release-commit "commit" string-or-false) + (downloads release-downloads) ; integer + (id release-id) ; integer + (max-minetest-version release-max-minetest-version string-or-false) + (min-minetest-version release-min-minetest-version string-or-false) + (release-date release-data) ; string + (title release-title) ; string + (url release-url)) ; string + +(define-json-mapping <dependency> make-dependency dependency? + json->dependency + (optional? dependency-optional? "is_optional") ; bool + (name dependency-name) ; string + (packages dependency-packages "packages" vector->list)) ; list of strings + +;; A structure returned by the /api/packages/?fmt=keys endpoint +(define-json-mapping <package-keys> make-package-keys package-keys? + json->package-keys + (author package-keys-author) ; string + (name package-keys-name) ; string + (type package-keys-type)) ; string + +(define (package-mod? package) + "Is the ContentDB package PACKAGE a mod?" + ;; ContentDB also has ‘games’ and ‘texture packs’. + (string=? (package-type package) "mod")) + + + +;;; +;;; Manipulating names of packages +;;; +;;; There are three kind of names: +;;; +;;; * names of guix packages, e.g. minetest-basic-materials. +;;; * names of mods on ContentDB, e.g. basic_materials +;;; * a combination of author and mod name on ContentDB, e.g. VanessaE/basic_materials +;;; + +(define (%construct-full-name author name) + (string-append author "/" name)) + +(define (package-full-name package) + "Given a <package> object, return the corresponding AUTHOR/NAME string." + (%construct-full-name (package-author package) (package-name package))) + +(define (package-keys-full-name package) + "Given a <package-keys> object, return the corresponding AUTHOR/NAME string." + (%construct-full-name (package-keys-author package) + (package-keys-name package))) + +(define (contentdb->package-name author/name) + "Given the AUTHOR/NAME of a package on ContentDB, return a Guix-compliant +name for the package." + ;; The author is not included, as the names of popular mods + ;; tend to be unique. + (string-append "minetest-" (snake-case (author/name->name author/name)))) + +(define (author/name->name author/name) + "Extract NAME from the AUTHOR/NAME string, or raise an error if AUTHOR/NAME +is ill-formatted." + (match (string-split author/name #\/) + ((author name) + (when (string-null? author) + (leave + (G_ "In ~a: author names must consist of at least a single character.~%") + author/name)) + (when (string-null? name) + (leave + (G_ "In ~a: mod names must consist of at least a single character.~%") + author/name)) + name) + ((too many . components) + (leave + (G_ "In ~a: author names and mod names may not contain forward slashes.~%") + author/name)) + ((name) + (if (string-null? name) + (leave (G_ "mod names may not be empty.~%")) + (leave (G_ "The name of the author is missing in ~a.~%") + author/name))))) + +(define* (elaborate-contentdb-name name #:key (sort %default-sort-key)) + "If NAME is an AUTHOR/NAME string, return it. Otherwise, try to determine +the author and return an appropriate AUTHOR/NAME string. If that fails, +raise an exception." + (if (or (string-contains name "/") (string-null? name)) + ;; Call 'author/name->name' to verify that NAME seems reasonable + ;; and raise an appropriate exception if it isn't. + (begin + (author/name->name name) + name) + (let* ((package-keys (contentdb-query-packages name #:sort sort)) + (correctly-named + (filter (lambda (package-key) + (string=? name (package-keys-name package-key))) + package-keys))) + (match correctly-named + ((one) (package-keys-full-name one)) + ((too . many) + (warning (G_ "~a is ambiguous, presuming ~a (other options include: ~a)~%") + name (package-keys-full-name too) + (map package-keys-full-name many)) + (package-keys-full-name too)) + (() + (leave (G_ "No mods with name ~a were found.~%") name)))))) + + + +;;; +;;; API endpoints +;;; + +(define contentdb-fetch + (mlambda (author/name) + "Return a <package> record for package AUTHOR/NAME, or #f on failure." + (and=> (json-fetch + (string-append (%contentdb-api) "packages/" author/name "/")) + json->package))) + +(define (contentdb-fetch-releases author/name) + "Return a list of <release> records for package NAME by AUTHOR, or #f +on failure." + (and=> (json-fetch (string-append (%contentdb-api) "packages/" author/name + "/releases/")) + (lambda (json) + (map json->release (vector->list json))))) + +(define (latest-release author/name) + "Return the latest source release for package NAME by AUTHOR, +or #f if this package does not exist." + (and=> (contentdb-fetch-releases author/name) + car)) + +(define (contentdb-fetch-dependencies author/name) + "Return an alist of lists of <dependency> records for package NAME by AUTHOR +and possibly some other packages as well, or #f on failure." + (define url (string-append (%contentdb-api) "packages/" author/name + "/dependencies/")) + (and=> (json-fetch url) + (lambda (json) + (map (match-lambda + ((key . value) + (cons key (map json->dependency (vector->list value))))) + json)))) + +(define* (contentdb-query-packages q #:key + (type "mod") + (limit 50) + (sort %default-sort-key) + (order "desc")) + "Search ContentDB for Q (a string). Sort by SORT, in ascending order +if ORDER is \"asc\" or descending order if ORDER is \"desc\". TYPE must +be \"mod\", \"game\" or \"txp\", restricting the search results to +respectively mods, games and texture packs. Limit to at most LIMIT +results. The return value is a list of <package-keys> records." + ;; XXX does Guile have something for constructing (and, when necessary, + ;; escaping) query strings? + (define url (string-append (%contentdb-api) "packages/?type=" type + "&q=" q "&fmt=keys" + "&limit=" (number->string limit) + "&order=" order + "&sort=" sort)) + (let ((json (json-fetch url))) + (if json + (map json->package-keys (vector->list json)) + (leave + (G_ "The package search API doesn't exist anymore.~%"))))) + + + +;; XXX copied from (guix import elpa) +(define* (download-git-repository url ref) + "Fetch the given REF from the Git repository at URL." + (with-store store + (latest-repository-commit store url #:ref ref))) + +;; XXX adapted from (guix scripts hash) +(define (file-hash file) + "Compute the hash of FILE." + (let-values (((port get-hash) (open-sha256-port))) + (write-file file port) + (force-output port) + (get-hash))) + +(define (make-minetest-sexp author/name version repository commit + inputs home-page synopsis + description media-license license) + "Return a S-expression for the minetest package with the given author/NAME, +VERSION, REPOSITORY, COMMIT, INPUTS, HOME-PAGE, SYNOPSIS, DESCRIPTION, +MEDIA-LICENSE and LICENSE." + `(package + (name ,(contentdb->package-name author/name)) + (version ,version) + (source + (origin + (method git-fetch) + (uri (git-reference + (url ,repository) + (commit ,commit))) + (sha256 + (base32 + ;; The git commit is not always available. + ,(and commit + (bytevector->nix-base32-string + (file-hash + (download-git-repository repository + `(commit . ,commit))))))) + (file-name (git-file-name name version)))) + (build-system minetest-mod-build-system) + ,@(maybe-propagated-inputs (map contentdb->package-name inputs)) + (home-page ,home-page) + (synopsis ,(delete-cr synopsis)) + (description ,(delete-cr description)) + (license ,(if (eq? media-license license) + license + `(list ,media-license ,license))) + ;; The Minetest updater (not yet in Guix; it requires not-yet-submitted + ;; patches to (guix upstream) that require some work) needs to know both + ;; the author name and mod name for efficiency. + (properties ,(list 'quasiquote `((upstream-name . ,author/name)))))) + +(define (package-home-page package) + "Guess the home page of the ContentDB package PACKAGE. + +In order of preference, try the 'website', the forum topic on the +official Minetest forum and the Git repository (if any)." + (define (topic->url-sexp topic) + ;; 'minetest-topic' is a procedure defined in (gnu packages minetest) + `(minetest-topic ,topic)) + (or (package-website package) + (and=> (package-forums package) topic->url-sexp) + (package-repository package))) + +(define (release-version release) + "Guess the version of RELEASE from the release title." + (define title (release-title release)) + (if (string-prefix? "v" title) + ;; Remove "v" prefix from release titles like ‘v1.0.1’. + (substring title 1) + title)) + +;; If the default sort key is changed, make sure to modify 'show-help' +;; in (guix scripts import minetest) appropriately as well. +(define %default-sort-key "score") + +(define* (sort-packages packages #:key (sort %default-sort-key)) + "Sort PACKAGES by SORT, in descending order." + (define package->key + (match sort + ("score" package-score) + ("downloads" package-downloads))) + (define (greater x y) + (> (package->key x) (package->key y))) + (sort-list packages greater)) + +(define builtin-mod? + (let ((%builtin-mods + (alist->hash-table + (map (lambda (x) (cons x #t)) + '("beds" "binoculars" "boats" "bones" "bucket" "butterflies" + "carts" "creative" "default" "doors" "dungeon_loot" "dye" + "env_sounds" "farming" "fire" "fireflies" "flowers" + "game_commands" "give_initial_stuff" "map" "mtg_craftguide" + "player_api" "screwdriver" "sethome" "sfinv" "spawn" "stairs" + "tnt" "vessels" "walls" "weather" "wool" "xpanes"))))) + (lambda (mod) + "Is MOD provided by the default minetest subgame?" + (hash-ref %builtin-mods mod)))) + +(define* (important-dependencies dependencies author/name + #:key (sort %default-sort-key)) + "Return the hard dependencies of AUTHOR/NAME in the association list +DEPENDENCIES as a list of AUTHOR/NAME strings." + (define dependency-list + (assoc-ref dependencies author/name)) + ;; A mod can have multiple dependencies implemented by the same mod, + ;; so remove duplicate mod names. + (define (filter-deduplicate-map f list) + (delete-duplicates (filter-map f list))) + (filter-deduplicate-map + (lambda (dependency) + (and (not (dependency-optional? dependency)) + (not (builtin-mod? (dependency-name dependency))) + ;; The dependency information contains symbolic names + ;; that can be ‘provided’ by multiple mods, so we need to choose one + ;; of the implementations. + (let* ((implementations + (par-map contentdb-fetch (dependency-packages dependency))) + ;; Fetching package information about the packages is racy: + ;; some packages might be removed from ContentDB between the + ;; construction of DEPENDENCIES and the call to + ;; 'contentdb-fetch'. So filter out #f. + ;; + ;; Filter out ‘games’ that include the requested mod -- it's + ;; the mod itself we want. + (mods (filter (lambda (p) (and=> p package-mod?)) + implementations)) + (sorted-mods (sort-packages mods #:sort sort))) + (match sorted-mods + ((package) (package-full-name package)) + ((too . many) + (warning + (G_ "The dependency ~a of ~a has multiple different implementations ~a.~%") + (dependency-name dependency) + author/name + (map package-full-name sorted-mods)) + (match sort + ("score" + (warning + (G_ "The implementation with the highest score will be choosen!~%"))) + ("downloads" + (warning + (G_ "The implementation that has been downloaded the most will be choosen!~%")))) + (package-full-name too)) + (() + (warning + (G_ "The dependency ~a of ~a does not have any implementation. It will be ignored!~%") + (dependency-name dependency) author/name) + #f))))) + dependency-list)) + +(define* (%minetest->guix-package author/name #:key (sort %default-sort-key)) + "Fetch the metadata for AUTHOR/NAME from https://content.minetest.net, and +return the 'package' S-expression corresponding to that package, or raise an +exception on failure. On success, also return the upstream dependencies as a +list of AUTHOR/NAME strings." + ;; Call 'author/name->name' to verify that AUTHOR/NAME seems reasonable. + (author/name->name author/name) + (define package (contentdb-fetch author/name)) + (unless package + (leave (G_ "no package metadata for ~a on ContentDB~%") author/name)) + (define dependencies (contentdb-fetch-dependencies author/name)) + (unless dependencies + (leave (G_ "no dependency information for ~a on ContentDB~%") author/name)) + (define release (latest-release author/name)) + (unless release + (leave (G_ "no release of ~a on ContentDB~%") author/name)) + (define important-upstream-dependencies + (important-dependencies dependencies author/name #:sort sort)) + (values (make-minetest-sexp author/name + (release-version release) + (package-repository package) + (release-commit release) + important-upstream-dependencies + (package-home-page package) + (package-short-description package) + (package-long-description package) + (spdx-string->license + (package-media-license package)) + (spdx-string->license + (package-license package))) + important-upstream-dependencies)) + +(define minetest->guix-package + (memoize %minetest->guix-package)) + +(define* (minetest-recursive-import author/name #:key (sort %default-sort-key)) + (define* (minetest->guix-package* author/name #:key repo version) + (minetest->guix-package author/name #:sort sort)) + (recursive-import author/name + #:repo->guix-package minetest->guix-package* + #:guix-name contentdb->package-name)) diff --git a/guix/import/opam.scm b/guix/import/opam.scm index 670973b193..fe13d29f03 100644 --- a/guix/import/opam.scm +++ b/guix/import/opam.scm @@ -1,6 +1,8 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; Copyright © 2021 Alice Brenon <alice.brenon@ens-lyon.fr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,21 +23,24 @@ #:use-module (ice-9 ftw) #:use-module (ice-9 match) #:use-module (ice-9 peg) + #:use-module ((ice-9 popen) #:select (open-pipe*)) #:use-module (ice-9 receive) - #:use-module ((ice-9 rdelim) #:select (read-line)) #:use-module (ice-9 textual-ports) #:use-module (ice-9 vlist) #:use-module (srfi srfi-1) #:use-module (srfi srfi-2) - #:use-module (web uri) + #:use-module ((srfi srfi-26) #:select (cut)) + #:use-module ((web uri) #:select (string->uri uri->string)) + #:use-module ((guix build utils) #:select (dump-port find-files mkdir-p)) #:use-module (guix build-system) #:use-module (guix build-system ocaml) #:use-module (guix http-client) - #:use-module (guix git) #:use-module (guix ui) #:use-module (guix packages) #:use-module (guix upstream) - #:use-module (guix utils) + #:use-module ((guix utils) #:select (cache-directory + version>? + call-with-temporary-output-file)) #:use-module (guix import utils) #:use-module ((guix licenses) #:prefix license:) #:export (opam->guix-package @@ -65,7 +70,7 @@ (range #\# #\頋))) (define-peg-pattern operator all (or "=" "!" "<" ">")) -(define-peg-pattern records body (* (and (or record weird-record) (* SP)))) +(define-peg-pattern records body (and (* SP) (* (and (or record weird-record) (* SP))))) (define-peg-pattern record all (and key COLON (* SP) value)) (define-peg-pattern weird-record all (and key (* SP) dict)) (define-peg-pattern key body (+ (or (range #\a #\z) "-"))) @@ -120,51 +125,83 @@ (define-peg-pattern condition-string all (and QUOTE (* STRCHR) QUOTE)) (define-peg-pattern condition-var all (+ (or (range #\a #\z) "-" ":"))) -(define* (get-opam-repository #:optional repo) +(define (opam-cache-directory path) + (string-append (cache-directory) "/opam/" path)) + +(define known-repositories + '((opam . "https://opam.ocaml.org") + (coq . "https://coq.inria.fr/opam/released") + (coq-released . "https://coq.inria.fr/opam/released") + (coq-core-dev . "https://coq.inria.fr/opam/core-dev") + (coq-extra-dev . "https://coq.inria.fr/opam/extra-dev") + (grew . "http://opam.grew.fr"))) + +(define (get-uri repo-root) + (let ((archive-file (string-append repo-root "/index.tar.gz"))) + (or (string->uri archive-file) + (begin + (warning (G_ "'~a' is not a valid URI~%") archive-file) + 'bad-repo)))) + +(define (repo-type repo) + (match (assoc-ref known-repositories (string->symbol repo)) + (#f (if (file-exists? repo) + `(local ,repo) + `(remote ,(get-uri repo)))) + (url `(remote ,(get-uri url))))) + +(define (update-repository input) + "Make sure the cache for opam repository INPUT is up-to-date" + (let* ((output (opam-cache-directory (basename (port-filename input)))) + (cached-date (if (file-exists? output) + (stat:mtime (stat output)) + (begin (mkdir-p output) 0)))) + (when (> (stat:mtime (stat input)) cached-date) + (call-with-port + (open-pipe* OPEN_WRITE "tar" "xz" "-C" output "-f" "-") + (cut dump-port input <>))) + output)) + +(define* (get-opam-repository #:optional (repo "opam")) "Update or fetch the latest version of the opam repository and return the path to the repository." - (let ((url (cond - ((or (not repo) (equal? repo 'opam)) - "https://github.com/ocaml/opam-repository") - ((string-prefix? "coq-" (symbol->string repo)) - "https://github.com/coq/opam-coq-archive") - ((equal? repo 'coq) "https://github.com/coq/opam-coq-archive") - (else (throw 'unknown-repository repo))))) - (receive (location commit _) - (update-cached-checkout url) - (cond - ((or (not repo) (equal? repo 'opam)) - location) - ((equal? repo 'coq) - (string-append location "/released")) - ((string-prefix? "coq-" (symbol->string repo)) - (string-append location "/" (substring (symbol->string repo) 4))) - (else location))))) + (match (repo-type repo) + (('local p) p) + (('remote 'bad-repo) #f) ; to weed it out during filter-map in opam-fetch + (('remote r) (call-with-port (http-fetch/cached r) update-repository)))) ;; Prevent Guile 3 from inlining this procedure so we can mock it in tests. (set! get-opam-repository get-opam-repository) -(define (latest-version versions) - "Find the most recent version from a list of versions." - (fold (lambda (a b) (if (version>? a b) a b)) (car versions) versions)) +(define (get-version-and-file path) + "Analyse a candidate path and return an list containing information for proper + version comparison as well as the source path for metadata." + (and-let* ((metadata-file (string-append path "/opam")) + (filename (basename path)) + (version (string-join (cdr (string-split filename #\.)) "."))) + (and (file-exists? metadata-file) + (eq? 'regular (stat:type (stat metadata-file))) + (if (string-prefix? "v" version) + `(V ,(substring version 1) ,metadata-file) + `(digits ,version ,metadata-file))))) + +(define (keep-max-version a b) + "Version comparison on the lists returned by the previous function taking the + janestreet re-versioning into account (v-prefixed come first)." + (match (cons a b) + ((('V va _) . ('V vb _)) (if (version>? va vb) a b)) + ((('V _ _) . _) a) + ((_ . ('V _ _)) b) + ((('digits va _) . ('digits vb _)) (if (version>? va vb) a b)))) (define (find-latest-version package repository) "Get the latest version of a package as described in the given repository." - (let* ((dir (string-append repository "/packages/" package)) - (versions (scandir dir (lambda (name) (not (string-prefix? "." name)))))) - (if versions - (let ((versions (map - (lambda (dir) - (string-join (cdr (string-split dir #\.)) ".")) - versions))) - ;; Workaround for janestreet re-versionning - (let ((v-versions (filter (lambda (version) (string-prefix? "v" version)) versions))) - (if (null? v-versions) - (latest-version versions) - (string-append "v" (latest-version (map (lambda (version) (substring version 1)) v-versions)))))) - (begin - (format #t (G_ "Package not found in opam repository: ~a~%") package) - #f)))) + (let ((packages (string-append repository "/packages")) + (filter (make-regexp (string-append "^" package "\\.")))) + (reduce keep-max-version #f + (filter-map + get-version-and-file + (find-files packages filter #:directories? #t))))) (define (get-metadata opam-file) (with-input-from-file opam-file @@ -265,28 +302,30 @@ path to the repository." (define (depends->native-inputs depends) (filter (lambda (name) (not (equal? "" name))) - (map dependency->native-input depends))) + (map dependency->native-input depends))) (define (dependency-list->inputs lst) (map - (lambda (dependency) - (list dependency (list 'unquote (string->symbol dependency)))) - (ocaml-names->guix-names lst))) + (lambda (dependency) + (list dependency (list 'unquote (string->symbol dependency)))) + (ocaml-names->guix-names lst))) -(define* (opam-fetch name #:optional (repository (get-opam-repository))) - (and-let* ((repository repository) - (version (find-latest-version name repository)) - (file (string-append repository "/packages/" name "/" name "." version "/opam"))) - `(("metadata" ,@(get-metadata file)) - ("version" . ,(if (string-prefix? "v" version) - (substring version 1) - version))))) +(define* (opam-fetch name #:optional (repositories-specs '("opam"))) + (or (fold (lambda (repository others) + (match (find-latest-version name repository) + ((_ version file) `(("metadata" ,@(get-metadata file)) + ("version" . ,version))) + (_ others))) + #f + (filter-map get-opam-repository repositories-specs)) + (leave (G_ "package '~a' not found~%") name))) -(define* (opam->guix-package name #:key (repo 'opam) version) - "Import OPAM package NAME from REPOSITORY (a directory name) or, if -REPOSITORY is #f, from the official OPAM repository. Return a 'package' sexp +(define* (opam->guix-package name #:key (repo '()) version) + "Import OPAM package NAME from REPOSITORIES (a list of names, URLs or local +paths, always including OPAM's official repository). Return a 'package' sexp or #f on failure." - (and-let* ((opam-file (opam-fetch name (get-opam-repository repo))) + (and-let* ((with-opam (if (member "opam" repo) repo (cons "opam" repo))) + (opam-file (opam-fetch name with-opam)) (version (assoc-ref opam-file "version")) (opam-content (assoc-ref opam-file "metadata")) (url-dict (metadata-ref opam-content "url")) @@ -311,9 +350,7 @@ or #f on failure." (values `(package (name ,(ocaml-name->guix-name name)) - (version ,(if (string-prefix? "v" version) - (substring version 1) - version)) + (version ,version) (source (origin (method url-fetch) @@ -335,7 +372,8 @@ or #f on failure." (home-page ,(metadata-ref opam-content "homepage")) (synopsis ,(metadata-ref opam-content "synopsis")) (description ,(metadata-ref opam-content "description")) - (license #f)) + (license ,(spdx-string->license + (metadata-ref opam-content "license")))) (filter (lambda (name) (not (member name '("dune" "jbuilder")))) diff --git a/guix/import/print.scm b/guix/import/print.scm index dcc38abc70..0310739b3a 100644 --- a/guix/import/print.scm +++ b/guix/import/print.scm @@ -60,9 +60,9 @@ when evaluated." (if var (symbol-append 'license: var) `(license - (name ,(license-name lic)) - (uri ,(license-uri lic)) - (comment ,(license-comment lic)))))) + ,(license-name lic) + ,(license-uri lic) + ,(license-comment lic))))) (define (search-path-specification->code spec) `(search-path-specification diff --git a/guix/import/pypi.scm b/guix/import/pypi.scm index bf4dc50138..b7859c8341 100644 --- a/guix/import/pypi.scm +++ b/guix/import/pypi.scm @@ -9,6 +9,7 @@ ;;; Copyright © 2020 Lars-Dominik Braun <ldb@leibniz-psychology.org> ;;; Copyright © 2020 Arun Isaac <arunisaac@systemreboot.net> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -163,12 +164,13 @@ package on PyPI." (hyphen-package-name->name+version (basename (file-sans-extension url)))) - (match (and=> (package-source package) origin-uri) - ((? string? url) - (url->pypi-name url)) - ((lst ...) - (any url->pypi-name lst)) - (#f #f))) + (or (assoc-ref (package-properties package) 'upstream-name) + (match (and=> (package-source package) origin-uri) + ((? string? url) + (url->pypi-name url)) + ((lst ...) + (any url->pypi-name lst)) + (#f #f)))) (define (wheel-url->extracted-directory wheel-url) (match (string-split (basename wheel-url) #\-) @@ -229,8 +231,8 @@ the input field." '("test" "dev"))) (define (parse-requires.txt requires.txt) - "Given REQUIRES.TXT, a Setuptools requires.txt file, return a list of lists -of requirements. + "Given REQUIRES.TXT, a path to a Setuptools requires.txt file, return a list +of lists of requirements. The first list contains the required dependencies while the second the optional test dependencies. Note that currently, optional, non-test @@ -423,6 +425,11 @@ return the unaltered list of upstream dependency names." description license) "Return the `package' s-expression for a python package with the given NAME, VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." + (define (maybe-upstream-name name) + (if (string-match ".*\\-[0-9]+" (pk name)) + `((properties ,`'(("upstream-name" . ,name)))) + '())) + (call-with-temporary-output-file (lambda (temp port) (and (url-fetch source-url temp) @@ -461,6 +468,7 @@ VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." (sha256 (base32 ,(guix-hash-url temp))))) + ,@(maybe-upstream-name name) (build-system python-build-system) ,@(maybe-inputs required-inputs 'propagated-inputs) ,@(maybe-inputs native-inputs 'native-inputs) diff --git a/guix/import/snix.scm b/guix/import/snix.scm deleted file mode 100644 index 56934e8cf9..0000000000 --- a/guix/import/snix.scm +++ /dev/null @@ -1,467 +0,0 @@ -;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org> -;;; -;;; This file is part of GNU Guix. -;;; -;;; GNU Guix is free software; you can redistribute it and/or modify it -;;; under the terms of the GNU General Public License as published by -;;; the Free Software Foundation; either version 3 of the License, or (at -;;; your option) any later version. -;;; -;;; GNU Guix is distributed in the hope that it will be useful, but -;;; WITHOUT ANY WARRANTY; without even the implied warranty of -;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;;; GNU General Public License for more details. -;;; -;;; You should have received a copy of the GNU General Public License -;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. - -(define-module (guix import snix) - #:use-module (sxml ssax) - #:use-module (ice-9 popen) - #:use-module (ice-9 match) - #:use-module (ice-9 rdelim) - #:use-module (ice-9 format) - #:use-module (ice-9 regex) - #:use-module (ice-9 vlist) - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-9) - #:use-module (srfi srfi-11) - #:use-module (srfi srfi-26) - #:use-module (srfi srfi-37) - #:use-module (system foreign) - #:use-module (rnrs bytevectors) - - ;; Use the 'package-name->name+version' procedure that works with - ;; hyphen-separate name/version, not the one that works with '@'-separated - ;; name/version. Subtle! - #:use-module ((guix utils) #:hide (package-name->name+version)) - #:use-module ((guix build utils) #:select (package-name->name+version)) - - #:use-module (guix import utils) - #:use-module (guix base16) - #:use-module (guix base32) - #:use-module (guix config) - #:use-module (guix gnu-maintenance) - #:export (open-nixpkgs - xml->snix - nixpkgs->guix-package)) - -;;; Commentary: -;;; -;;; Converting Nix code to s-expressions, and then to Guix `package' -;;; declarations, using the XML output of `nix-instantiate'. -;;; -;;; Code: - - -;;; -;;; SNix. -;;; - -;; Nix object types visible in the XML output of `nix-instantiate' and -;; mapping to S-expressions (we map to sexps, not records, so that we -;; can do pattern matching): -;; -;; at (at varpat attrspat) -;; attr (attribute loc name value) -;; attrs (attribute-set attributes) -;; attrspat (attribute-set-pattern patterns) -;; bool #f|#t -;; derivation (derivation drv-path out-path attributes) -;; ellipsis '... -;; expr (snix loc body ...) -;; function (function loc at|attrspat|varpat) -;; int int -;; list list -;; null 'null -;; path string -;; string string -;; unevaluated 'unevaluated -;; varpat (varpat name) -;; -;; Initially ATTRIBUTES in `derivation' and `attribute-set' was a promise; -;; however, handling `repeated' nodes makes it impossible to do anything -;; lazily because the whole SXML tree has to be traversed to maintain the -;; list of known derivations. - -(define (xml-element->snix elem attributes body derivations) - "Return an SNix element corresponding to XML element ELEM." - - (define (loc) - (location (assq-ref attributes 'path) - (assq-ref attributes 'line) - (assq-ref attributes 'column))) - - (case elem - ((at) - (values `(at ,(car body) ,(cadr body)) derivations)) - ((attr) - (let ((name (assq-ref attributes 'name))) - (cond ((null? body) - (values `(attribute-pattern ,name) derivations)) - ((and (pair? body) (null? (cdr body))) - (values `(attribute ,(loc) ,name ,(car body)) - derivations)) - (else - (error "invalid attribute body" name (loc) body))))) - ((attrs) - (values `(attribute-set ,(reverse body)) derivations)) - ((attrspat) - (values `(attribute-set-pattern ,body) derivations)) - ((bool) - (values (string-ci=? "true" (assq-ref attributes 'value)) - derivations)) - ((derivation) - (let ((drv-path (assq-ref attributes 'drvPath)) - (out-path (assq-ref attributes 'outPath))) - (if (equal? body '(repeated)) - (let ((body (vhash-assoc drv-path derivations))) - (if (pair? body) - (values `(derivation ,drv-path ,out-path ,(cdr body)) - derivations) - - ;; DRV-PATH hasn't been encountered yet but may be later - ;; (see <http://article.gmane.org/gmane.linux.distributions.nixos/5946>.) - ;; Return an `unresolved' node. - (values `(unresolved - ,(lambda (derivations) - (let ((body (vhash-assoc drv-path derivations))) - (if (pair? body) - `(derivation ,drv-path ,out-path - ,(cdr body)) - (error "no previous occurrence of derivation" - drv-path))))) - derivations))) - (values `(derivation ,drv-path ,out-path ,body) - (vhash-cons drv-path body derivations))))) - ((ellipsis) - (values '... derivations)) - ((expr) - (values `(snix ,(loc) ,@body) derivations)) - ((function) - (values `(function ,(loc) ,body) derivations)) - ((int) - (values (string->number (assq-ref attributes 'value)) - derivations)) - ((list) - (values body derivations)) - ((null) - (values 'null derivations)) - ((path) - (values (assq-ref attributes 'value) derivations)) - ((repeated) - (values 'repeated derivations)) - ((string) - (values (assq-ref attributes 'value) derivations)) - ((unevaluated) - (values 'unevaluated derivations)) - ((varpat) - (values `(varpat ,(assq-ref attributes 'name)) derivations)) - (else (error "unhandled Nix XML element" elem)))) - -(define (resolve snix derivations) - "Return a new SNix tree where `unresolved' nodes from SNIX have been -replaced by the result of their application to DERIVATIONS, a vhash." - (let loop ((node snix) - (seen vlist-null)) - (if (vhash-assq node seen) - (values node seen) - (match node - (('unresolved proc) - (let ((n (proc derivations))) - (values n seen))) - ((tag body ...) - (let ((body+seen (fold (lambda (n body+seen) - (call-with-values - (lambda () - (loop n (cdr body+seen))) - (lambda (n* seen) - (cons (cons n* (car body+seen)) - (vhash-consq n #t seen))))) - (cons '() (vhash-consq node #t seen)) - body))) - (values (cons tag (reverse (car body+seen))) - (vhash-consq node #t (cdr body+seen))))) - (anything - (values anything seen)))))) - -(define xml->snix - (let ((parse - (ssax:make-parser NEW-LEVEL-SEED - (lambda (elem-gi attributes namespaces expected-content - seed) - (cons '() (cdr seed))) - - FINISH-ELEMENT - (lambda (elem-gi attributes namespaces parent-seed - seed) - (let ((snix (car seed)) - (derivations (cdr seed))) - (let-values (((snix derivations) - (xml-element->snix elem-gi - attributes - snix - derivations))) - (cons (cons snix (car parent-seed)) - derivations)))) - - CHAR-DATA-HANDLER - (lambda (string1 string2 seed) - ;; Discard inter-node strings, which are blanks. - seed)))) - (lambda (port) - "Return the SNix represention of TREE, an SXML tree as returned by -parsing the XML output of `nix-instantiate' on Nixpkgs." - (match (parse port (cons '() vlist-null)) - (((snix) . derivations) - (resolve snix derivations)))))) - -(define (attribute-value attribute) - "Return the value of ATTRIBUTE." - (match attribute - (('attribute _ _ value) value))) - -(define (derivation-source derivation) - "Return the \"src\" attribute of DERIVATION or #f if not found." - (match derivation - (('derivation _ _ (attributes ...)) - (find-attribute-by-name "src" attributes)))) - -(define (derivation-output-path derivation) - "Return the output path of DERIVATION." - (match derivation - (('derivation _ out-path _) - out-path) - (_ #f))) - -(define (source-output-path src) - "Return the output path of SRC, the \"src\" attribute of a derivation." - (derivation-output-path (attribute-value src))) - -(define (source-urls src) - "Return the URLs of SRC, the \"src\" attribute of a derivation." - (match src - (('attribute _ _ ('derivation _ _ (attributes ...))) - (match (find-attribute-by-name "urls" attributes) - (('attribute _ _ value) - value))) - (_ #f))) - -(define (source-sha256 src) - "Return the sha256 of SRC, the \"src\" attribute of a derivation, as a -bytevector." - (match src - (('attribute _ _ ('derivation _ _ (attributes ...))) - (match (find-attribute-by-name "outputHash" attributes) - (('attribute _ _ value) - (match value - ((= string-length 52) - (nix-base32-string->bytevector value)) - ((= string-length 64) - (base16-string->bytevector value)) - (_ - (error "unsupported hash format" value)))))) - (_ #f))) - -(define (derivation-source-output-path derivation) - "Return the output path of the \"src\" attribute of DERIVATION or #f -if DERIVATION lacks an \"src\" attribute." - (and=> (derivation-source derivation) source-output-path)) - -(define* (open-nixpkgs nixpkgs #:optional attribute) - "Return an input pipe to the XML representation of Nixpkgs. When -ATTRIBUTE is true, only that attribute is considered." - (with-fluids ((%default-port-encoding "UTF-8")) - (let ((cross-system (format #f "{ - config = \"i686-guix-linux-gnu\"; - libc = \"glibc\"; - arch = \"guix\"; - withTLS = true; - float = \"hard\"; - openssl.system = \"linux-generic32\"; - platform = (import ~a/pkgs/top-level/platforms.nix).sheevaplug; -}" nixpkgs))) - (apply open-pipe* OPEN_READ - "nix-instantiate" "--strict" "--eval-only" "--xml" - - ;; Pass a dummy `crossSystem' argument so that `buildInputs' and - ;; `nativeBuildInputs' are not coalesced. - ;; XXX: This is hacky and has other problems. - ;"--arg" "crossSystem" cross-system - - `(,@(if attribute - `("-A" ,attribute) - '()) - ,nixpkgs))))) - -(define (pipe-failed? pipe) - "Close pipe and return its status if it failed." - (let ((status (close-pipe pipe))) - (if (or (status:term-sig status) - (not (= (status:exit-val status) 0))) - status - #f))) - -(define (find-attribute-by-name name attributes) - "Return attribute NAME in ATTRIBUTES, an attribute set or list of SNix -attributes, or #f if NAME cannot be found." - (find (lambda (a) - (match a - (('attribute _ (? (cut string=? <> name)) _) - a) - (_ #f))) - (match attributes - (('attribute-set (attributes ...)) - attributes) - (_ - attributes)))) - -(define (license-variable license) - "Return the name of the (guix licenses) variable for LICENSE." - (match license - ("GPLv2+" 'gpl2+) - ("GPLv3+" 'gpl3+) - ("LGPLv2+" 'lgpl2.1+) - ("LGPLv2.1+" 'lgpl2.1+) - ("LGPLv3+" 'lgpl3+) - (('attribute-set _ ...) - ;; At some point in 2013, Nixpkgs switched to attribute sets to represent - ;; licenses. These are listed in lib/licenses.nix. - (match (and=> (find-attribute-by-name "shortName" license) - attribute-value) - ("agpl3Plus" 'agpl3+) - ("gpl2Plus" 'gpl2+) - ("gpl3Plus" 'gpl3+) - ("lgpl2Plus" 'lgpl2.0+) - ("lgpl21Plus" 'lgpl2.1+) - ("lgpl3Plus" 'lgpl3+) - ((? string? x) x) - (_ license))) - (_ license))) - -(define (package-source-output-path package) - "Return the output path of the \"src\" derivation of PACKAGE." - (derivation-source-output-path (attribute-value package))) - - -;;; -;;; Conversion of "Nix expressions" to "Guix expressions". -;;; - -(define (snix-derivation->guix-package derivation) - "Return the `package' s-expression corresponding to SNix DERIVATION, a -Nixpkgs `stdenv.mkDerivation'-style derivation, and the original source -location of DERIVATION." - (match derivation - (('derivation _ _ (attributes ...)) - (let*-values (((full-name loc) - (match (find-attribute-by-name "name" attributes) - (('attribute loc _ value) - (values value loc)) - (_ - (values #f #f)))) - ((name version) - (package-name->name+version full-name))) - (define (convert-inputs type) - ;; Convert the derivation's input from a list of SNix derivations to - ;; a list of name/variable pairs. - (match (and=> (find-attribute-by-name type attributes) - attribute-value) - (#f - '()) - ((inputs ...) - ;; Inputs can be either derivations or the null value. - (filter-map (match-lambda - (('derivation _ _ (attributes ...)) - (let* ((full-name - (attribute-value - (find-attribute-by-name "name" attributes))) - (name (package-name->name+version full-name))) - (list name - (list 'unquote (string->symbol name))))) - ('null #f)) - inputs)))) - - (define (maybe-inputs guix-name inputs) - (match inputs - (() - '()) - ((inputs ...) - (list (list guix-name - (list 'quasiquote inputs)))))) - - (define (pretty-uri uri version) - (if version - (match (factorize-uri uri version) - ((items ...) - `(string-append ,@items)) - (x x)) - uri)) - - (let* ((source (find-attribute-by-name "src" attributes)) - (urls (source-urls source)) - (sha256 (source-sha256 source)) - (meta (and=> (find-attribute-by-name "meta" attributes) - attribute-value))) - (values - `(package - (name ,name) - (version ,version) - (source (origin - (method url-fetch) - (uri ,(pretty-uri (car urls) version)) - (sha256 - (base32 - ,(bytevector->nix-base32-string sha256))))) - (build-system gnu-build-system) - - ;; When doing a native Nixpkgs build, `buildInputs' is empty and - ;; everything is in `nativeBuildInputs'. So we can't distinguish - ;; between both, here. - ;; - ;; Note that `nativeBuildInputs' was renamed from - ;; `buildNativeInputs' in Nixpkgs sometime around March 2013. - ,@(maybe-inputs 'inputs - (convert-inputs "nativeBuildInputs")) - ,@(maybe-inputs 'propagated-inputs - (convert-inputs "propagatedNativeBuildInputs")) - - (home-page ,(and=> (find-attribute-by-name "homepage" meta) - attribute-value)) - (synopsis - ;; For GNU packages, prefer the official synopsis. - ,(or (false-if-exception - (and=> (find (lambda (gnu-package) - (equal? (gnu-package-name gnu-package) - name)) - (official-gnu-packages)) - gnu-package-doc-summary)) - (and=> (find-attribute-by-name "description" meta) - attribute-value))) - (description - ;; Likewise, prefer the official description of GNU packages. - ,(or (false-if-exception - (and=> (find (lambda (gnu-package) - (equal? (gnu-package-name gnu-package) - name)) - (official-gnu-packages)) - gnu-package-doc-description)) - (and=> (find-attribute-by-name "longDescription" meta) - attribute-value))) - (license ,(and=> (find-attribute-by-name "license" meta) - (compose license-variable attribute-value)))) - loc)))))) - -(define (nixpkgs->guix-package nixpkgs attribute) - "Evaluate ATTRIBUTE in NIXPKGS, the file name of a Nixpkgs checkout, -and return the `package' s-expression corresponding to that package." - (let ((port (open-nixpkgs nixpkgs attribute))) - (match (xml->snix port) - (('snix loc (and drv ('derivation _ ...))) - (and (not (pipe-failed? port)) - (snix-derivation->guix-package drv))) - (_ - (not (pipe-failed? port)))))) - -;;; snix.scm ends here diff --git a/guix/import/stackage.scm b/guix/import/stackage.scm index bbd903a2cd..731e69651e 100644 --- a/guix/import/stackage.scm +++ b/guix/import/stackage.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2017 Federico Beffa <beffa@fbengineering.ch> ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2021 Xinglu Chem <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,10 +22,12 @@ (define-module (guix import stackage) #:use-module (ice-9 match) #:use-module (ice-9 regex) + #:use-module (ice-9 control) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) + #:use-module (srfi srfi-43) #:use-module (guix import json) #:use-module (guix import hackage) #:use-module (guix import utils) @@ -141,11 +144,23 @@ PACKAGE or #f if the package is not included in the Stackage LTS release." (version version) (urls (list url)))))))))) +(define (stackage-package? package) + "Whether PACKAGE is available on the default Stackage LTS release." + (and (hackage-package? package) + (let ((packages (lts-info-packages + (stackage-lts-info-fetch %default-lts-version))) + (hackage-name (guix-package->hackage-name package))) + (vector-any identity + (vector-map + (lambda (_ metadata) + (string=? (cdr (list-ref metadata 2)) hackage-name)) + packages))))) + (define %stackage-updater (upstream-updater (name 'stackage) (description "Updater for Stackage LTS packages") - (pred hackage-package?) + (pred stackage-package?) (latest latest-lts-release))) ;;; stackage.scm ends here diff --git a/guix/import/utils.scm b/guix/import/utils.scm index d817318a91..a180742ca3 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -8,6 +8,7 @@ ;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> ;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -133,8 +134,14 @@ of the string VERSION is replaced by the symbol 'version." ;; Please update guix/licenses.scm when modifying ;; this list to avoid mismatches. (match str + ;; "GPL-N+" has been deprecated in favour of "GPL-N-or-later". + ;; "GPL-N" has been deprecated in favour of "GPL-N-only" + ;; or "GPL-N-or-later" as appropriate. Likewise for LGPL + ;; and AGPL ("AGPL-1.0" 'license:agpl1) ("AGPL-3.0" 'license:agpl3) + ("AGPL-3.0-only" 'license:agpl3) + ("AGPL-3.0-or-later" 'license:agpl3+) ("Apache-1.1" 'license:asl1.1) ("Apache-2.0" 'license:asl2.0) ("BSL-1.0" 'license:boost1.0) @@ -161,11 +168,17 @@ of the string VERSION is replaced by the symbol 'version." ("GFDL-1.3" 'license:fdl1.3+) ("Giftware" 'license:giftware) ("GPL-1.0" 'license:gpl1) + ("GPL-1.0-only" 'license:gpl1) ("GPL-1.0+" 'license:gpl1+) + ("GPL-1.0-or-later" 'license:gpl1+) ("GPL-2.0" 'license:gpl2) + ("GPL-2.0-only" 'license:gpl2) ("GPL-2.0+" 'license:gpl2+) + ("GPL-2.0-or-later" 'license:gpl2+) ("GPL-3.0" 'license:gpl3) + ("GPL-3.0-only" 'license:gpl3) ("GPL-3.0+" 'license:gpl3+) + ("GPL-3.0-or-later" 'license:gpl3+) ("ISC" 'license:isc) ("IJG" 'license:ijg) ("Imlib2" 'license:imlib2) @@ -173,11 +186,17 @@ of the string VERSION is replaced by the symbol 'version." ("IPL-1.0" 'license:ibmpl1.0) ("LAL-1.3" 'license:lal1.3) ("LGPL-2.0" 'license:lgpl2.0) + ("LGPL-2.0-only" 'license:lgpl2.0) ("LGPL-2.0+" 'license:lgpl2.0+) + ("LGPL-2.0-or-later" 'license:lgpl2.0+) ("LGPL-2.1" 'license:lgpl2.1) + ("LGPL-2.1-only" 'license:lgpl2.1) ("LGPL-2.1+" 'license:lgpl2.1+) + ("LGPL-2.1-or-later" 'license:lgpl2.1+) ("LGPL-3.0" 'license:lgpl3) + ("LGPL-3.0-only" 'license:lgpl3) ("LGPL-3.0+" 'license:lgpl3+) + ("LGPL-3.0-or-later" 'license:lgpl3+) ("MPL-1.0" 'license:mpl1.0) ("MPL-1.1" 'license:mpl1.1) ("MPL-2.0" 'license:mpl2.0) @@ -471,15 +490,16 @@ to obtain the Guix package name corresponding to the upstream name." (name (list name #f))) dependencies))) (make-node name version package normalized-deps))) - (map node-package - (topological-sort (list (lookup-node package-name version)) - (lambda (node) - (map (lambda (name-version) - (apply lookup-node name-version)) - (remove (lambda (name-version) - (apply exists? name-version)) - (node-dependencies node)))) - (lambda (node) - (string-append - (node-name node) - (or (node-version node) "")))))) + (filter-map + node-package + (topological-sort (list (lookup-node package-name version)) + (lambda (node) + (map (lambda (name-version) + (apply lookup-node name-version)) + (remove (lambda (name-version) + (apply exists? name-version)) + (node-dependencies node)))) + (lambda (node) + (string-append + (node-name node) + (or (node-version node) "")))))) diff --git a/guix/inferior.scm b/guix/inferior.scm index eb457f81f9..81958baaa5 100644 --- a/guix/inferior.scm +++ b/guix/inferior.scm @@ -48,7 +48,7 @@ #:use-module (gcrypt hash) #:autoload (guix cache) (maybe-remove-expired-cache-entries file-expiration-time) - #:autoload (guix ui) (show-what-to-build*) + #:autoload (guix ui) (build-notifier) #:autoload (guix build utils) (mkdir-p) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) @@ -90,6 +90,7 @@ inferior-package-native-search-paths inferior-package-transitive-native-search-paths inferior-package-search-paths + inferior-package-replacement inferior-package-provenance inferior-package-derivation @@ -462,6 +463,27 @@ package." (define inferior-package-transitive-native-search-paths (cut %inferior-package-search-paths <> 'package-transitive-native-search-paths)) +(define (inferior-package-replacement package) + "Return the replacement for PACKAGE. This will either be an inferior +package, or #f." + (match (inferior-package-field + package + '(compose (match-lambda + ((? package? package) + (let ((id (object-address package))) + (hashv-set! %package-table id package) + (list id + (package-name package) + (package-version package)))) + (#f #f)) + package-replacement)) + (#f #f) + ((id name version) + (inferior-package (inferior-package-inferior package) + name + version + id)))) + (define (inferior-package-provenance package) "Return a \"provenance sexp\" for PACKAGE, an inferior package. The result is similar to the sexp returned by 'package-provenance' for regular packages." @@ -779,8 +801,10 @@ determines whether CHANNELS are authenticated." (profile (channel-instances->derivation instances))) (mbegin %store-monad - (show-what-to-build* (list profile)) + ;; It's up to the caller to install a build handler to report + ;; what's going to be built. (built-derivations (list profile)) + ;; Note: Caching is fine even when AUTHENTICATE? is false because ;; we always call 'latest-channel-instances?'. (symlink* (derivation->output-path profile) cached) @@ -799,10 +823,14 @@ This is a convenience procedure that people may use in manifests passed to 'guix package -m', for instance." (define cached (with-store store - (cached-channel-instance store - channels - #:cache-directory cache-directory - #:ttl ttl))) + ;; XXX: Install a build notifier out of convenience, so users know + ;; what's going on. However, we cannot be sure that its options, such + ;; as #:use-substitutes?, correspond to the daemon's default settings. + (with-build-handler (build-notifier) + (cached-channel-instance store + channels + #:cache-directory cache-directory + #:ttl ttl)))) (open-inferior cached)) ;;; Local Variables: diff --git a/guix/licenses.scm b/guix/licenses.scm index 4718ccf83f..c071aae4a9 100644 --- a/guix/licenses.scm +++ b/guix/licenses.scm @@ -17,6 +17,8 @@ ;;; Copyright © 2020 André Batista <nandre@riseup.net> ;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> ;;; Copyright © 2021 Felix Gruber <felgru@posteo.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; Copyright © 2021 Noisytoot <noisytoot@disroot.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -40,12 +42,11 @@ apsl2 asl1.1 asl2.0 boost1.0 - bsd-0 bsd-2 bsd-3 bsd-4 + bsd-0 bsd-1 bsd-2 bsd-3 bsd-4 non-copyleft cc0 cc-by2.0 cc-by3.0 cc-by4.0 cc-by-sa2.0 cc-by-sa3.0 cc-by-sa4.0 - cc-sampling-plus-1.0 cddl1.0 cddl1.1 cecill cecill-b cecill-c artistic2.0 clarified-artistic @@ -55,6 +56,7 @@ edl1.0 epl1.0 epl2.0 + eupl1.2 expat freetype freebsd-doc @@ -166,6 +168,11 @@ "https://spdx.org/licenses/0BSD.html" "https://opensource.org/licenses/0BSD")) +(define bsd-1 + (license "BSD 1-Clause" + "https://spdx.org/licenses/BSD-1-Clause.html" + "https://opensource.org/licenses/BSD-1-Clause")) + (define bsd-2 (license "FreeBSD" "http://directory.fsf.org/wiki/License:FreeBSD" @@ -227,11 +234,6 @@ at URI, which may be a file:// URI pointing the package's tree." "http://creativecommons.org/licenses/by/2.0/" "Creative Commons Attribution 2.0 Generic")) -(define cc-sampling-plus-1.0 - (license "CC-Sampling+ 1.0" - "https://creativecommons.org/licenses/sampling+/1.0" - "Creative Commons Sampling Plus 1.0")) - (define cddl1.0 (license "CDDL 1.0" "http://directory.fsf.org/wiki/License:CDDLv1.0" @@ -301,6 +303,11 @@ at URI, which may be a file:// URI pointing the package's tree." "https://www.eclipse.org/legal/epl-2.0/" "https://www.gnu.org/licenses/license-list#EPL2")) +(define eupl1.2 + (license "EUPL 1.2" + "https://directory.fsf.org/wiki/License:EUPL-1.2" + "https://www.gnu.org/licenses/license-list#EUPL-1.2")) + (define expat (license "Expat" "http://directory.fsf.org/wiki/License:Expat" diff --git a/guix/lint.scm b/guix/lint.scm index a7d6bbba4f..527fda165a 100644 --- a/guix/lint.scm +++ b/guix/lint.scm @@ -11,6 +11,9 @@ ;;; Copyright © 2018, 2019 Arun Isaac <arunisaac@systemreboot.net> ;;; Copyright © 2020 Chris Marusich <cmmarusich@gmail.com> ;;; Copyright © 2020 Timothy Sample <samplet@ngyro.com> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re> ;;; ;;; This file is part of GNU Guix. ;;; @@ -29,6 +32,7 @@ (define-module (guix lint) #:use-module (guix store) + #:autoload (guix base16) (bytevector->base16-string) #:use-module (guix base32) #:use-module (guix diagnostics) #:use-module (guix download) @@ -37,7 +41,8 @@ #:use-module (guix packages) #:use-module (guix i18n) #:use-module ((guix gexp) - #:select (local-file? local-file-absolute-file-name)) + #:select (gexp? local-file? local-file-absolute-file-name + gexp->approximate-sexp)) #:use-module (guix licenses) #:use-module (guix records) #:use-module (guix grafts) @@ -65,6 +70,7 @@ . guix:open-connection-for-uri))) #:use-module (web request) #:use-module (web response) + #:autoload (gnutls) (error->string) #:use-module (srfi srfi-1) #:use-module (srfi srfi-6) ;Unicode string ports #:use-module (srfi srfi-9) @@ -76,14 +82,17 @@ #:export (check-description-style check-inputs-should-be-native check-inputs-should-not-be-an-input-at-all + check-wrapper-inputs check-patch-file-names check-patch-headers check-synopsis-style check-derivation check-home-page + check-name check-source check-source-file-name check-source-unstable-tarball + check-optional-tests check-mirror-url check-github-url check-license @@ -93,6 +102,7 @@ check-archival check-profile-collisions check-haskell-stackage + check-tests-true lint-warning lint-warning? @@ -155,6 +165,78 @@ ;;; +;;; Procedures for analysing Scheme code in package definitions +;;; + +(define* (find-procedure-body expression found + #:key (not-found (const '()))) + "Try to find the body of the procedure defined inline by EXPRESSION. +If it was found, call FOUND with its body. If it wasn't, call +the thunk NOT-FOUND." + (match expression + (`(,(or 'let 'let*) . ,_) + (find-procedure-body (car (last-pair expression)) found + #:not-found not-found)) + (`(,(or 'lambda 'lambda*) ,_ . ,code) + (found code)) + (_ (not-found)))) + +(define* (report-bogus-phase-deltas package bogus-deltas) + "Report a bogus invocation of ‘modify-phases’." + (list (make-warning package + ;; TRANSLATORS: 'modify-phases' is a Scheme syntax + ;; and should not be translated. + (G_ "incorrect call to ‘modify-phases’") + #:field 'arguments))) + +(define* (find-phase-deltas package found + #:key (not-found (const '())) + (bogus + (cut report-bogus-phase-deltas package <>))) + "Try to find the clauses of the ‘modify-phases’ form in the phases +specification of PACKAGE. If they were found, all FOUND with a list +of the clauses. If they weren't (e.g. because ‘modify-phases’ wasn't +used at all), call the thunk NOT-FOUND instead. If ‘modify-phases’ +was used, but the clauses don't form a list, call BOGUS with the +not-a-list." + (apply (lambda* (#:key phases #:allow-other-keys) + (define phases/sexp + (if (gexp? phases) + (gexp->approximate-sexp phases) + phases)) + (match phases/sexp + (`(modify-phases ,_ . ,changes) + ((if (list? changes) found bogus) changes)) + (_ (not-found)))) + (package-arguments package))) + +(define (report-bogus-phase-procedure package) + "Report a syntactically-invalid phase clause." + (list (make-warning package + ;; TRANSLATORS: See ‘modify-phases’ in the manual. + (G_ "invalid phase clause") + #:field 'arguments))) + +(define* (find-phase-procedure package expression found + #:key (not-found (const '())) + (bogus (cut report-bogus-phase-procedure + package))) + "Try to find the procedure in the phase clause EXPRESSION. If it was +found, call FOUND with the procedure expression. If EXPRESSION isn't +actually a phase clause, call the thunk BOGUS. If the phase form doesn't +have a procedure, call the thunk NOT-FOUND." + (match expression + (('add-after before after proc-expr) + (found proc-expr)) + (('add-before after before proc-expr) + (found proc-expr)) + (('replace _ proc-expr) + (found proc-expr)) + (('delete _) (not-found)) + (_ (bogus)))) + + +;;; ;;; Checkers ;;; @@ -173,14 +255,40 @@ (define (check-name package) "Check whether PACKAGE's name matches our guidelines." (let ((name (package-name package))) - ;; Currently checks only whether the name is too short. - (if (and (<= (string-length name) 1) - (not (string=? name "r"))) ; common-sense exception - (list - (make-warning package - (G_ "name should be longer than a single character") - #:field 'name)) - '()))) + (cond + ;; Currently checks only whether the name is too short. + ((and (<= (string-length name) 1) + (not (string=? name "r"))) ; common-sense exception + (list + (make-warning package + (G_ "name should be longer than a single character") + #:field 'name))) + ((string-index name #\_) + (list + (make-warning package + (G_ "name should use hyphens instead of underscores") + #:field 'name))) + (else '())))) + +(define (check-tests-true package) + "Check whether PACKAGE explicitly requests to run tests, which is +superfluous when building natively and incorrect when cross-compiling." + (define (tests-explicitly-enabled?) + (apply (lambda* (#:key tests? #:allow-other-keys) + (eq? tests? #t)) + (package-arguments package))) + (if (and (tests-explicitly-enabled?) + ;; Some packages, e.g. gnutls, set #:tests? + ;; differently depending on whether it is being + ;; cross-compiled. + (parameterize ((%current-target-system "aarch64-linux-gnu")) + (tests-explicitly-enabled?))) + (list (make-warning package + ;; TRANSLATORS: #:tests? and #t are Scheme constants + ;; and must not be translated. + (G_ "#:tests? must not be explicitly set to #t") + #:field 'arguments)) + '())) (define (properly-starts-sentence? s) (string-match "^[(\"'`[:upper:][:digit:]]" s)) @@ -269,6 +377,24 @@ by two spaces; possible infraction~p at ~{~a~^, ~}") infractions) #:field 'description))))) + (define (check-no-leading-whitespace description) + "Check that DESCRIPTION doesn't have trailing whitespace." + (if (string-prefix? " " description) + (list + (make-warning package + (G_ "description contains leading whitespace") + #:field 'description)) + '())) + + (define (check-no-trailing-whitespace description) + "Check that DESCRIPTION doesn't have trailing whitespace." + (if (string-suffix? " " description) + (list + (make-warning package + (G_ "description contains trailing whitespace") + #:field 'description)) + '())) + (let ((description (package-description package))) (if (string? description) (append @@ -278,6 +404,8 @@ by two spaces; possible infraction~p at ~{~a~^, ~}") ;; Use raw description for this because Texinfo rendering ;; automatically fixes end of sentence space. (check-end-of-sentence-space description) + (check-no-leading-whitespace description) + (check-no-trailing-whitespace description) (match (check-texinfo-markup description) ((and warning (? lint-warning?)) (list warning)) (plain-description @@ -375,6 +503,49 @@ of a package, and INPUT-NAMES, a list of package specifications such as (package-input-intersection (package-direct-inputs package) input-names)))) +(define (report-wrap-program-error package wrapper-name) + "Warn that \"bash-minimal\" is missing from 'inputs', while WRAPPER-NAME +requires it." + (make-warning package + (G_ "\"bash-minimal\" should be in 'inputs' when '~a' is used") + (list wrapper-name))) + +(define (check-wrapper-inputs package) + "Emit a warning if PACKAGE uses 'wrap-program' or similar, but \"bash\" +or \"bash-minimal\" is not in its inputs. 'wrap-script' is not supported." + (define input-names '("bash" "bash-minimal")) + (define has-bash-input? + (pair? (package-input-intersection (package-inputs package) + input-names))) + (define (check-procedure-body body) + (match body + ;; Explicitely setting an interpreter is acceptable, + ;; #:sh support is added on 'core-updates'. + ;; TODO(core-updates): remove mention of core-updates. + (('wrap-program _ '#:sh . _) '()) + (('wrap-program _ . _) + (list (report-wrap-program-error package 'wrap-program))) + ;; Wrapper of 'wrap-program' for Qt programs. + ;; TODO #:sh is not yet supported but probably will be. + (('wrap-qt-program _ '#:sh . _) '()) + (('wrap-qt-program _ . _) + (list (report-wrap-program-error package 'wrap-qt-program))) + ((x . y) + (append (check-procedure-body x) (check-procedure-body y))) + (_ '()))) + (define (check-phase-procedure expression) + (find-procedure-body expression check-procedure-body)) + (define (check-delta expression) + (find-phase-procedure package expression check-phase-procedure)) + (define (check-deltas deltas) + (append-map check-delta deltas)) + (if has-bash-input? + ;; "bash" (or "bash-minimal") is in 'inputs', so everything seems ok. + '() + ;; "bash" is not in 'inputs'. Verify 'wrap-program' and friends + ;; are unused + (find-phase-deltas package check-deltas))) + (define (package-name-regexp package) "Return a regexp that matches PACKAGE's name as a word at the beginning of a line." @@ -447,13 +618,23 @@ markup is valid return a plain-text version of SYNOPSIS, otherwise #f." (G_ "Texinfo markup in synopsis is invalid") #:field 'synopsis))))) + (define (check-no-trailing-whitespace synopsis) + "Check that SYNOPSIS doesn't have trailing whitespace." + (if (string-suffix? " " synopsis) + (list + (make-warning package + (G_ "synopsis contains trailing whitespace") + #:field 'synopsis)) + '())) + (define checks (list check-proper-start check-final-period check-start-article check-start-with-package-name check-synopsis-length - check-texinfo-markup)) + check-texinfo-markup + check-no-trailing-whitespace)) (match (package-synopsis package) ("" @@ -565,6 +746,51 @@ for connections to complete; when TIMEOUT is #f, wait as long as needed." (_ (values 'unknown-protocol #f))))) +(define (call-with-networking-fail-safe message error-value proc) + "Call PROC catching any network-related errors. Upon a networking error, +display a message including MESSAGE and return ERROR-VALUE." + (guard (c ((http-get-error? c) + (warning (G_ "~a: HTTP GET error for ~a: ~a (~s)~%") + message + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c)) + error-value)) + (catch #t + proc + (match-lambda* + (('getaddrinfo-error errcode) + (warning (G_ "~a: host lookup failure: ~a~%") + message + (gai-strerror errcode)) + error-value) + (('tls-certificate-error args ...) + (warning (G_ "~a: TLS certificate error: ~a") + message + (tls-certificate-error-string args)) + error-value) + (('gnutls-error error function _ ...) + (warning (G_ "~a: TLS error in '~a': ~a~%") + message + function (error->string error)) + error-value) + ((and ('system-error _ ...) args) + (let ((errno (system-error-errno args))) + (if (member errno (list ECONNRESET ECONNABORTED ECONNREFUSED)) + (let ((details (call-with-output-string + (lambda (port) + (print-exception port #f (car args) + (cdr args)))))) + (warning (G_ "~a: ~a~%") message details) + error-value) + (apply throw args)))) + (args + (apply throw args)))))) + +(define-syntax-rule (with-networking-fail-safe message error-value exp ...) + (call-with-networking-fail-safe message error-value + (lambda () exp ...))) + (define (tls-certificate-error-string args) "Return a string explaining the 'tls-certificate-error' arguments ARGS." (call-with-output-string @@ -750,7 +976,8 @@ warnings." ((blank? line) (loop)) ((or (string-prefix? "--- " line) - (string-prefix? "+++ " line)) + (string-prefix? "+++ " line) + (string-prefix? "diff --git " line)) (list (make-warning package (G_ "~a: patch lacks comment and \ upstream status") @@ -982,69 +1209,91 @@ descriptions maintained upstream." (eqv? (origin-method origin) url-fetch)) (filter-map (lambda (uri) - (and=> (follow-redirects-to-github uri) + (and=> (with-networking-fail-safe + (format #f (G_ "while accessing '~a'") uri) + #f + (follow-redirects-to-github uri)) (lambda (github-uri) - (if (string=? github-uri uri) - #f - (make-warning - package - (G_ "URL should be '~a'") - (list github-uri) - #:field 'source))))) + (and (not (string=? github-uri uri)) + (make-warning + package + (G_ "URL should be '~a'") + (list github-uri) + #:field 'source))))) (origin-uris origin)) '()))) -(cond-expand - (guile-3 - ;; Guile 3.0.0 does not export this predicate. - (define exception-with-kind-and-args? - (exception-predicate &exception-with-kind-and-args))) - (else ;Guile 2 - (define exception-with-kind-and-args? - (const #f)))) +;; Guile 3.0.0 does not export this predicate. +(define exception-with-kind-and-args? + (exception-predicate &exception-with-kind-and-args)) + +(define (check-optional-tests package) + "Emit a warning if the test suite is run unconditionally." + (define (sexp-contains-atom? sexp atom) + "Test if SEXP contains ATOM." + (if (pair? sexp) + (or (sexp-contains-atom? (car sexp) atom) + (sexp-contains-atom? (cdr sexp) atom)) + (eq? sexp atom))) + (define (sexp-uses-tests?? sexp) + "Test if SEXP contains the symbol 'tests?'." + (sexp-contains-atom? sexp 'tests?)) + (define (check-procedure-body code) + (if (sexp-uses-tests?? code) + '() + (list (make-warning package + ;; TRANSLATORS: check and #:tests? are a + ;; Scheme symbol and keyword respectively + ;; and should not be translated. + (G_ "the 'check' phase should respect #:tests?") + #:field 'arguments)))) + (define (check-check-procedure expression) + (find-procedure-body expression check-procedure-body)) + (define (check-phases-delta delta) + (match delta + (`(replace 'check ,expression) + (check-check-procedure expression)) + (_ '()))) + (define (check-phases-deltas deltas) + (append-map check-phases-delta deltas)) + (find-phase-deltas package check-phases-deltas)) (define* (check-derivation package #:key store) "Emit a warning if we fail to compile PACKAGE to a derivation." (define (try store system) - (catch #t ;TODO: Remove 'catch' when Guile 2.x is no longer supported. - (lambda () - (guard (c ((store-protocol-error? c) - (make-warning package - (G_ "failed to create ~a derivation: ~a") - (list system - (store-protocol-error-message c)))) - ((exception-with-kind-and-args? c) - (make-warning package - (G_ "failed to create ~a derivation: ~s") - (list system - (cons (exception-kind c) - (exception-args c))))) - ((message-condition? c) - (make-warning package - (G_ "failed to create ~a derivation: ~a") - (list system - (condition-message c)))) - ((formatted-message? c) - (let ((str (apply format #f - (formatted-message-string c) - (formatted-message-arguments c)))) - (make-warning package - (G_ "failed to create ~a derivation: ~a") - (list system str))))) - (parameterize ((%graft? #f)) - (package-derivation store package system #:graft? #f) + (guard (c ((store-protocol-error? c) + (make-warning package + (G_ "failed to create ~a derivation: ~a") + (list system + (store-protocol-error-message c)))) + ((exception-with-kind-and-args? c) + (make-warning package + (G_ "failed to create ~a derivation: ~s") + (list system + (cons (exception-kind c) + (exception-args c))))) + ((message-condition? c) + (make-warning package + (G_ "failed to create ~a derivation: ~a") + (list system + (condition-message c)))) + ((formatted-message? c) + (let ((str (apply format #f + (formatted-message-string c) + (formatted-message-arguments c)))) + (make-warning package + (G_ "failed to create ~a derivation: ~a") + (list system str))))) + (parameterize ((%graft? #f)) + (package-derivation store package system #:graft? #f) - ;; If there's a replacement, make sure we can compute its - ;; derivation. - (match (package-replacement package) - (#f #t) - (replacement - (package-derivation store replacement system - #:graft? #f)))))) - (lambda args - (make-warning package - (G_ "failed to create ~a derivation: ~s") - (list system args))))) + ;; If there's a replacement, make sure we can compute its + ;; derivation. + (match (package-replacement package) + (#f #t) + (replacement + (package-derivation store replacement system + #:graft? #f)))))) (define (check-with-store store) (filter lint-warning? @@ -1098,46 +1347,6 @@ of the propagated inputs it pulls in." (make-warning package (G_ "invalid license field") #:field 'license))))) -(define (call-with-networking-fail-safe message error-value proc) - "Call PROC catching any network-related errors. Upon a networking error, -display a message including MESSAGE and return ERROR-VALUE." - (guard (c ((http-get-error? c) - (warning (G_ "~a: HTTP GET error for ~a: ~a (~s)~%") - message - (uri->string (http-get-error-uri c)) - (http-get-error-code c) - (http-get-error-reason c)) - error-value)) - (catch #t - proc - (match-lambda* - (('getaddrinfo-error errcode) - (warning (G_ "~a: host lookup failure: ~a~%") - message - (gai-strerror errcode)) - error-value) - (('tls-certificate-error args ...) - (warning (G_ "~a: TLS certificate error: ~a") - message - (tls-certificate-error-string args)) - error-value) - ((and ('system-error _ ...) args) - (let ((errno (system-error-errno args))) - (if (member errno (list ECONNRESET ECONNABORTED ECONNREFUSED)) - (let ((details (call-with-output-string - (lambda (port) - (print-exception port #f (car args) - (cdr args)))))) - (warning (G_ "~a: ~a~%") message details) - error-value) - (apply throw args)))) - (args - (apply throw args)))))) - -(define-syntax-rule (with-networking-fail-safe message error-value exp ...) - (call-with-networking-fail-safe message error-value - (lambda () exp ...))) - (define (current-vulnerabilities*) "Like 'current-vulnerabilities', but return the empty list upon networking or HTTP errors. This allows network-less operation and makes problems with @@ -1219,6 +1428,43 @@ upstream releases") #:field 'source))))))) +(define (lookup-disarchive-spec hash) + "If Disarchive mirrors have a spec for HASH, return the list of SWH +directory identifiers the spec refers to. Otherwise return #f." + (define (extract-swh-id spec) + ;; Return the list of SWH directory identifiers SPEC refers to, where SPEC + ;; is a Disarchive sexp. Instead of attempting to parse it, traverse it + ;; in a pretty unintelligent fashion. + (let loop ((sexp spec) + (ids '())) + (match sexp + ((? string? str) + (let ((prefix "swh:1:dir:")) + (if (string-prefix? prefix str) + (cons (string-drop str (string-length prefix)) ids) + ids))) + ((head tail ...) + (loop tail (loop head ids))) + (_ ids)))) + + (any (lambda (mirror) + (with-networking-fail-safe + (format #f (G_ "failed to access Disarchive database at ~a") + mirror) + #f + (guard (c ((http-get-error? c) #f)) + (let* ((url (string-append mirror + (symbol->string + (content-hash-algorithm hash)) + "/" + (bytevector->base16-string + (content-hash-value hash)))) + (port (http-fetch (string->uri url) #:text? #t)) + (spec (read port))) + (close-port port) + (extract-swh-id spec))))) + %disarchive-mirrors)) + (define (check-archival package) "Check whether PACKAGE's source code is archived on Software Heritage. If it's not, and if its source code is a VCS snapshot, then send a \"save\" @@ -1294,13 +1540,33 @@ try again later") (symbol->string (content-hash-algorithm hash))) (#f - (list (make-warning package - (G_ "source not archived on Software \ -Heritage") - #:field 'source))) + ;; If SWH doesn't have HASH as is, it may be because it's + ;; a hand-crafted tarball. In that case, check whether + ;; the Disarchive database has an entry for that tarball. + (match (lookup-disarchive-spec hash) + (#f + (list (make-warning package + (G_ "source not archived on Software \ +Heritage and missing from the Disarchive database") + #:field 'source))) + (directory-ids + (match (find (lambda (id) + (not (lookup-directory id))) + directory-ids) + (#f '()) + (id + (list (make-warning package + (G_ " +Disarchive entry refers to non-existent SWH directory '~a'") + (list id) + #:field 'source))))))) ((? content?) '()))) - '())))) + '())) + (_ + (list (make-warning package + (G_ "unsupported source type") + #:field 'source))))) (match-lambda* (('swh-error url method response) (response->warning url method response)) @@ -1474,6 +1740,10 @@ them for PACKAGE." (description "Validate package names") (check check-name)) (lint-checker + (name 'tests-true) + (description "Check if tests are explicitly enabled") + (check check-tests-true)) + (lint-checker (name 'description) (description "Validate package descriptions") (check check-description-style)) @@ -1486,6 +1756,10 @@ them for PACKAGE." (description "Identify inputs that shouldn't be inputs at all") (check check-inputs-should-not-be-an-input-at-all)) (lint-checker + (name 'wrapper-inputs) + (description "Make sure 'wrap-program' can finds its interpreter.") + (check check-wrapper-inputs)) + (lint-checker (name 'license) ;; TRANSLATORS: <license> is the name of a data type and must not be ;; translated. @@ -1493,6 +1767,10 @@ them for PACKAGE." or a list thereof") (check check-license)) (lint-checker + (name 'optional-tests) + (description "Make sure tests are only run when requested") + (check check-optional-tests)) + (lint-checker (name 'mirror-url) (description "Suggest 'mirror://' URLs") (check check-mirror-url)) diff --git a/guix/packages.scm b/guix/packages.scm index 55e5e70b8c..8c3a0b0b7b 100644 --- a/guix/packages.scm +++ b/guix/packages.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> -;;; Copyright © 2014, 2015, 2017, 2018 Mark H Weaver <mhw@netris.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2017, 2018, 2019 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2015 Eric Bavier <bavier@member.fsf.org> ;;; Copyright © 2016 Alex Kost <alezost@gmail.com> ;;; Copyright © 2017, 2019, 2020 Efraim Flashner <efraim@flashner.co.il> @@ -52,6 +52,7 @@ #:re-export (%current-system %current-target-system search-path-specification) ;for convenience + #:replace ((define-public* . define-public)) #:export (content-hash content-hash? content-hash-algorithm @@ -99,6 +100,7 @@ package-supported-systems package-properties package-location + package-definition-location hidden-package hidden-package? package-superseded @@ -342,6 +344,27 @@ name of its URI." ;; git, svn, cvs, etc. reference #f)))) +;; Work around limitations in the 'snippet' mechanism. It is not possible for +;; a 'snippet' to produce a tarball with a different base name than the +;; original downloaded source. Moreover, cherry picking dozens of upsteam +;; patches and applying them suddenly is often impractical; especially when a +;; comprehensive code reformatting is done upstream. Mainly designed for +;; Linux and IceCat packages. +;; XXXX: do not make part of public API (export) such radical capability +;; before a detailed review process. +(define* (computed-origin-method gexp-promise hash-algo hash + #:optional (name "source") + #:key (system (%current-system)) + (guile (default-guile))) + "Return a derivation that executes the G-expression that results +from forcing GEXP-PROMISE." + (mlet %store-monad ((guile (package->derivation guile system))) + (gexp->derivation (or name "computed-origin") + (force gexp-promise) + #:graft? #f ;nothing to graft + #:system system + #:guile-for-build guile))) + (define %supported-systems ;; This is the list of system types that are supported. By default, we @@ -360,6 +383,59 @@ name of its URI." ;; <https://lists.gnu.org/archive/html/guix-devel/2017-03/msg00790.html>. (fold delete %supported-systems '("mips64el-linux"))) +(define-syntax current-location-vector + (lambda (s) + "Like 'current-source-location' but expand to a literal vector with +one-indexed line numbers." + ;; Storing a literal vector in .go files is more efficient than storing an + ;; alist: less initialization code, fewer relocations, etc. + (syntax-case s () + ((_) + (match (syntax-source s) + (#f #f) + (properties + (let ((file (assq-ref properties 'filename)) + (line (assq-ref properties 'line)) + (column (assq-ref properties 'column))) + (and file line column + #`#(#,file #,(+ 1 line) #,column))))))))) + +(define-inlinable (sanitize-location loc) + ;; Convert LOC to a vector or to #f. + (cond ((vector? loc) loc) + ((not loc) loc) + (else (vector (location-file loc) + (location-line loc) + (location-column loc))))) + +(define-syntax-parameter current-definition-location + ;; Location of the encompassing 'define-public'. + (const #f)) + +(define-syntax define-public* + (lambda (s) + "Like 'define-public' but set 'current-definition-location' for the +lexical scope of its body." + (define location + (match (syntax-source s) + (#f #f) + (properties + (let ((line (assq-ref properties 'line)) + (column (assq-ref properties 'column))) + ;; Don't repeat the file name since it's redundant with 'location'. + ;; Encode the whole thing so that it fits in a fixnum on 32-bit + ;; platforms, which leaves us 29 bits: 7 bits for COLUMN (which is + ;; almost always zero), and 22 bits for LINE. + (and line column + (logior (ash (logand #x7f column) 22) + (logand (- (expt 2 22) 1) (+ 1 line)))))))) + + (syntax-case s () + ((_ prototype body ...) + #`(define-public prototype + (syntax-parameterize ((current-definition-location + (lambda (s) #,location))) + body ...)))))) ;; A package. (define-record-type* <package> @@ -404,10 +480,12 @@ name of its URI." (properties package-properties (default '())) ; alist for anything else - (location package-location - (default (and=> (current-source-location) - source-properties->location)) - (innate))) + (location package-location-vector + (default (current-location-vector)) + (innate) (sanitize sanitize-location)) + (definition-location package-definition-location-code + (default (current-definition-location)) + (innate))) (set-record-type-printer! <package> (lambda (package port) @@ -425,6 +503,25 @@ name of its URI." package) 16))))) +(define (package-location package) + "Return the source code location of PACKAGE as a <location> record, or #f if +it is not known." + (match (package-location-vector package) + (#f #f) + (#(file line column) (location file line column)))) + +(define (package-definition-location package) + "Like 'package-location', but return the location of the definition +itself--i.e., that of the enclosing 'define-public' form, if any, or #f." + (match (package-definition-location-code package) + (#f #f) + (code + (let ((column (bit-extract code 22 29)) + (line (bit-extract code 0 21))) + (match (package-location-vector package) + (#f #f) + (#(file _ _) (location file line column))))))) + (define-syntax-rule (package/inherit p overrides ...) "Like (package (inherit P) OVERRIDES ...), except that the same transformation is done to the package P's replacement, if any. P must be a bare @@ -790,7 +887,8 @@ specifies modules in scope when evaluating SNIPPET." "Return package ORIGINAL with PATCHES applied." (package (inherit original) (source (origin (inherit (package-source original)) - (patches patches))))) + (patches patches))) + (location (package-location original)))) (define (package-with-extra-patches original patches) "Return package ORIGINAL with all PATCHES appended to its list of patches." diff --git a/guix/profiles.scm b/guix/profiles.scm index 67d90532c1..2486f91d09 100644 --- a/guix/profiles.scm +++ b/guix/profiles.scm @@ -4,13 +4,14 @@ ;;; Copyright © 2014, 2016 Alex Kost <alezost@gmail.com> ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2015 Sou Bunnbu <iyzsong@gmail.com> -;;; Copyright © 2016, 2018, 2019 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2016, 2018, 2019, 2021 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2016 Chris Marusich <cmmarusich@gmail.com> ;;; Copyright © 2017 Huang Ying <huang.ying.caritas@gmail.com> ;;; Copyright © 2017 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; Copyright © 2019 Kyle Meyer <kyle@kyleam.com> ;;; Copyright © 2019 Mathieu Othacehe <m.othacehe@gmail.com> ;;; Copyright © 2020 Danny Milosavljevic <dannym@scratchpost.org> +;;; Copyright © 2014 David Thompson <davet@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -54,6 +55,7 @@ #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) + #:autoload (srfi srfi-98) (get-environment-variables) #:export (&profile-error profile-error? profile-error-profile @@ -127,6 +129,7 @@ %default-profile-hooks profile-derivation profile-search-paths + load-profile profile profile? @@ -334,7 +337,10 @@ file name." (filter-map (lambda (entry) (let ((other (lookup (manifest-entry-name entry) (manifest-entry-output entry)))) - (and other (list entry other)))) + (and other + (not (eq? (manifest-entry-item entry) + (manifest-entry-item other))) + (list entry other)))) (manifest-transitive-entries manifest))) (define lower-pair @@ -1115,6 +1121,46 @@ MANIFEST. Single-file bundles are required by programs such as Git and Lynx." `((type . profile-hook) (hook . ca-certificate-bundle)))) +(define (emacs-subdirs manifest) + (define build + (with-imported-modules (source-module-closure + '((guix build profiles) + (guix build utils))) + #~(begin + (use-modules (guix build utils) + (guix build profiles) + (ice-9 ftw) ; scandir + (srfi srfi-1) ; append-map + (srfi srfi-26)) + + (let ((destdir (string-append #$output "/share/emacs/site-lisp")) + (subdirs + (append-map + (lambda (dir) + (filter + file-is-directory? + (map (cute string-append dir "/" <>) + (scandir dir (negate (cute member <> '("." ".."))))))) + (filter file-exists? + (map (cute string-append <> "/share/emacs/site-lisp") + '#$(manifest-inputs manifest)))))) + (mkdir-p destdir) + (with-directory-excursion destdir + (call-with-output-file "subdirs.el" + (lambda (port) + (write + `(normal-top-level-add-to-load-path + (list ,@(delete-duplicates subdirs))) + port) + (newline port) + #t))))))) + (gexp->derivation "emacs-subdirs" build + #:local-build? #t + #:substitutable? #f + #:properties + `((type . profile-hook) + (hook . emacs-subdirs)))) + (define (glib-schemas manifest) "Return a derivation that unions all schemas from manifest entries and creates the Glib 'gschemas.compiled' file." @@ -1627,12 +1673,22 @@ MANIFEST." (cons (gexp-input thing output) (append-map entry->texlive-input deps)) '())))) + (define texlive-bin + (module-ref (resolve-interface '(gnu packages tex)) 'texlive-bin)) + (define coreutils + (module-ref (resolve-interface '(gnu packages base)) 'coreutils)) + (define sed + (module-ref (resolve-interface '(gnu packages base)) 'sed)) + (define updmap.cfg + (module-ref (resolve-interface '(gnu packages tex)) + 'texlive-default-updmap.cfg)) (define build (with-imported-modules '((guix build utils) (guix build union)) #~(begin (use-modules (guix build utils) - (guix build union)) + (guix build union) + (ice-9 popen)) ;; Build a modifiable union of all texlive inputs. We do this so ;; that TeX live can resolve the parent and grandparent directories @@ -1650,19 +1706,60 @@ MANIFEST." (("^TEXMFROOT = .*") (string-append "TEXMFROOT = " #$output "/share\n")) (("^TEXMF = .*") - "TEXMF = $TEXMFROOT/share/texmf-dist\n")))) + "TEXMF = $TEXMFROOT/share/texmf-dist\n")) + + ;; XXX: This is annoying, but it's necessary because texlive-bin + ;; does not provide wrapped executables. + (setenv "PATH" + (string-append #$(file-append coreutils "/bin") + ":" + #$(file-append sed "/bin"))) + (setenv "PERL5LIB" #$(file-append texlive-bin "/share/tlpkg")) + (setenv "TEXMF" (string-append #$output "/share/texmf-dist")) + + ;; Remove invalid maps from config file. + (let* ((web2c (string-append #$output "/share/texmf-config/web2c/")) + (maproot (string-append #$output "/share/texmf-dist/fonts/map/")) + (updmap.cfg (string-append web2c "updmap.cfg"))) + (mkdir-p web2c) + + ;; Some profiles may already have this file, which prevents us + ;; from copying it. Since we need to generate it from scratch + ;; anyway, we delete it here. + (when (file-exists? updmap.cfg) + (delete-file updmap.cfg)) + (copy-file #$updmap.cfg updmap.cfg) + (make-file-writable updmap.cfg) + (let* ((port (open-pipe* OPEN_WRITE + #$(file-append texlive-bin "/bin/updmap-sys") + "--syncwithtrees" + "--nohash" + "--force" + (string-append "--cnffile=" web2c "updmap.cfg")))) + (display "Y\n" port) + (when (not (zero? (status:exit-val (close-pipe port)))) + (error "failed to filter updmap.cfg"))) + + ;; Generate font maps. + (invoke #$(file-append texlive-bin "/bin/updmap-sys") + (string-append "--cnffile=" web2c "updmap.cfg") + (string-append "--dvipdfmxoutputdir=" + maproot "updmap/dvipdfmx/") + (string-append "--dvipsoutputdir=" + maproot "updmap/dvips/") + (string-append "--pdftexoutputdir=" + maproot "updmap/pdftex/"))))) #t))) - (with-monad %store-monad - (if (any (cut string-prefix? "texlive-" <>) - (map manifest-entry-name (manifest-entries manifest))) - (gexp->derivation "texlive-configuration" build - #:substitutable? #f - #:local-build? #t - #:properties - `((type . profile-hook) - (hook . texlive-configuration))) - (return #f)))) + (mlet %store-monad ((texlive-base (manifest-lookup-package manifest "texlive-base"))) + (if texlive-base + (gexp->derivation "texlive-configuration" build + #:substitutable? #f + #:local-build? #t + #:properties + `((type . profile-hook) + (hook . texlive-configuration))) + (return #f)))) (define %default-profile-hooks ;; This is the list of derivation-returning procedures that are called by @@ -1672,6 +1769,7 @@ MANIFEST." fonts-dir-file ghc-package-cache-file ca-certificate-bundle + emacs-subdirs glib-schemas gtk-icon-themes gtk-im-modules @@ -1717,12 +1815,10 @@ are cross-built for TARGET." (mapm/accumulate-builds (lambda (hook) (hook manifest)) hooks)))) - (define inputs - (append (filter-map (lambda (drv) - (and (derivation? drv) - (gexp-input drv))) - extras) - (manifest-inputs manifest))) + (define extra-inputs + (filter-map (lambda (drv) + (and (derivation? drv) (gexp-input drv))) + extras)) (define glibc-utf8-locales ;lazy reference (module-ref (resolve-interface '(gnu packages base)) @@ -1756,20 +1852,11 @@ are cross-built for TARGET." #+(if locales? set-utf8-locale #t) - (define search-paths - ;; Search paths of MANIFEST's packages, converted back to their - ;; record form. - (map sexp->search-path-specification - (delete-duplicates - '#$(map search-path-specification->sexp - (manifest-search-paths manifest))))) - - (build-profile #$output '#$inputs + (build-profile #$output '#$(manifest->gexp manifest) + #:extra-inputs '#$extra-inputs #:symlink #$(if relative-symlinks? #~symlink-relative - #~symlink) - #:manifest '#$(manifest->gexp manifest) - #:search-paths search-paths)))) + #~symlink))))) (gexp->derivation name builder #:system system @@ -1832,6 +1919,44 @@ already effective." (evaluate-search-paths (manifest-search-paths manifest) (list profile) getenv)) +(define %precious-variables + ;; Environment variables in the default 'load-profile' white list. + '("HOME" "USER" "LOGNAME" "DISPLAY" "XAUTHORITY" "TERM" "TZ" "PAGER")) + +(define (purify-environment white-list white-list-regexps) + "Unset all environment variables except those that match the regexps in +WHITE-LIST-REGEXPS and those listed in WHITE-LIST." + (for-each unsetenv + (remove (lambda (variable) + (or (member variable white-list) + (find (cut regexp-exec <> variable) + white-list-regexps))) + (match (get-environment-variables) + (((names . _) ...) + names))))) + +(define* (load-profile profile + #:optional (manifest (profile-manifest profile)) + #:key pure? (white-list-regexps '()) + (white-list %precious-variables)) + "Set the environment variables specified by MANIFEST for PROFILE. When +PURE? is #t, unset the variables in the current environment except those that +match the regexps in WHITE-LIST-REGEXPS and those listed in WHITE-LIST. +Otherwise, augment existing environment variables with additional search +paths." + (when pure? + (purify-environment white-list white-list-regexps)) + (for-each (match-lambda + ((($ <search-path-specification> variable _ separator) . value) + (let ((current (getenv variable))) + (setenv variable + (if (and current (not pure?)) + (if separator + (string-append value separator current) + value) + value))))) + (profile-search-paths profile manifest))) + (define (profile-regexp profile) "Return a regular expression that matches PROFILE's name and number." (make-regexp (string-append "^" (regexp-quote (basename profile)) diff --git a/guix/progress.scm b/guix/progress.scm index 334bd40547..0cbc804ec1 100644 --- a/guix/progress.scm +++ b/guix/progress.scm @@ -347,15 +347,25 @@ should be a <progress-reporter> object." (report total) (loop total (get-bytevector-n! in buffer 0 buffer-size)))))))) -(define* (progress-report-port reporter port #:key (close? #t)) +(define* (progress-report-port reporter port + #:key + (close? #t) + download-size) "Return a port that continuously reports the bytes read from PORT using REPORTER, which should be a <progress-reporter> object. When CLOSE? is true, -PORT is closed when the returned port is closed." +PORT is closed when the returned port is closed. + +When DOWNLOAD-SIZE is passed, do not read more than DOWNLOAD-SIZE bytes from +PORT. This is important to avoid blocking when the remote side won't close +the underlying connection." (match reporter (($ <progress-reporter> start report stop) (let* ((total 0) (read! (lambda (bv start count) - (let ((n (match (get-bytevector-n! port bv start count) + (let* ((count (if download-size + (min count (- download-size total)) + count)) + (n (match (get-bytevector-n! port bv start count) ((? eof-object?) 0) (x x)))) (set! total (+ total n)) diff --git a/guix/records.scm b/guix/records.scm index 3d54a51956..ed94c83dac 100644 --- a/guix/records.scm +++ b/guix/records.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Mark H Weaver <mhw@netris.org> ;;; ;;; This file is part of GNU Guix. @@ -120,7 +120,8 @@ context of the definition of a thunked field." "Make the syntactic constructor NAME for TYPE, that calls CTOR, and expects all of EXPECTED fields to be initialized. DEFAULTS is the list of FIELD/DEFAULT-VALUE tuples, THUNKED is the list of identifiers of thunked -fields, and DELAYED is the list of identifiers of delayed fields. +fields, DELAYED is the list of identifiers of delayed fields, and SANITIZERS +is the list of FIELD/SANITIZER tuples. ABI-COOKIE is the cookie (an integer) against which to check the run-time ABI of TYPE matches the expansion-time ABI." @@ -130,6 +131,7 @@ of TYPE matches the expansion-time ABI." #:this-identifier this-identifier #:delayed delayed #:innate innate + #:sanitizers sanitizers #:defaults defaults) (define-syntax name (lambda (s) @@ -169,19 +171,30 @@ of TYPE matches the expansion-time ABI." (define (innate-field? f) (memq (syntax->datum f) 'innate)) + (define field-sanitizer + (let ((lst (map (match-lambda + ((f p) + (list (syntax->datum f) p))) + #'sanitizers))) + (lambda (f) + (or (and=> (assoc-ref lst (syntax->datum f)) car) + #'(lambda (x) x))))) + (define (wrap-field-value f value) - (cond ((thunked-field? f) - #`(lambda (x) - (syntax-parameterize ((#,this-identifier - (lambda (s) - (syntax-case s () - (id - (identifier? #'id) - #'x))))) - #,value))) - ((delayed-field? f) - #`(delay #,value)) - (else value))) + (let* ((sanitizer (field-sanitizer f)) + (value #`(#,sanitizer #,value))) + (cond ((thunked-field? f) + #`(lambda (x) + (syntax-parameterize ((#,this-identifier + (lambda (s) + (syntax-case s () + (id + (identifier? #'id) + #'x))))) + #,value))) + ((delayed-field? f) + #`(delay #,value)) + (else value)))) (define default-values ;; List of symbol/value tuples. @@ -291,6 +304,19 @@ can access the record it belongs to via the 'this-thing' identifier. A field can also be marked as \"delayed\" instead of \"thunked\", in which case its value is effectively wrapped in a (delay …) form. +A field can also have an associated \"sanitizer\", which is a procedure that +takes a user-supplied field value and returns a \"sanitized\" value for the +field: + + (define-record-type* <thing> thing make-thing + thing? + this-thing + (name thing-name + (sanitize (lambda (value) + (cond ((string? value) value) + ((symbol? value) (symbol->string value)) + (else (throw 'bad! value))))))) + It is possible to copy an object 'x' created with 'thing' like this: (thing (inherit x) (name \"bar\")) @@ -307,6 +333,14 @@ inherited." (field-default-value #'(field properties ...))) (_ #f))) + (define (field-sanitizer s) + (syntax-case s (sanitize) + ((field (sanitize proc) _ ...) + (list #'field #'proc)) + ((field _ properties ...) + (field-sanitizer #'(field properties ...))) + (_ #f))) + (define-field-property-predicate delayed-field? delayed) (define-field-property-predicate thunked-field? thunked) (define-field-property-predicate innate-field? innate) @@ -376,6 +410,8 @@ inherited." (innate (filter-map innate-field? field-spec)) (defaults (filter-map field-default-value #'((field properties ...) ...))) + (sanitizers (filter-map field-sanitizer + #'((field properties ...) ...))) (cookie (compute-abi-cookie field-spec))) (with-syntax (((field-spec* ...) (map field-spec->srfi-9 field-spec)) @@ -421,6 +457,7 @@ of a record instantiation" #:this-identifier #'this-identifier #:delayed #,delayed #:innate #,innate + #:sanitizers #,sanitizers #:defaults #,defaults))))) ((_ type syntactic-ctor ctor pred (field get properties ...) ...) diff --git a/guix/scripts/archive.scm b/guix/scripts/archive.scm index ceac640432..f8678aa5f9 100644 --- a/guix/scripts/archive.scm +++ b/guix/scripts/archive.scm @@ -260,6 +260,9 @@ build and a list of store files to transfer." resulting archive to the standard output port." (let-values (((drv files) (options->derivations+files store opts))) + (when (null? files) + (warning (G_ "no arguments specified; creating an empty archive~%"))) + (if (build-derivations store drv) (export-paths store files (current-output-port) #:recursive? (assoc-ref opts 'export-recursive?)) diff --git a/guix/scripts/build.scm b/guix/scripts/build.scm index 2decdb45ed..97e2f5a167 100644 --- a/guix/scripts/build.scm +++ b/guix/scripts/build.scm @@ -679,6 +679,9 @@ needed." (_ #f)) opts))) + (when (and (null? drv) (null? items)) + (warning (G_ "no arguments specified, nothing to do~%"))) + (cond ((assoc-ref opts 'log-file?) ;; Pass 'show-build-log' the output file names, not the ;; derivation file names, because there can be several diff --git a/guix/scripts/challenge.scm b/guix/scripts/challenge.scm index 4ec3be99ca..69c2781abb 100644 --- a/guix/scripts/challenge.scm +++ b/guix/scripts/challenge.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2016, 2017, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2017, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -253,10 +253,12 @@ taken since we do not import the archives." NARINFO." (let*-values (((uri compression size) (narinfo-best-uri narinfo)) - ((port response) + ((port actual-size) (http-fetch uri))) (define reporter - (progress-reporter/file (narinfo-path narinfo) size + (progress-reporter/file (narinfo-path narinfo) + (and size + (max size (or actual-size 0))) ;defensive #:abbreviation (const (uri-host uri)))) (define result diff --git a/guix/scripts/copy.scm b/guix/scripts/copy.scm index 52b476db54..07357af420 100644 --- a/guix/scripts/copy.scm +++ b/guix/scripts/copy.scm @@ -62,6 +62,10 @@ number (or #f) corresponding to SPEC." (x (leave (G_ "~a: invalid SSH specification~%") spec)))) +(define (warn-if-empty items) + (when (null? items) + (warning (G_ "no arguments specified, nothing to copy~%")))) + (define (send-to-remote-host local target opts) "Send ITEMS to TARGET. ITEMS is a list of store items or package names; for ; package names, build the underlying packages before sending them." @@ -69,6 +73,7 @@ package names, build the underlying packages before sending them." (ssh-spec->user+host+port target)) ((drv items) (options->derivations+files local opts))) + (warn-if-empty items) (and (build-derivations local drv) (let* ((session (open-ssh-session host #:user user #:port (or port 22))) @@ -94,7 +99,9 @@ package names, build the underlying packages before sending them." (let*-values (((drv items) (options->derivations+files local opts)) ((retrieved) - (retrieve-files local items remote #:recursive? #t))) + (begin + (warn-if-empty items) + (retrieve-files local items remote #:recursive? #t)))) (close-connection remote) (disconnect! session) (format #t "~{~a~%~}" retrieved) diff --git a/guix/scripts/deploy.scm b/guix/scripts/deploy.scm index 0725fba54b..1707622c4f 100644 --- a/guix/scripts/deploy.scm +++ b/guix/scripts/deploy.scm @@ -1,7 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019 David Thompson <davet@gnu.org> ;;; Copyright © 2019 Jakob L. Kreuze <zerodaysfordays@sdf.org> -;;; Copyright © 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -28,6 +28,8 @@ #:use-module (guix utils) #:use-module (guix grafts) #:use-module (guix status) + #:use-module (guix diagnostics) + #:use-module (guix i18n) #:use-module (ice-9 format) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) @@ -125,16 +127,20 @@ Perform the deployment specified by FILE.\n")) ;; and include a '&message'. However, that message only contains ;; the format string. Thus, special-case it here to avoid ;; displaying a bare format string. - ((cond-expand - (guile-3 - ((exception-predicate &exception-with-kind-and-args) c)) - (else #f)) + (((exception-predicate &exception-with-kind-and-args) c) (raise c)) ((message-condition? c) - (report-error (G_ "failed to deploy ~a: ~a~%") - (machine-display-name machine) - (condition-message c))) + (leave (G_ "failed to deploy ~a: ~a~%") + (machine-display-name machine) + (condition-message c))) + ((formatted-message? c) + (leave (G_ "failed to deploy ~a: ~a~%") + (machine-display-name machine) + (apply format #f + (gettext (formatted-message-string c) + %gettext-domain) + (formatted-message-arguments c)))) ((deploy-error? c) (when (deploy-error-should-roll-back c) (info (G_ "rolling back ~a...~%") @@ -156,7 +162,10 @@ Perform the deployment specified by FILE.\n")) (let* ((opts (parse-command-line args %options (list %default-options) #:argument-handler handle-argument)) (file (assq-ref opts 'file)) - (machines (or (and file (load-source-file file)) '()))) + (machines (and file (load-source-file file)))) + (unless file + (leave (G_ "missing deployment file argument~%"))) + (show-what-to-deploy machines) (with-status-verbosity (assoc-ref opts 'verbosity) diff --git a/guix/scripts/describe.scm b/guix/scripts/describe.scm index b5f6249176..a3e3338f7e 100644 --- a/guix/scripts/describe.scm +++ b/guix/scripts/describe.scm @@ -301,4 +301,11 @@ text. The hyperlink links to a web view of COMMIT, when available." (channels (display-profile-info #f format channels)))) (profile - (display-profile-info (canonicalize-profile profile) format)))))) + ;; For the current profile, resort to 'current-channels', which has a + ;; fallback to metadata from (guix config) in case PROFILE lacks it. + (let ((channels (if (and (current-profile) + (string=? profile (current-profile))) + (current-channels) + (profile-channels profile)))) + (display-profile-info (canonicalize-profile profile) + format channels))))))) diff --git a/guix/scripts/discover.scm b/guix/scripts/discover.scm index be1eaa6e95..dadade81bb 100644 --- a/guix/scripts/discover.scm +++ b/guix/scripts/discover.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2020 Mathieu Othacehe <othacehe@gnu.org> +;;; Copyright © 2020, 2021 Mathieu Othacehe <othacehe@gnu.org> ;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. @@ -26,6 +26,7 @@ #:use-module (guix build syscalls) #:use-module (guix build utils) #:use-module (guix scripts publish) + #:use-module (avahi) #:use-module (ice-9 rdelim) #:use-module (srfi srfi-37) #:export (read-substitute-urls @@ -138,5 +139,16 @@ to synchronize with the writer." (parameterize ((%publish-file publish-file)) (mkdir-p (dirname publish-file)) (false-if-exception (delete-file publish-file)) - (avahi-browse-service-thread service-proc - #:types %services))))) + (catch 'avahi-error + (lambda () + (avahi-browse-service-thread service-proc + #:types %services)) + (lambda (key err function . _) + (cond + ((eq? err error/no-daemon) + (warning (G_ "Avahi daemon is not running, \ +cannot auto-discover substitutes servers.~%"))) + (else + (report-error (G_ "an Avahi error was raised by `~a': ~a~%") + function (error->string err)))) + (exit 1))))))) diff --git a/guix/scripts/edit.scm b/guix/scripts/edit.scm index b4c0507591..a2e1ffb434 100644 --- a/guix/scripts/edit.scm +++ b/guix/scripts/edit.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2016, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2015 Mathieu Lirzin <mthl@gnu.org> ;;; Copyright © 2020, 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; @@ -91,6 +91,8 @@ line." (with-error-handling (let* ((specs (reverse (parse-arguments))) (locations (map specification->location specs))) + (when (null? specs) + (leave (G_ "no packages specified, nothing to edit~%"))) (catch 'system-error (lambda () diff --git a/guix/scripts/environment.scm b/guix/scripts/environment.scm index 0360761683..6958bd6238 100644 --- a/guix/scripts/environment.scm +++ b/guix/scripts/environment.scm @@ -52,50 +52,9 @@ #:export (assert-container-features guix-environment)) -;; Protect some env vars from purification. Borrowed from nix-shell. -(define %precious-variables - '("HOME" "USER" "LOGNAME" "DISPLAY" "TERM" "TZ" "PAGER")) - (define %default-shell (or (getenv "SHELL") "/bin/sh")) -(define (purify-environment white-list) - "Unset all environment variables except those that match the regexps in -WHITE-LIST and those listed in %PRECIOUS-VARIABLES. A small number of -variables such as 'HOME' and 'USER' are left untouched." - (for-each unsetenv - (remove (lambda (variable) - (or (member variable %precious-variables) - (find (cut regexp-exec <> variable) - white-list))) - (match (get-environment-variables) - (((names . _) ...) - names))))) - -(define* (create-environment profile manifest - #:key pure? (white-list '())) - "Set the environment variables specified by MANIFEST for PROFILE. When -PURE? is #t, unset the variables in the current environment except those that -match the regexps in WHITE-LIST. Otherwise, augment existing environment -variables with additional search paths." - (when pure? - (purify-environment white-list)) - (for-each (match-lambda - ((($ <search-path-specification> variable _ separator) . value) - (let ((current (getenv variable))) - (setenv variable - (if (and current (not pure?)) - (if separator - (string-append value separator current) - value) - value))))) - (profile-search-paths profile manifest)) - - ;; Give users a way to know that they're in 'guix environment', so they can - ;; adjust 'PS1' accordingly, for instance. Set it to PROFILE so users can - ;; conveniently access its contents. - (setenv "GUIX_ENVIRONMENT" profile)) - (define* (show-search-paths profile manifest #:key pure?) "Display the search paths of MANIFEST applied to PROFILE. When PURE? is #t, do not augment existing environment variables with additional search paths." @@ -425,8 +384,14 @@ regexps in WHITE-LIST." ;; Properly handle SIGINT, so pressing C-c in an interactive terminal ;; application works. (sigaction SIGINT SIG_DFL) - (create-environment profile manifest - #:pure? pure? #:white-list white-list) + (load-profile profile manifest + #:pure? pure? #:white-list-regexps white-list) + + ;; Give users a way to know that they're in 'guix environment', so they can + ;; adjust 'PS1' accordingly, for instance. Set it to PROFILE so users can + ;; conveniently access its contents. + (setenv "GUIX_ENVIRONMENT" profile) + (match command ((program . args) (apply execlp program program args)))) @@ -755,6 +720,9 @@ message if any test fails." (> (length (manifest-entries manifest-from-opts)) 0)) (leave (G_ "'--profile' cannot be used with package options~%"))) + (when (null? (manifest-entries manifest)) + (warning (G_ "no packages specified; creating an empty environment~%"))) + (set-build-options-from-command-line store opts) ;; Use the bootstrap Guile when requested. diff --git a/guix/scripts/graph.scm b/guix/scripts/graph.scm index ddfc6ba497..439fae0b52 100644 --- a/guix/scripts/graph.scm +++ b/guix/scripts/graph.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2019 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. @@ -500,6 +500,10 @@ package modules, while attempting to retain user package modules." (lambda (opt name arg result) (alist-cons 'backend (lookup-backend arg) result))) + (option '(#\M "max-depth") #t #f + (lambda (opt name arg result) + (alist-cons 'max-depth (string->number* arg) + result))) (option '("list-backends") #f #f (lambda (opt name arg result) (list-backends) @@ -538,6 +542,8 @@ Emit a representation of the dependency graph of PACKAGE...\n")) (display (G_ " --list-types list the available graph types")) (display (G_ " + --max-depth=DEPTH limit to nodes within distance DEPTH")) + (display (G_ " --path display the shortest path between the given nodes")) (display (G_ " -e, --expression=EXPR consider the package EXPR evaluates to")) @@ -559,6 +565,7 @@ Emit a representation of the dependency graph of PACKAGE...\n")) (define %default-options `((node-type . ,%package-node-type) (backend . ,%graphviz-backend) + (max-depth . +inf.0) (system . ,(%current-system)))) @@ -582,6 +589,7 @@ Emit a representation of the dependency graph of PACKAGE...\n")) (with-store store (let* ((transform (options->transformation opts)) + (max-depth (assoc-ref opts 'max-depth)) (items (filter-map (match-lambda (('argument . (? store-path? item)) item) @@ -593,6 +601,9 @@ Emit a representation of the dependency graph of PACKAGE...\n")) (read/eval-package-expression exp))) (_ #f)) opts))) + (when (null? items) + (warning (G_ "no arguments specified; creating an empty graph~%"))) + (run-with-store store ;; XXX: Since grafting can trigger unsolicited builds, disable it. (mlet %store-monad ((_ (set-grafting #f)) @@ -610,7 +621,8 @@ nodes (given ~a)~%") (export-graph (concatenate nodes) (current-output-port) #:node-type type - #:backend backend))) + #:backend backend + #:max-depth max-depth))) #:system (assq-ref opts 'system))))) #t) diff --git a/guix/scripts/home.scm b/guix/scripts/home.scm new file mode 100644 index 0000000000..75df6d707d --- /dev/null +++ b/guix/scripts/home.scm @@ -0,0 +1,512 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Andrew Tropin <andrew@trop.in> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix scripts home) + #:use-module (gnu packages admin) + #:use-module ((gnu services) #:hide (delete)) + #:use-module (gnu packages) + #:use-module (gnu home) + #:use-module (gnu home-services) + #:use-module (guix channels) + #:use-module (guix derivations) + #:use-module (guix ui) + #:use-module (guix grafts) + #:use-module (guix packages) + #:use-module (guix profiles) + #:use-module (guix store) + #:use-module (guix utils) + #:use-module (guix scripts) + #:use-module (guix scripts package) + #:use-module (guix scripts build) + #:use-module (guix scripts system search) + #:autoload (guix scripts pull) (channel-commit-hyperlink) + #:use-module (guix scripts home import) + #:use-module ((guix status) #:select (with-status-verbosity)) + #:use-module (guix gexp) + #:use-module (guix monads) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) + #:use-module (srfi srfi-35) + #:use-module (srfi srfi-37) + #:use-module (ice-9 match) + #:export (guix-home)) + + +;;; +;;; Options. +;;; + +(define %user-module + (make-user-module '((gnu home)))) + +(define %guix-home + (string-append %profile-directory "/guix-home")) + +(define (show-help) + (display (G_ "Usage: guix home [OPTION ...] ACTION [ARG ...] [FILE] +Build the home environment declared in FILE according to ACTION. +Some ACTIONS support additional ARGS.\n")) + (newline) + (display (G_ "The valid values for ACTION are:\n")) + (newline) + (display (G_ "\ + search search for existing service types\n")) + (display (G_ "\ + reconfigure switch to a new home environment configuration\n")) + (display (G_ "\ + roll-back switch to the previous home environment configuration\n")) + (display (G_ "\ + describe describe the current home environment\n")) + (display (G_ "\ + list-generations list the home environment generations\n")) + (display (G_ "\ + switch-generation switch to an existing home environment configuration\n")) + (display (G_ "\ + delete-generations delete old home environment generations\n")) + (display (G_ "\ + build build the home environment without installing anything\n")) + (display (G_ "\ + import generates a home environment definition from dotfiles\n")) + + (show-build-options-help) + (display (G_ " + -v, --verbosity=LEVEL use the given verbosity LEVEL")) + (newline) + (display (G_ " + -h, --help display this help and exit")) + (display (G_ " + -V, --version display version information and exit")) + (newline) + (show-bug-report-information)) + +(define (verbosity-level opts) + "Return the verbosity level based on OPTS, the alist of parsed options." + (or (assoc-ref opts 'verbosity) + (if (eq? (assoc-ref opts 'action) 'build) + 2 1))) + +(define %options + ;; Specification of the command-line options. + (cons* (option '(#\h "help") #f #f + (lambda args + (show-help) + (exit 0))) + (option '(#\V "version") #f #f + (lambda args + (show-version-and-exit "guix show"))) + (option '(#\v "verbosity") #t #f + (lambda (opt name arg result) + (let ((level (string->number* arg))) + (alist-cons 'verbosity level + (alist-delete 'verbosity result))))) + %standard-build-options)) + +(define %default-options + `((build-mode . ,(build-mode normal)) + (graft? . #t) + (substitutes? . #t) + (offload? . #t) + (print-build-trace? . #t) + (print-extended-build-trace? . #t) + (multiplexed-build-output? . #t) + (verbosity . 3) + (debug . 0))) + + +;;; +;;; Actions. +;;; + +(define* (perform-action action he + #:key + dry-run? + derivations-only? + use-substitutes?) + "Perform ACTION for home environment. " + + (define println + (cut format #t "~a~%" <>)) + + (mlet* %store-monad + ((he-drv (home-environment-derivation he)) + (drvs (mapm/accumulate-builds lower-object (list he-drv))) + (% (if derivations-only? + (return + (for-each (compose println derivation-file-name) drvs)) + (built-derivations drvs))) + + (he-out-path -> (derivation->output-path he-drv))) + (if (or dry-run? derivations-only?) + (return #f) + (begin + (for-each (compose println derivation->output-path) drvs) + + (case action + ((reconfigure) + (let* ((number (generation-number %guix-home)) + (generation (generation-file-name + %guix-home (+ 1 number)))) + + (switch-symlinks generation he-out-path) + (switch-symlinks %guix-home generation) + (setenv "GUIX_NEW_HOME" he-out-path) + (primitive-load (string-append he-out-path "/activate")) + (setenv "GUIX_NEW_HOME" #f) + (return he-out-path))) + (else + (newline) + (return he-out-path))))))) + +(define (process-action action args opts) + "Process ACTION, a sub-command, with the arguments are listed in ARGS. +ACTION must be one of the sub-commands that takes a home environment +declaration as an argument (a file name.) OPTS is the raw alist of options +resulting from command-line parsing." + (define (ensure-home-environment file-or-exp obj) + (unless (home-environment? obj) + (leave (G_ "'~a' does not return a home environment ~%") + file-or-exp)) + obj) + + (let* ((file (match args + (() #f) + ((x . _) x))) + (expr (assoc-ref opts 'expression)) + (system (assoc-ref opts 'system)) + + (transform (lambda (obj) + (home-environment-with-provenance obj file))) + + (home-environment + (transform + (ensure-home-environment + (or file expr) + (cond + ((and expr file) + (leave + (G_ "both file and expression cannot be specified~%"))) + (expr + (read/eval expr)) + (file + (load* file %user-module + #:on-error (assoc-ref opts 'on-error))) + (else + (leave (G_ "no configuration specified~%"))))))) + + (dry? (assoc-ref opts 'dry-run?))) + + (with-store store + (set-build-options-from-command-line store opts) + (with-build-handler (build-notifier #:use-substitutes? + (assoc-ref opts 'substitutes?) + #:verbosity + (verbosity-level opts) + #:dry-run? + (assoc-ref opts 'dry-run?)) + + (run-with-store store + (mbegin %store-monad + (set-guile-for-build (default-guile)) + + (case action + (else + (perform-action action home-environment + #:dry-run? dry? + #:derivations-only? (assoc-ref opts 'derivations-only?) + #:use-substitutes? (assoc-ref opts 'substitutes?)) + )))))) + (warn-about-disk-space))) + + +(define (process-command command args opts) + "Process COMMAND, one of the 'guix home' sub-commands. ARGS is its +argument list and OPTS is the option alist." + (define-syntax-rule (with-store* store exp ...) + (with-store store + (set-build-options-from-command-line store opts) + exp ...)) + (case command + ;; The following commands do not need to use the store, and they do not need + ;; an home environment file. + ((search) + (apply search args)) + ((import) + (let* ((profiles (delete-duplicates + (match (filter-map (match-lambda + (('profile . p) p) + (_ #f)) + opts) + (() (list %current-profile)) + (lst (reverse lst))))) + (manifest (concatenate-manifests + (map profile-manifest profiles)))) + (import-manifest manifest (current-output-port)))) + ((describe) + (match (generation-number %guix-home) + (0 + (error (G_ "no home environment generation, nothing to describe~%"))) + (generation + (display-home-environment-generation generation)))) + ((list-generations) + (let ((pattern (match args + (() #f) + ((pattern) pattern) + (x (leave (G_ "wrong number of arguments~%")))))) + (list-generations pattern))) + ((switch-generation) + (let ((pattern (match args + ((pattern) pattern) + (x (leave (G_ "wrong number of arguments~%")))))) + (with-store* store + (switch-to-home-environment-generation store pattern)))) + ((roll-back) + (let ((pattern (match args + (() "") + (x (leave (G_ "wrong number of arguments~%")))))) + (with-store* store + (roll-back-home-environment store)))) + ((delete-generations) + (let ((pattern (match args + (() #f) + ((pattern) pattern) + (x (leave (G_ "wrong number of arguments~%")))))) + (with-store* + store + (delete-matching-generations store %guix-home pattern)))) + (else (process-action command args opts)))) + +(define-command (guix-home . args) + (synopsis "build and deploy home environments") + + (define (parse-sub-command arg result) + ;; Parse sub-command ARG and augment RESULT accordingly. + (if (assoc-ref result 'action) + (alist-cons 'argument arg result) + (let ((action (string->symbol arg))) + (case action + ((build + reconfigure + extension-graph shepherd-graph + list-generations describe + delete-generations roll-back + switch-generation search + import) + (alist-cons 'action action result)) + (else (leave (G_ "~a: unknown action~%") action)))))) + + (define (match-pair car) + ;; Return a procedure that matches a pair with CAR. + (match-lambda + ((head . tail) + (and (eq? car head) tail)) + (_ #f))) + + (define (option-arguments opts) + ;; Extract the plain arguments from OPTS. + (let* ((args (reverse (filter-map (match-pair 'argument) opts))) + (count (length args)) + (action (assoc-ref opts 'action)) + (expr (assoc-ref opts 'expression))) + (define (fail) + (leave (G_ "wrong number of arguments for action '~a'~%") + action)) + + (unless action + (format (current-error-port) + (G_ "guix home: missing command name~%")) + (format (current-error-port) + (G_ "Try 'guix home --help' for more information.~%")) + (exit 1)) + + (case action + ((build reconfigure) + (unless (or (= count 1) + (and expr (= count 0))) + (fail))) + ((init) + (unless (= count 2) + (fail)))) + args)) + + (with-error-handling + (let* ((opts (parse-command-line args %options + (list %default-options) + #:argument-handler + parse-sub-command)) + (args (option-arguments opts)) + (command (assoc-ref opts 'action))) + (parameterize ((%graft? (assoc-ref opts 'graft?))) + (with-status-verbosity (verbosity-level opts) + (process-command command args opts)))))) + + +;;; +;;; Searching. +;;; + +(define service-type-name* + (compose symbol->string service-type-name)) + +(define (service-type-description-string type) + "Return the rendered and localised description of TYPE, a service type." + (and=> (service-type-description type) + (compose texi->plain-text P_))) + +(define %service-type-metrics + ;; Metrics used to estimate the relevance of a search result. + `((,service-type-name* . 3) + (,service-type-description-string . 2) + (,(lambda (type) + (match (and=> (service-type-location type) location-file) + ((? string? file) + (basename file ".scm")) + (#f + ""))) + . 1))) + +(define (find-service-types regexps) + "Return a list of service type/score pairs: service types whose name or +description matches REGEXPS sorted by relevance, and their score." + (let ((matches (fold-home-service-types + (lambda (type result) + (match (relevance type regexps + %service-type-metrics) + ((? zero?) + result) + (score + (cons (cons type score) result)))) + '()))) + (sort matches + (lambda (m1 m2) + (match m1 + ((type1 . score1) + (match m2 + ((type2 . score2) + (if (= score1 score2) + (string>? (service-type-name* type1) + (service-type-name* type2)) + (> score1 score2)))))))))) + +(define (search . args) + (with-error-handling + (let* ((regexps (map (cut make-regexp* <> regexp/icase) args)) + (matches (find-service-types regexps))) + (leave-on-EPIPE + (display-search-results matches (current-output-port) + #:print service-type->recutils + #:command "guix home search"))))) + + +;;; +;;; Generations. +;;; + +(define* (display-home-environment-generation + number + #:optional (profile %guix-home)) + "Display a summary of home-environment generation NUMBER in a +human-readable format." + (define (display-channel channel) + (format #t " ~a:~%" (channel-name channel)) + (format #t (G_ " repository URL: ~a~%") (channel-url channel)) + (when (channel-branch channel) + (format #t (G_ " branch: ~a~%") (channel-branch channel))) + (format #t (G_ " commit: ~a~%") + (if (supports-hyperlinks?) + (channel-commit-hyperlink channel) + (channel-commit channel)))) + + (unless (zero? number) + (let* ((generation (generation-file-name profile number))) + (define-values (channels config-file) + ;; The function will work for home environments too, we just + ;; need to keep provenance file. + (system-provenance generation)) + + (display-generation profile number) + (format #t (G_ " file name: ~a~%") generation) + (format #t (G_ " canonical file name: ~a~%") (readlink* generation)) + ;; TRANSLATORS: Please preserve the two-space indentation. + + (unless (null? channels) + ;; TRANSLATORS: Here "channel" is the same terminology as used in + ;; "guix describe" and "guix pull --channels". + (format #t (G_ " channels:~%")) + (for-each display-channel channels)) + (when config-file + (format #t (G_ " configuration file: ~a~%") + (if (supports-hyperlinks?) + (file-hyperlink config-file) + config-file)))))) + +(define* (list-generations pattern #:optional (profile %guix-home)) + "Display in a human-readable format all the home environment +generations matching PATTERN, a string. When PATTERN is #f, display +all the home environment generations." + (cond ((not (file-exists? profile)) ; XXX: race condition + (raise (condition (&profile-not-found-error + (profile profile))))) + ((not pattern) + (for-each display-home-environment-generation (profile-generations profile))) + ((matching-generations pattern profile) + => + (lambda (numbers) + (if (null-list? numbers) + (exit 1) + (leave-on-EPIPE + (for-each display-home-environment-generation numbers))))))) + + +;;; +;;; Switch generations. +;;; + +;; TODO: Make it public in (guix scripts system) +(define-syntax-rule (unless-file-not-found exp) + (catch 'system-error + (lambda () + exp) + (lambda args + (if (= ENOENT (system-error-errno args)) + #f + (apply throw args))))) + +(define (switch-to-home-environment-generation store spec) + "Switch the home-environment profile to the generation specified by +SPEC. STORE is an open connection to the store." + (let* ((number (relative-generation-spec->number %guix-home spec)) + (generation (generation-file-name %guix-home number)) + (activate (string-append generation "/activate"))) + (if number + (begin + (setenv "GUIX_NEW_HOME" (readlink generation)) + (switch-to-generation* %guix-home number) + (unless-file-not-found (primitive-load activate)) + (setenv "GUIX_NEW_HOME" #f)) + (leave (G_ "cannot switch to home environment generation '~a'~%") spec)))) + + +;;; +;;; Roll-back. +;;; + +(define (roll-back-home-environment store) + "Roll back the home-environment profile to its previous generation. +STORE is an open connection to the store." + (switch-to-home-environment-generation store "-1")) diff --git a/guix/scripts/home/import.scm b/guix/scripts/home/import.scm new file mode 100644 index 0000000000..79fb23a2fd --- /dev/null +++ b/guix/scripts/home/import.scm @@ -0,0 +1,245 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; Copyright © 2021 Andrew Tropin <andrew@trop.in> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix scripts home import) + #:use-module (guix profiles) + #:use-module (guix ui) + #:use-module (guix utils) + #:use-module (guix packages) + #:use-module (gnu packages) + #:use-module (ice-9 match) + #:use-module (ice-9 pretty-print) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-26) + #:export (import-manifest)) + +;;; Commentary: +;;; +;;; This module provides utilities for generating home service +;;; configurations from existing "dotfiles". +;;; +;;; Code: + + +(define (generate-bash-module+configuration) + (let ((rc (string-append (getenv "HOME") "/.bashrc")) + (profile (string-append (getenv "HOME") "/.bash_profile")) + (logout (string-append (getenv "HOME") "/.bash_logout"))) + `((gnu home-services bash) + (service home-bash-service-type + (home-bash-configuration + ,@(if (file-exists? rc) + `((bashrc + (list (slurp-file-gexp (local-file ,rc))))) + '()) + ,@(if (file-exists? profile) + `((bash-profile + (list (slurp-file-gexp + (local-file ,profile))))) + '()) + ,@(if (file-exists? logout) + `((bash-logout + (list (slurp-file-gexp + (local-file ,logout))))) + '())))))) + + +(define %files-configurations-alist + `((".bashrc" . ,generate-bash-module+configuration) + (".bash_profile" . ,generate-bash-module+configuration) + (".bash_logout" . ,generate-bash-module+configuration))) + +(define (modules+configurations) + (let ((configurations (delete-duplicates + (filter-map (match-lambda + ((file . proc) + (if (file-exists? + (string-append (getenv "HOME") "/" file)) + proc + #f))) + %files-configurations-alist) + (lambda (x y) + (equal? (procedure-name x) (procedure-name y)))))) + (map (lambda (proc) (proc)) configurations))) + +;; Based on `manifest->code' from (guix profiles) +;; MAYBE: Upstream it? +(define* (manifest->code manifest + #:key + (entry-package-version (const "")) + (home-environment? #f)) + "Return an sexp representing code to build an approximate version of +MANIFEST; the code is wrapped in a top-level 'begin' form. If +HOME-ENVIRONMENT? is #t, return an <home-environment> definition. +Call ENTRY-PACKAGE-VERSION to determine the version number to use in +the spec for a given entry; it can be set to 'manifest-entry-version' +for fully-specified version numbers, or to some other procedure to +disambiguate versions for packages for which several versions are +available." + (define (entry-transformations entry) + ;; Return the transformations that apply to ENTRY. + (assoc-ref (manifest-entry-properties entry) 'transformations)) + + (define transformation-procedures + ;; List of transformation options/procedure name pairs. + (let loop ((entries (manifest-entries manifest)) + (counter 1) + (result '())) + (match entries + (() result) + ((entry . tail) + (match (entry-transformations entry) + (#f + (loop tail counter result)) + (options + (if (assoc-ref result options) + (loop tail counter result) + (loop tail (+ 1 counter) + (alist-cons options + (string->symbol + (format #f "transform~a" counter)) + result))))))))) + + (define (qualified-name entry) + ;; Return the name of ENTRY possibly with "@" followed by a version. + (match (entry-package-version entry) + ("" (manifest-entry-name entry)) + (version (string-append (manifest-entry-name entry) + "@" version)))) + + (if (null? transformation-procedures) + (let ((specs (map (lambda (entry) + (match (manifest-entry-output entry) + ("out" (qualified-name entry)) + (output (string-append (qualified-name entry) + ":" output)))) + (manifest-entries manifest)))) + (if home-environment? + (let ((modules+configurations (modules+configurations))) + `(begin + (use-modules (gnu home) + (gnu packages) + ,@(map first modules+configurations)) + ,(home-environment-template + #:specs specs + #:services (map second modules+configurations)))) + `(begin + (use-modules (gnu packages)) + + (specifications->manifest + (list ,@specs))))) + (let* ((transform (lambda (options exp) + (if (not options) + exp + (let ((proc (assoc-ref transformation-procedures + options))) + `(,proc ,exp))))) + (packages (map (lambda (entry) + (define options + (entry-transformations entry)) + + (define name + (qualified-name entry)) + + (match (manifest-entry-output entry) + ("out" + (transform options + `(specification->package ,name))) + (output + `(list ,(transform + options + `(specification->package ,name)) + ,output)))) + (manifest-entries manifest))) + (transformations (map (match-lambda + ((options . name) + `(define ,name + (options->transformation ',options)))) + transformation-procedures))) + (if home-environment? + (let ((modules+configurations (modules+configurations))) + `(begin + (use-modules (guix transformations) + (gnu home) + (gnu packages) + ,@(map first modules+configurations)) + + ,@transformations + + ,(home-environment-template + #:packages packages + #:services (map second modules+configurations)))) + `(begin + (use-modules (guix transformations) + (gnu packages)) + + ,@transformations + + (packages->manifest + (list ,@packages))))))) + +(define* (home-environment-template #:key (packages #f) (specs #f) services) + "Return an S-exp containing a <home-environment> declaration +containing PACKAGES, or SPECS (package specifications), and SERVICES." + `(home-environment + (packages + ,@(if packages + `((list ,@packages)) + `((map specification->package + (list ,@specs))))) + (services (list ,@services)))) + +(define* (import-manifest + manifest + #:optional (port (current-output-port))) + "Write to PORT a <home-environment> corresponding to MANIFEST." + (define (version-spec entry) + (let ((name (manifest-entry-name entry))) + (match (map package-version (find-packages-by-name name)) + ((_) + ;; A single version of NAME is available, so do not specify the + ;; version number, even if the available version doesn't match ENTRY. + "") + (versions + ;; If ENTRY uses the latest version, don't specify any version. + ;; Otherwise return the shortest unique version prefix. Note that + ;; this is based on the currently available packages, which could + ;; differ from the packages available in the revision that was used + ;; to build MANIFEST. + (let ((current (manifest-entry-version entry))) + (if (every (cut version>? current <>) + (delete current versions)) + "" + (version-unique-prefix (manifest-entry-version entry) + versions))))))) + + (match (manifest->code manifest + #:entry-package-version version-spec + #:home-environment? #t) + (('begin exp ...) + (format port (G_ "\ +;; This \"home-environment\" file can be passed to 'guix home reconfigure' +;; to reproduce the content of your profile. This is \"symbolic\": it only +;; specifies package names. To reproduce the exact same profile, you also +;; need to capture the channels being used, as returned by \"guix describe\". +;; See the \"Replicating Guix\" section in the manual.\n")) + (for-each (lambda (exp) + (newline port) + (pretty-print exp port)) + exp)))) diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm index 98554ef79b..40fa6759ae 100644 --- a/guix/scripts/import.scm +++ b/guix/scripts/import.scm @@ -1,8 +1,10 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2014, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2012, 2013, 2014, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2014 David Thompson <davet@gnu.org> ;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> ;;; Copyright © 2019 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -76,8 +78,9 @@ rather than \\n." ;;; Entry point. ;;; -(define importers '("gnu" "nix" "pypi" "cpan" "hackage" "stackage" "elpa" "gem" - "go" "cran" "crate" "texlive" "json" "opam")) +(define importers '("gnu" "pypi" "cpan" "hackage" "stackage" "egg" "elpa" + "gem" "go" "cran" "crate" "texlive" "json" "opam" + "minetest")) (define (resolve-importer name) (let ((module (resolve-interface @@ -116,7 +119,8 @@ Run IMPORTER with ARGS.\n")) (if (member importer importers) (let ((print (lambda (expr) (pretty-print expr (newline-rewriting-port - (current-output-port)))))) + (current-output-port)) + #:max-expr-width 80)))) (match (apply (resolve-importer importer) args) ((and expr (or ('package _ ...) ('let _ ...) @@ -129,4 +133,9 @@ Run IMPORTER with ARGS.\n")) expressions)) (x (leave (G_ "'~a' import failed~%") importer)))) - (leave (G_ "~a: invalid importer~%") importer))))) + (let ((hint (string-closest importer importers #:threshold 3))) + (report-error (G_ "~a: invalid importer~%") importer) + (when hint + (display-hint + (format #f (G_ "Did you mean @code{~a}?~%") hint))) + (exit 1)))))) diff --git a/guix/scripts/import/cpan.scm b/guix/scripts/import/cpan.scm index 77ffe1f38e..bdf5a1e423 100644 --- a/guix/scripts/import/cpan.scm +++ b/guix/scripts/import/cpan.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 Eric Bavier <bavier@member.fsf.org> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -66,12 +67,8 @@ Import and convert the CPAN package for PACKAGE-NAME.\n")) (define (guix-import-cpan . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/import/cran.scm b/guix/scripts/import/cran.scm index aa3ef324e0..3e4b038cc4 100644 --- a/guix/scripts/import/cran.scm +++ b/guix/scripts/import/cran.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2014 Eric Bavier <bavier@member.fsf.org> ;;; Copyright © 2015, 2017, 2019, 2020 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -86,12 +87,8 @@ Import and convert the CRAN package for PACKAGE-NAME.\n")) (define (guix-import-cran . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/import/crate.scm b/guix/scripts/import/crate.scm index 3a96defb86..97152904ac 100644 --- a/guix/scripts/import/crate.scm +++ b/guix/scripts/import/crate.scm @@ -3,6 +3,7 @@ ;;; Copyright © 2014 David Thompson <davet@gnu.org> ;;; Copyright © 2016 David Craven <david@craven.ch> ;;; Copyright © 2019, 2020 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -75,13 +76,8 @@ Import and convert the crates.io package for PACKAGE-NAME.\n")) (define (guix-import-crate . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) - + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/import/nix.scm b/guix/scripts/import/egg.scm index 45ca7e3fcf..829cdc2ca0 100644 --- a/guix/scripts/import/nix.scm +++ b/guix/scripts/import/egg.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2012, 2013, 2016 Ludovic Courtès <ludo@gnu.org> -;;; Copyright © 2014 David Thompson <davet@gnu.org> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -17,17 +17,18 @@ ;;; You should have received a copy of the GNU General Public License ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. -(define-module (guix scripts import nix) +(define-module (guix scripts import egg) #:use-module (guix ui) #:use-module (guix utils) #:use-module (guix scripts) - #:use-module (guix import snix) + #:use-module (guix import egg) #:use-module (guix scripts import) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) #:use-module (srfi srfi-37) #:use-module (ice-9 match) - #:export (guix-import-nix)) + #:use-module (ice-9 format) + #:export (guix-import-egg)) ;;; @@ -38,11 +39,13 @@ '()) (define (show-help) - (display (G_ "Usage: guix import nix NIXPKGS ATTRIBUTE -Import and convert the Nix expression ATTRIBUTE of NIXPKGS.\n")) + (display (G_ "Usage: guix import egg PACKAGE-NAME +Import and convert the egg package for PACKAGE-NAME.\n")) (display (G_ " -h, --help display this help and exit")) (display (G_ " + -r, --recursive import packages recursively")) + (display (G_ " -V, --version display version information and exit")) (newline) (show-bug-report-information)) @@ -55,7 +58,10 @@ Import and convert the Nix expression ATTRIBUTE of NIXPKGS.\n")) (exit 0))) (option '(#\V "version") #f #f (lambda args - (show-version-and-exit "guix import nix"))) + (show-version-and-exit "guix import egg"))) + (option '(#\r "recursive") #f #f + (lambda (opt name arg result) + (alist-cons 'recursive #t result))) %standard-import-options)) @@ -63,28 +69,36 @@ Import and convert the Nix expression ATTRIBUTE of NIXPKGS.\n")) ;;; Entry point. ;;; -(define (guix-import-nix . args) +(define (guix-import-egg . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) + (repo (and=> (assoc-ref opts 'repo) string->symbol)) (args (filter-map (match-lambda (('argument . value) value) (_ #f)) (reverse opts)))) (match args - ((nixpkgs attribute) - (let-values (((expr loc) - (nixpkgs->guix-package nixpkgs attribute))) - (format #t ";; converted from ~a:~a~%~%" - (location-file loc) (location-line loc)) - expr)) - (x - (leave (G_ "wrong number of arguments~%")))))) + ((package-name) + (if (assoc-ref opts 'recursive) + ;; Recursive import + (map (match-lambda + ((and ('package ('name name) . rest) pkg) + `(define-public ,(string->symbol name) + ,pkg)) + (_ #f)) + (egg-recursive-import package-name)) + ;; Single import + (let ((sexp (egg->guix-package package-name))) + (unless sexp + (leave (G_ "failed to download meta-data for package '~a'~%") + package-name)) + sexp))) + (() + (leave (G_ "too few arguments~%"))) + ((many ...) + (leave (G_ "too many arguments~%")))))) diff --git a/guix/scripts/import/elpa.scm b/guix/scripts/import/elpa.scm index d6b38e5c4b..052b0cc0e7 100644 --- a/guix/scripts/import/elpa.scm +++ b/guix/scripts/import/elpa.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2015 Federico Beffa <beffa@fbengineering.ch> ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -80,12 +81,8 @@ Import the latest package named PACKAGE-NAME from an ELPA repository.\n")) (define (guix-import-elpa . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/import/gem.scm b/guix/scripts/import/gem.scm index c64596b514..328d20b946 100644 --- a/guix/scripts/import/gem.scm +++ b/guix/scripts/import/gem.scm @@ -1,6 +1,8 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 David Thompson <davet@gnu.org> ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> +;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re> ;;; ;;; This file is part of GNU Guix. ;;; @@ -59,7 +61,7 @@ Import and convert the RubyGems package for PACKAGE-NAME.\n")) (exit 0))) (option '(#\V "version") #f #f (lambda args - (show-version-and-exit "guix import pypi"))) + (show-version-and-exit "guix import gem"))) (option '(#\r "recursive") #f #f (lambda (opt name arg result) (alist-cons 'recursive #t result))) @@ -73,12 +75,8 @@ Import and convert the RubyGems package for PACKAGE-NAME.\n")) (define (guix-import-gem . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/import/gnu.scm b/guix/scripts/import/gnu.scm index ae98370037..344e363abe 100644 --- a/guix/scripts/import/gnu.scm +++ b/guix/scripts/import/gnu.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014, 2016 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -81,12 +82,8 @@ Return a package declaration template for PACKAGE, a GNU package.\n")) (define (guix-import-gnu . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm index 04b07f80cc..f5cfea8683 100644 --- a/guix/scripts/import/go.scm +++ b/guix/scripts/import/go.scm @@ -1,6 +1,8 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com> ;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2021 Zheng Junjie <873216071@qq.com> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -68,9 +70,7 @@ that are not yet in Guix")) (alist-cons 'recursive #t result))) (option '(#\p "goproxy") #t #f (lambda (opt name arg result) - (alist-cons 'goproxy - (string->symbol arg) - (alist-delete 'goproxy result)))) + (alist-cons 'goproxy arg (alist-delete 'goproxy result)))) (option '("pin-versions") #f #f (lambda (opt name arg result) (alist-cons 'pin-versions? #t result))) @@ -84,12 +84,8 @@ that are not yet in Guix")) (define (guix-import-go . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda @@ -116,10 +112,10 @@ that are not yet in Guix")) (map package->definition* (apply go-module-recursive-import arguments)) ;; Single import. - (let ((sexp (apply go-module->guix-package arguments))) + (let ((sexp (apply go-module->guix-package* arguments))) (unless sexp - (leave (G_ "failed to download meta-data for module '~a'~%") - module-name)) + (leave (G_ "failed to download meta-data for module '~a'.~%") + name)) (package->definition* sexp)))))) (() (leave (G_ "too few arguments~%"))) diff --git a/guix/scripts/import/hackage.scm b/guix/scripts/import/hackage.scm index 906dca24b1..83128fb816 100644 --- a/guix/scripts/import/hackage.scm +++ b/guix/scripts/import/hackage.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 Federico Beffa <beffa@fbengineering.ch> ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -105,12 +106,8 @@ version.\n")) (define (guix-import-hackage . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (define (run-importer package-name opts error-fn) (let* ((arguments (list diff --git a/guix/scripts/import/json.scm b/guix/scripts/import/json.scm index d8d5c3a4af..a3b5e6d79c 100644 --- a/guix/scripts/import/json.scm +++ b/guix/scripts/import/json.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2014 Eric Bavier <bavier@member.fsf.org> ;;; Copyright © 2015, 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -74,12 +75,8 @@ Import and convert the JSON package definition in PACKAGE-FILE.\n")) (define (guix-import-json . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/import/minetest.scm b/guix/scripts/import/minetest.scm new file mode 100644 index 0000000000..5f204d90fc --- /dev/null +++ b/guix/scripts/import/minetest.scm @@ -0,0 +1,117 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2014 David Thompson <davet@gnu.org> +;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix scripts import minetest) + #:use-module (guix ui) + #:use-module (guix utils) + #:use-module (guix scripts) + #:use-module (guix import minetest) + #:use-module (guix import utils) + #:use-module (guix scripts import) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-37) + #:use-module (ice-9 match) + #:use-module (ice-9 format) + #:export (guix-import-minetest)) + + +;;; +;;; Command-line options. +;;; + +(define %default-options + `((sort . ,%default-sort-key))) + +(define (show-help) + (display (G_ "Usage: guix import minetest AUTHOR/NAME +Import and convert the Minetest mod NAME by AUTHOR from ContentDB.\n")) + (display (G_ " + -h, --help display this help and exit")) + (display (G_ " + -r, --recursive import packages recursively")) + (display (G_ " + -V, --version display version information and exit")) + (display (G_ " + --sort=KEY when choosing between multiple implementations, + choose the one with the highest value for KEY + (one of \"score\" (standard) or \"downloads\")")) + (newline) + (show-bug-report-information)) + +(define (verify-sort-order sort) + "Verify SORT can be used to sort mods by." + (unless (member sort '("score" "downloads" "reviews")) + (leave (G_ "~a: not a valid key to sort by~%") sort)) + sort) + +(define %options + ;; Specification of the command-line options. + (cons* (option '(#\h "help") #f #f + (lambda args + (show-help) + (exit 0))) + (option '(#\V "version") #f #f + (lambda args + (show-version-and-exit "guix import minetest"))) + (option '(#\r "recursive") #f #f + (lambda (opt name arg result) + (alist-cons 'recursive #t result))) + (option '("sort") #t #f + (lambda (opt name arg result) + (alist-cons 'sort (verify-sort-order arg) result))) + %standard-import-options)) + + +;;; +;;; Entry point. +;;; + +(define (guix-import-minetest . args) + (define (parse-options) + ;; Return the alist of option values. + (args-fold* args %options + (lambda (opt name arg result) + (leave (G_ "~A: unrecognized option~%") name)) + (lambda (arg result) + (alist-cons 'argument arg result)) + %default-options)) + + (let* ((opts (parse-options)) + (args (filter-map (match-lambda + (('argument . value) + value) + (_ #f)) + (reverse opts)))) + (match args + ((name) + (with-error-handling + (let* ((sort (assoc-ref opts 'sort)) + (author/name (elaborate-contentdb-name name #:sort sort))) + (if (assoc-ref opts 'recursive) + ;; Recursive import + (filter-map package->definition + (minetest-recursive-import author/name #:sort sort)) + ;; Single import + (minetest->guix-package author/name #:sort sort))))) + (() + (leave (G_ "too few arguments~%"))) + ((many ...) + (leave (G_ "too many arguments~%")))))) diff --git a/guix/scripts/import/opam.scm b/guix/scripts/import/opam.scm index da9392821c..834ac34cb0 100644 --- a/guix/scripts/import/opam.scm +++ b/guix/scripts/import/opam.scm @@ -1,5 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018 Julien Lepiller <julien@lepiller.eu> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> +;;; Copyright © 2021 Alice Brenon <alice.brenon@ens-lyon.fr> ;;; ;;; This file is part of GNU Guix. ;;; @@ -45,7 +47,8 @@ Import and convert the opam package for PACKAGE-NAME.\n")) (display (G_ " -r, --recursive import packages recursively")) (display (G_ " - --repo import packages from this opam repository")) + --repo import packages from this opam repository (name, URL or local path) + can be used more than once")) (display (G_ " -V, --version display version information and exit")) (newline) @@ -76,15 +79,13 @@ Import and convert the opam package for PACKAGE-NAME.\n")) (define (guix-import-opam . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) - (repo (and=> (assoc-ref opts 'repo) string->symbol)) + (repo (filter-map (match-lambda + (('repo . name) name) + (_ #f)) opts)) (args (filter-map (match-lambda (('argument . value) value) diff --git a/guix/scripts/import/pypi.scm b/guix/scripts/import/pypi.scm index 33167174e2..9170a0b359 100644 --- a/guix/scripts/import/pypi.scm +++ b/guix/scripts/import/pypi.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 David Thompson <davet@gnu.org> ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -72,12 +73,8 @@ Import and convert the PyPI package for PACKAGE-NAME.\n")) (define (guix-import-pypi . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/import/stackage.scm b/guix/scripts/import/stackage.scm index d77328dcbf..211ac73ada 100644 --- a/guix/scripts/import/stackage.scm +++ b/guix/scripts/import/stackage.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Federico Beffa <beffa@fbengineering.ch> ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -89,12 +90,8 @@ Import and convert the LTS Stackage package for PACKAGE-NAME.\n")) (define (guix-import-stackage . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (define (run-importer package-name opts error-fn) (let* ((arguments (list diff --git a/guix/scripts/import/texlive.scm b/guix/scripts/import/texlive.scm index 1cceee7051..6f0818e274 100644 --- a/guix/scripts/import/texlive.scm +++ b/guix/scripts/import/texlive.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> +;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev> ;;; ;;; This file is part of GNU Guix. ;;; @@ -73,12 +74,8 @@ Import and convert the Texlive package for PACKAGE-NAME.\n")) (define (guix-import-texlive . args) (define (parse-options) ;; Return the alist of option values. - (args-fold* args %options - (lambda (opt name arg result) - (leave (G_ "~A: unrecognized option~%") name)) - (lambda (arg result) - (alist-cons 'argument arg result)) - %default-options)) + (parse-command-line args %options (list %default-options) + #:build-options? #f)) (let* ((opts (parse-options)) (args (filter-map (match-lambda diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index b653138f2c..9e1f270dfb 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -6,6 +6,7 @@ ;;; Copyright © 2018 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2020 Tobias Geerinckx-Rice <me@tobias.gr> ;;; Copyright © 2020 Eric Bavier <bavier@posteo.net> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -65,6 +66,7 @@ %compressors lookup-compressor self-contained-tarball + debian-archive docker-image squashfs-image @@ -172,22 +174,40 @@ dependencies are registered." (computed-file "store-database" build #:options `(#:references-graphs ,(zip labels items)))) -(define* (self-contained-tarball name profile - #:key target - (profile-name "guix-profile") - deduplicate? - entry-point - (compressor (first %compressors)) - localstatedir? - (symlinks '()) - (archiver tar)) - "Return a self-contained tarball containing a store initialized with the -closure of PROFILE, a derivation. The tarball contains /gnu/store; if -LOCALSTATEDIR? is true, it also contains /var/guix, including /var/guix/db -with a properly initialized store database. +(define-syntax-rule (define-with-source (variable args ...) body body* ...) + "Bind VARIABLE to a procedure accepting ARGS defined as BODY, also setting +its source property." + (begin + (define (variable args ...) + body body* ...) + (eval-when (load eval) + (set-procedure-property! variable 'source + '(define (variable args ...) body body* ...))))) -SYMLINKS must be a list of (SOURCE -> TARGET) tuples denoting symlinks to be -added to the pack." +(define-with-source (manifest->friendly-name manifest) + "Return a friendly name computed from the entries in MANIFEST, a +<manifest> object." + (let loop ((names (map manifest-entry-name + (manifest-entries manifest)))) + (define str (string-join names "-")) + (if (< (string-length str) 40) + str + (match names + ((_) str) + ((names ... _) (loop names)))))) + + +;;; +;;; Tarball format. +;;; +(define* (self-contained-tarball/builder profile + #:key (profile-name "guix-profile") + (compressor (first %compressors)) + localstatedir? + (symlinks '()) + (archiver tar) + (extra-options '())) + "Return the G-Expression of the builder used for self-contained-tarball." (define database (and localstatedir? (file-append (store-database (list profile)) @@ -209,126 +229,114 @@ added to the pack." (and (not-config? module) (not (equal? '(guix store deduplication) module)))) - (define build - (with-imported-modules (source-module-closure - `((guix build utils) - (guix build union) - (gnu build install)) - #:select? import-module?) - #~(begin - (use-modules (guix build utils) - ((guix build union) #:select (relative-file-name)) - (gnu build install) - (srfi srfi-1) - (srfi srfi-26) - (ice-9 match)) - - (define %root "root") - - (define symlink->directives - ;; Return "populate directives" to make the given symlink and its - ;; parent directories. - (match-lambda - ((source '-> target) - (let ((target (string-append #$profile "/" target)) - (parent (dirname source))) - ;; Never add a 'directory' directive for "/" so as to - ;; preserve its ownnership when extracting the archive (see - ;; below), and also because this would lead to adding the - ;; same entries twice in the tarball. - `(,@(if (string=? parent "/") - '() - `((directory ,parent))) - (,source - -> ,(relative-file-name parent target))))))) - - (define directives - ;; Fully-qualified symlinks. - (append-map symlink->directives '#$symlinks)) + (with-imported-modules (source-module-closure + `((guix build pack) + (guix build store-copy) + (guix build utils) + (guix build union) + (gnu build install)) + #:select? import-module?) + #~(begin + (use-modules (guix build pack) + (guix build store-copy) + (guix build utils) + ((guix build union) #:select (relative-file-name)) + (gnu build install) + (srfi srfi-1) + (srfi srfi-26) + (ice-9 match)) - ;; The --sort option was added to GNU tar in version 1.28, released - ;; 2014-07-28. For testing, we use the bootstrap tar, which is - ;; older and doesn't support it. - (define tar-supports-sort? - (zero? (system* (string-append #+archiver "/bin/tar") - "cf" "/dev/null" "--files-from=/dev/null" - "--sort=name"))) + (define %root "root") - ;; Make sure non-ASCII file names are properly handled. - #+set-utf8-locale + (define symlink->directives + ;; Return "populate directives" to make the given symlink and its + ;; parent directories. + (match-lambda + ((source '-> target) + (let ((target (string-append #$profile "/" target)) + (parent (dirname source))) + ;; Never add a 'directory' directive for "/" so as to + ;; preserve its ownership when extracting the archive (see + ;; below), and also because this would lead to adding the + ;; same entries twice in the tarball. + `(,@(if (string=? parent "/") + '() + `((directory ,parent))) + (,source + -> ,(relative-file-name parent target))))))) - ;; Add 'tar' to the search path. - (setenv "PATH" #+(file-append archiver "/bin")) + (define directives + ;; Fully-qualified symlinks. + (append-map symlink->directives '#$symlinks)) - ;; Note: there is not much to gain here with deduplication and there - ;; is the overhead of the '.links' directory, so turn it off. - ;; Furthermore GNU tar < 1.30 sometimes fails to extract tarballs - ;; with hard links: - ;; <http://lists.gnu.org/archive/html/bug-tar/2017-11/msg00009.html>. - (populate-single-profile-directory %root - #:profile #$profile - #:profile-name #$profile-name - #:closure "profile" - #:database #+database) + ;; Make sure non-ASCII file names are properly handled. + #+set-utf8-locale - ;; Create SYMLINKS. - (for-each (cut evaluate-populate-directive <> %root) - directives) + (define tar #+(file-append archiver "/bin/tar")) - ;; Create the tarball. Use GNU format so there's no file name - ;; length limitation. - (with-directory-excursion %root - (exit - (zero? (apply system* "tar" - #+@(if (compressor-command compressor) - #~("-I" - (string-join - '#+(compressor-command compressor))) - #~()) - "--format=gnu" + ;; Note: there is not much to gain here with deduplication and there + ;; is the overhead of the '.links' directory, so turn it off. + ;; Furthermore GNU tar < 1.30 sometimes fails to extract tarballs + ;; with hard links: + ;; <http://lists.gnu.org/archive/html/bug-tar/2017-11/msg00009.html>. + (populate-store (list "profile") %root #:deduplicate? #f) - ;; Avoid non-determinism in the archive. Use - ;; mtime = 1, not zero, because that is what the - ;; daemon does for files in the store (see the - ;; 'mtimeStore' constant in local-store.cc.) - (if tar-supports-sort? "--sort=name" "--mtime=@1") - "--mtime=@1" ;for files in /var/guix - "--owner=root:0" - "--group=root:0" + (when #+localstatedir? + (install-database-and-gc-roots %root #+database #$profile + #:profile-name #$profile-name)) - "--check-links" - "-cvf" #$output - ;; Avoid adding / and /var to the tarball, so - ;; that the ownership and permissions of those - ;; directories will not be overwritten when - ;; extracting the archive. Do not include /root - ;; because the root account might have a - ;; different home directory. - #$@(if localstatedir? - '("./var/guix") - '()) + ;; Create SYMLINKS. + (for-each (cut evaluate-populate-directive <> %root) + directives) - (string-append "." (%store-directory)) + ;; Create the tarball. + (with-directory-excursion %root + ;; GNU Tar recurses directories by default. Simply add the whole + ;; current directory, which contains all the generated files so far. + ;; This avoids creating duplicate files in the archives that would + ;; be stored as hard links by GNU Tar. + (apply invoke tar "-cvf" #$output "." + (tar-base-options + #:tar tar + #:compressor '#+(and=> compressor compressor-command))))))) - (delete-duplicates - (filter-map (match-lambda - (('directory directory) - (string-append "." directory)) - ((source '-> _) - (string-append "." source)) - (_ #f)) - directives))))))))) +(define* (self-contained-tarball name profile + #:key target + (profile-name "guix-profile") + deduplicate? + entry-point + (compressor (first %compressors)) + localstatedir? + (symlinks '()) + (archiver tar) + (extra-options '())) + "Return a self-contained tarball containing a store initialized with the +closure of PROFILE, a derivation. The tarball contains /gnu/store; if +LOCALSTATEDIR? is true, it also contains /var/guix, including /var/guix/db +with a properly initialized store database. +SYMLINKS must be a list of (SOURCE -> TARGET) tuples denoting symlinks to be +added to the pack." (when entry-point (warning (G_ "entry point not supported in the '~a' format~%") 'tarball)) - (gexp->derivation (string-append name ".tar" - (compressor-extension compressor)) - build - #:target target - #:references-graphs `(("profile" ,profile)))) + (gexp->derivation + (string-append name ".tar" + (compressor-extension compressor)) + (self-contained-tarball/builder profile + #:profile-name profile-name + #:compressor compressor + #:localstatedir? localstatedir? + #:symlinks symlinks + #:archiver archiver) + #:target target + #:references-graphs `(("profile" ,profile)))) + +;;; +;;; Singularity. +;;; (define (singularity-environment-file profile) "Return a shell script that defines the environment variables corresponding to the search paths of PROFILE." @@ -355,6 +363,10 @@ to the search paths of PROFILE." (computed-file "singularity-environment.sh" build)) + +;;; +;;; SquashFS image format. +;;; (define* (squashfs-image name profile #:key target (profile-name "guix-profile") @@ -362,7 +374,8 @@ to the search paths of PROFILE." entry-point localstatedir? (symlinks '()) - (archiver squashfs-tools)) + (archiver squashfs-tools) + (extra-options '())) "Return a squashfs image containing a store initialized with the closure of PROFILE, a derivation. The image contains a subset of /gnu/store, empty mount points for virtual file systems (like procfs), and optional symlinks. @@ -529,6 +542,10 @@ added to the pack." #:target target #:references-graphs `(("profile" ,profile)))) + +;;; +;;; Docker image format. +;;; (define* (docker-image name profile #:key target (profile-name "guix-profile") @@ -536,7 +553,8 @@ added to the pack." entry-point localstatedir? (symlinks '()) - (archiver tar)) + (archiver tar) + (extra-options '())) "Return a derivation to construct a Docker image of PROFILE. The image is a tarball conforming to the Docker Image Specification, compressed with COMPRESSOR. It can be passed to 'docker load'. If TARGET is true, it @@ -547,7 +565,7 @@ the image." (file-append (store-database (list profile)) "/db/db.sqlite"))) - (define defmod 'define-module) ;trick Geiser + (define defmod 'define-module) ;trick Geiser (define build ;; Guile-JSON and Guile-Gcrypt are required by (guix docker). @@ -565,6 +583,8 @@ the image." (srfi srfi-1) (srfi srfi-19) (ice-9 match)) + #$(procedure-source manifest->friendly-name) + (define environment (map (match-lambda ((spec . value) @@ -588,19 +608,6 @@ the image." `((directory "/tmp" ,(getuid) ,(getgid) #o1777) ,@(append-map symlink->directives '#$symlinks))) - (define tag - ;; Compute a meaningful "repository" name, which will show up in - ;; the output of "docker images". - (let ((manifest (profile-manifest #$profile))) - (let loop ((names (map manifest-entry-name - (manifest-entries manifest)))) - (define str (string-join names "-")) - (if (< (string-length str) 40) - str - (match names - ((_) str) - ((names ... _) (loop names))))))) ;drop one entry - (setenv "PATH" #+(file-append archiver "/bin")) (build-docker-image #$output @@ -608,9 +615,10 @@ the image." (call-with-input-file "profile" read-reference-graph)) #$profile - #:repository tag + #:repository (manifest->friendly-name + (profile-manifest #$profile)) #:database #+database - #:system (or #$target (utsname:machine (uname))) + #:system (or #$target %host-type) #:environment environment #:entry-point #$(and entry-point @@ -628,6 +636,192 @@ the image." ;;; +;;; Debian archive format. +;;; +;;; TODO: When relocatable option is selected, install to a unique prefix. +;;; This would enable installation of multiple deb packs with conflicting +;;; files at the same time. +(define* (debian-archive name profile + #:key target + (profile-name "guix-profile") + deduplicate? + entry-point + (compressor (first %compressors)) + localstatedir? + (symlinks '()) + (archiver tar) + (extra-options '())) + "Return a Debian archive (.deb) containing a store initialized with the +closure of PROFILE, a derivation. The archive contains /gnu/store; if +LOCALSTATEDIR? is true, it also contains /var/guix, including /var/guix/db +with a properly initialized store database. The supported compressors are +\"none\", \"gz\" or \"xz\". + +SYMLINKS must be a list of (SOURCE -> TARGET) tuples denoting symlinks to be +added to the pack. EXTRA-OPTIONS may contain the CONFIG-FILE, POSTINST-FILE +or TRIGGERS-FILE keyword arguments." + ;; For simplicity, limit the supported compressors to the superset of + ;; compressors able to compress both the control file (gz or xz) and the + ;; data tarball (gz, bz2 or xz). + (define %valid-compressors '("gzip" "xz" "none")) + + (let ((compressor-name (compressor-name compressor))) + (unless (member compressor-name %valid-compressors) + (leave (G_ "~a is not a valid Debian archive compressor. \ +Valid compressors are: ~a~%") compressor-name %valid-compressors))) + + (when entry-point + (warning (G_ "entry point not supported in the '~a' format~%") + 'deb)) + + (define data-tarball + (computed-file (string-append "data.tar" + (compressor-extension compressor)) + (self-contained-tarball/builder + profile + #:profile-name profile-name + #:compressor compressor + #:localstatedir? localstatedir? + #:symlinks symlinks + #:archiver archiver) + #:local-build? #f ;allow offloading + #:options (list #:references-graphs `(("profile" ,profile)) + #:target target))) + + (define build + (with-extensions (list guile-gcrypt) + (with-imported-modules `(((guix config) => ,(make-config.scm)) + ,@(source-module-closure + `((guix build pack) + (guix build utils) + (guix profiles)) + #:select? not-config?)) + #~(begin + (use-modules (guix build pack) + (guix build utils) + (guix profiles) + (ice-9 match) + ((oop goops) #:select (get-keyword)) + (srfi srfi-1)) + + (define machine-type + ;; Extract the machine type from the specified target, else from the + ;; current system. + (and=> (or #$target %host-type) + (lambda (triplet) + (first (string-split triplet #\-))))) + + (define (gnu-machine-type->debian-machine-type type) + "Translate machine TYPE from the GNU to Debian terminology." + ;; Debian has its own jargon, different from the one used in GNU, for + ;; machine types (see data/cputable in the sources of dpkg). + (match type + ("i486" "i386") + ("i586" "i386") + ("i686" "i386") + ("x86_64" "amd64") + ("aarch64" "arm64") + ("mipsisa32r6" "mipsr6") + ("mipsisa32r6el" "mipsr6el") + ("mipsisa64r6" "mips64r6") + ("mipsisa64r6el" "mips64r6el") + ("powerpcle" "powerpcel") + ("powerpc64" "ppc64") + ("powerpc64le" "ppc64el") + (machine machine))) + + (define architecture + (gnu-machine-type->debian-machine-type machine-type)) + + #$(procedure-source manifest->friendly-name) + + (define manifest (profile-manifest #$profile)) + + (define single-entry ;manifest entry + (match (manifest-entries manifest) + ((entry) + entry) + (() #f))) + + (define package-name (or (and=> single-entry manifest-entry-name) + (manifest->friendly-name manifest))) + + (define package-version + (or (and=> single-entry manifest-entry-version) + "0.0.0")) + + (define debian-format-version "2.0") + + ;; Generate the debian-binary file. + (call-with-output-file "debian-binary" + (lambda (port) + (format port "~a~%" debian-format-version))) + + (define data-tarball-file-name (strip-store-file-name + #+data-tarball)) + + (copy-file #+data-tarball data-tarball-file-name) + + ;; Generate the control archive. + (define control-file + (get-keyword #:control-file '#$extra-options)) + + (define postinst-file + (get-keyword #:postinst-file '#$extra-options)) + + (define triggers-file + (get-keyword #:triggers-file '#$extra-options)) + + (define control-tarball-file-name + (string-append "control.tar" + #$(compressor-extension compressor))) + + ;; Write the compressed control tarball. Only the control file is + ;; mandatory (see: 'man deb' and 'man deb-control'). + (if control-file + (copy-file control-file "control") + (call-with-output-file "control" + (lambda (port) + (format port "\ +Package: ~a +Version: ~a +Description: Debian archive generated by GNU Guix. +Maintainer: GNU Guix +Architecture: ~a +Priority: optional +Section: misc +~%" package-name package-version architecture)))) + + (when postinst-file + (copy-file postinst-file "postinst") + (chmod "postinst" #o755)) + + (when triggers-file + (copy-file triggers-file "triggers")) + + (define tar (string-append #+archiver "/bin/tar")) + + (apply invoke tar + `(,@(tar-base-options + #:tar tar + #:compressor '#+(and=> compressor compressor-command)) + "-cvf" ,control-tarball-file-name + "control" + ,@(if postinst-file '("postinst") '()) + ,@(if triggers-file '("triggers") '()))) + + ;; Create the .deb archive using GNU ar. + (invoke (string-append #+binutils "/bin/ar") "-rv" #$output + "debian-binary" + control-tarball-file-name data-tarball-file-name))))) + + (gexp->derivation (string-append name ".deb") + build + #:target target + #:references-graphs `(("profile" ,profile)))) + + +;;; ;;; Compiling C programs. ;;; @@ -958,7 +1152,8 @@ last resort for relocation." ;; Supported pack formats. `((tarball . ,self-contained-tarball) (squashfs . ,squashfs-image) - (docker . ,docker-image))) + (docker . ,docker-image) + (deb . ,debian-archive))) (define (show-formats) ;; Print the supported pack formats. @@ -970,8 +1165,38 @@ last resort for relocation." squashfs Squashfs image suitable for Singularity")) (display (G_ " docker Tarball ready for 'docker load'")) + (display (G_ " + deb Debian archive installable via dpkg/apt")) (newline)) +(define %deb-format-options + (let ((required-option (lambda (symbol) + (option (list (symbol->string symbol)) #t #f + (lambda (opt name arg result . rest) + (apply values + (alist-cons symbol arg result) + rest)))))) + (list (required-option 'control-file) + (required-option 'postinst-file) + (required-option 'triggers-file)))) + +(define (show-deb-format-options) + (display (G_ " + --help-deb-format list options specific to the deb format"))) + +(define (show-deb-format-options/detailed) + (display (G_ " + --control-file=FILE + Embed the provided control FILE")) + (display (G_ " + --postinst-file=FILE + Embed the provided postinst script")) + (display (G_ " + --triggers-file=FILE + Embed the provided triggers FILE")) + (newline) + (exit 0)) + (define %options ;; Specifications of the command-line options. (cons* (option '(#\h "help") #f #f @@ -1065,7 +1290,12 @@ last resort for relocation." (lambda (opt name arg result) (alist-cons 'bootstrap? #t result))) - (append %transformation-options + (option '("help-deb-format") #f #f + (lambda args + (show-deb-format-options/detailed))) + + (append %deb-format-options + %transformation-options %standard-build-options))) (define (show-help) @@ -1075,6 +1305,8 @@ Create a bundle of PACKAGE.\n")) (newline) (show-transformation-options-help) (newline) + (show-deb-format-options) + (newline) (display (G_ " -f, --format=FORMAT build a pack in the given FORMAT")) (display (G_ " @@ -1184,6 +1416,18 @@ Create a bundle of PACKAGE.\n")) (else (packages->manifest packages)))))) + (define (process-file-arg opts name) + ;; Validate that the file exists and return it as a <local-file> object, + ;; else #f. + (let ((value (assoc-ref opts name))) + (match value + ((and (? string?) (not (? file-exists?))) + (leave (G_ "file provided with option ~a does not exist: ~a~%") + (string-append "--" (symbol->string name)) value)) + ((? string?) + (local-file value)) + (#f #f)))) + (with-error-handling (with-store store (with-status-verbosity (assoc-ref opts 'verbosity) @@ -1216,8 +1460,15 @@ Create a bundle of PACKAGE.\n")) manifest) manifest))) (pack-format (assoc-ref opts 'format)) - (name (string-append (symbol->string pack-format) - "-pack")) + (extra-options (match pack-format + ('deb + (list #:control-file + (process-file-arg opts 'control-file) + #:postinst-file + (process-file-arg opts 'postinst-file) + #:triggers-file + (process-file-arg opts 'triggers-file))) + (_ '()))) (target (assoc-ref opts 'target)) (bootstrap? (assoc-ref opts 'bootstrap?)) (compressor (if bootstrap? @@ -1251,7 +1502,10 @@ Create a bundle of PACKAGE.\n")) (hooks (if bootstrap? '() %default-profile-hooks)) - (locales? (not bootstrap?))))) + (locales? (not bootstrap?)))) + (name (string-append (manifest->friendly-name manifest) + "-" (symbol->string pack-format) + "-pack"))) (define (lookup-package package) (manifest-lookup manifest (manifest-pattern (name package)))) @@ -1279,7 +1533,9 @@ to your package list."))) #:profile-name profile-name #:archiver - archiver))) + archiver + #:extra-options + extra-options))) (mbegin %store-monad (mwhen derivation? (return (format #t "~a~%" diff --git a/guix/scripts/package.scm b/guix/scripts/package.scm index e3d40d5142..a34ecdcb54 100644 --- a/guix/scripts/package.scm +++ b/guix/scripts/package.scm @@ -9,6 +9,7 @@ ;;; Copyright © 2019 Tobias Geerinckx-Rice <me@tobias.gr> ;;; Copyright © 2020 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2020 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2018 Steve Sprang <scs@stevesprang.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -831,15 +832,14 @@ processed, #f otherwise." (map profile-manifest profiles))) (installed (manifest-entries manifest))) (leave-on-EPIPE - (for-each (match-lambda - (($ <manifest-entry> name version output path _) - (when (or (not regexp) - (regexp-exec regexp name)) - (format #t "~a\t~a\t~a\t~a~%" - name (or version "?") output path)))) - - ;; Show most recently installed packages last. - (reverse installed)))) + (let ((rows (filter-map + (match-lambda + (($ <manifest-entry> name version output path _) + (and (regexp-exec regexp name) + (list name (or version "?") output path)))) + installed))) + ;; Show most recently installed packages last. + (pretty-print-table (reverse rows))))) #t) (('list-available regexp) @@ -862,16 +862,15 @@ processed, #f otherwise." result)) '()))) (leave-on-EPIPE - (for-each (match-lambda - ((name version outputs location) - (format #t "~a\t~a\t~a\t~a~%" - name version - (string-join outputs ",") - (location->string location)))) - (sort available - (match-lambda* - (((name1 . _) (name2 . _)) - (string<? name1 name2)))))) + (let ((rows (map (match-lambda + ((name version outputs location) + (list name version (string-join outputs ",") + (location->string location)))) + (sort available + (match-lambda* + (((name1 . _) (name2 . _)) + (string<? name1 name2))))))) + (pretty-print-table rows))) #t)) (('list-profiles _) @@ -1044,6 +1043,17 @@ processed, #f otherwise." (warn-about-old-distro) + (when (and (null? files) (manifest-transaction-null? trans) + (not (any (match-lambda + ((key . _) (assoc-ref %actions key))) + opts))) + ;; We can reach this point because the user did not specify any action + ;; (as in "guix package"), did not specify any package (as in "guix + ;; install"), or because there's nothing to upgrade (as when running + ;; "guix upgrade" on an up-to-date profile). We cannot distinguish + ;; among these here; all we can say is that there's nothing to do. + (warning (G_ "nothing to do~%"))) + (unless (manifest-transaction-null? trans) ;; When '--manifest' is used, display information about TRANS as if we ;; were starting from an empty profile. diff --git a/guix/scripts/perform-download.scm b/guix/scripts/perform-download.scm index 8d409092ba..6889bcef79 100644 --- a/guix/scripts/perform-download.scm +++ b/guix/scripts/perform-download.scm @@ -54,7 +54,8 @@ actual output is different from that when we're doing a 'bmCheck' or (output* "out") (executable "executable") (mirrors "mirrors") - (content-addressed-mirrors "content-addressed-mirrors")) + (content-addressed-mirrors "content-addressed-mirrors") + (disarchive-mirrors "disarchive-mirrors")) (unless url (leave (G_ "~a: missing URL~%") (derivation-file-name drv))) @@ -79,6 +80,10 @@ actual output is different from that when we're doing a 'bmCheck' or (lambda (port) (eval (read port) %user-module))) '()) + #:disarchive-mirrors + (if disarchive-mirrors + (call-with-input-file disarchive-mirrors read) + '()) #:hashes `((,algo . ,hash)) ;; Since DRV's output hash is known, X.509 certificate diff --git a/guix/scripts/publish.scm b/guix/scripts/publish.scm index 39bb224cad..25846b7dc2 100644 --- a/guix/scripts/publish.scm +++ b/guix/scripts/publish.scm @@ -1,9 +1,10 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2015 David Thompson <davet@gnu.org> ;;; Copyright © 2020 by Amar M. Singh <nly@disroot.org> -;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2021 Mathieu Othacehe <othacehe@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,6 +26,7 @@ #:use-module (ice-9 binary-ports) #:use-module (ice-9 format) #:use-module (ice-9 match) + #:use-module (ice-9 poll) #:use-module (ice-9 regex) #:use-module (ice-9 rdelim) #:use-module (ice-9 threads) @@ -33,6 +35,7 @@ #:use-module (srfi srfi-2) #:use-module (srfi srfi-9) #:use-module (srfi srfi-9 gnu) + #:use-module (srfi srfi-11) #:use-module (srfi srfi-19) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) @@ -102,6 +105,8 @@ Publish ~a over HTTP.\n") %store-directory) (display (G_ " --ttl=TTL announce narinfos can be cached for TTL seconds")) (display (G_ " + --negative-ttl=TTL announce missing narinfos can be cached for TTL seconds")) + (display (G_ " --nar-path=PATH use PATH as the prefix for nar URLs")) (display (G_ " --public-key=FILE use FILE as the public key for signatures")) @@ -224,6 +229,13 @@ usage." (leave (G_ "~a: invalid duration~%") arg)) (alist-cons 'narinfo-ttl (time-second duration) result)))) + (option '("negative-ttl") #t #f + (lambda (opt name arg result) + (let ((duration (string->duration arg))) + (unless duration + (leave (G_ "~a: invalid duration~%") arg)) + (alist-cons 'narinfo-negative-ttl (time-second duration) + result)))) (option '("nar-path") #t #f (lambda (opt name arg result) (alist-cons 'nar-path arg result))) @@ -309,7 +321,7 @@ with COMPRESSION, starting at NAR-PATH." (format #f "URL: ~a~%Compression: ~a~%~@[FileSize: ~a~%~]" url (compression-type compression) file-size))) -(define* (narinfo-string store store-path key +(define* (narinfo-string store store-path #:key (compressions (list %no-compression)) (nar-path "nar") (file-sizes '())) "Generate a narinfo key/value string for STORE-PATH; an exception is raised @@ -347,23 +359,13 @@ References: ~a~%" compression))) compressions) hash size references)) - ;; Do not render a "Deriver" or "System" line if we are rendering - ;; info for a derivation. + ;; Do not render a "Deriver" line if we are rendering info for a + ;; derivation. Also do not render a "System" line that would be + ;; expensive to compute and is currently unused. (info (if (not deriver) base-info - (catch 'system-error - (lambda () - (let ((drv (read-derivation-from-file deriver))) - (format #f "~aSystem: ~a~%Deriver: ~a~%" - base-info (derivation-system drv) - (basename deriver)))) - (lambda args - ;; DERIVER might be missing, but that's fine: - ;; it's only used for <substitutable> where it's - ;; optional. 'System' is currently unused. - (if (= ENOENT (system-error-errno args)) - base-info - (apply throw args)))))) + (format #f "~aDeriver: ~a~%" + base-info (basename deriver)))) (signature (base64-encode-string (canonical-sexp->string (signed-string info))))) (format #f "~aSignature: 1;~a;~a~%" info (gethostname) signature))) @@ -390,20 +392,20 @@ References: ~a~%" (define* (render-narinfo store request hash #:key ttl (compressions (list %no-compression)) - (nar-path "nar")) + (nar-path "nar") negative-ttl) "Render metadata for the store path corresponding to HASH. If TTL is true, advertise it as the maximum validity period (in seconds) via the 'Cache-Control' header. This allows 'guix substitute' to cache it for an appropriate duration. NAR-PATH specifies the prefix for nar URLs." (let ((store-path (hash-part->path store hash))) (if (string-null? store-path) - (not-found request #:phrase "") + (not-found request #:phrase "" #:ttl negative-ttl) (values `((content-type . (application/x-nix-narinfo)) ,@(if ttl `((cache-control (max-age . ,ttl))) '())) (cut display - (narinfo-string store store-path (%private-key) + (narinfo-string store store-path #:nar-path nar-path #:compressions compressions) <>))))) @@ -512,7 +514,7 @@ interpreted as the basename of a store item." (define* (render-narinfo/cached store request hash #:key ttl (compressions (list %no-compression)) - (nar-path "nar") + (nar-path "nar") negative-ttl cache pool) "Respond to the narinfo request for REQUEST. If the narinfo is available in CACHE, then send it; otherwise, return 404 and \"bake\" that nar and narinfo @@ -536,7 +538,7 @@ requested using POOL." #:compression (first compressions))))) (cond ((string-null? item) - (not-found request)) + (not-found request #:ttl negative-ttl)) ((file-exists? cached) ;; Narinfo is in cache, send it. (values `((content-type . (application/x-nix-narinfo)) @@ -555,7 +557,6 @@ requested using POOL." (single-baker item ;; Check whether CACHED has been produced in the meantime. (unless (file-exists? cached) - ;; (format #t "baking ~s~%" item) (bake-narinfo+nar cache item #:ttl ttl #:compressions compressions @@ -584,7 +585,7 @@ requested using POOL." #:phrase "We're baking it" #:ttl 300))) ;should be available within 5m (else - (not-found request #:phrase ""))))) + (not-found request #:phrase "" #:ttl negative-ttl))))) (define (compress-nar cache item compression) "Save in directory CACHE the nar for ITEM compressed with COMPRESSION." @@ -643,7 +644,6 @@ requested using POOL." (with-store store (let ((sizes (filter-map compressed-nar-size compression))) (display (narinfo-string store item - (%private-key) #:nar-path nar-path #:compressions compressions #:file-sizes sizes) @@ -860,60 +860,115 @@ example: \"/foo/bar\" yields '(\"foo\" \"bar\")." exp ...) (const #f))) -(define (nar-response-port response compression) - "Return a port on which to write the body of RESPONSE, the response of a -/nar request, according to COMPRESSION." +(define (nar-compressed-port port compression) + "Return a port on which to write the body of the response of a /nar request, +according to COMPRESSION." (match compression (($ <compression> 'gzip level) ;; Note: We cannot used chunked encoding here because ;; 'make-gzip-output-port' wants a file port. - (make-gzip-output-port (response-port response) + (make-gzip-output-port port #:level level #:buffer-size %default-buffer-size)) (($ <compression> 'lzip level) - (make-lzip-output-port (response-port response) + (make-lzip-output-port port #:level level)) (($ <compression> 'zstd level) - (make-zstd-output-port (response-port response) + (make-zstd-output-port port #:level level)) (($ <compression> 'none) - (response-port response)) + port) (#f - (response-port response)))) + port))) (define (http-write server client response body) "Write RESPONSE and BODY to CLIENT, possibly in a separate thread to avoid blocking." + ;; XXX: The default Guile web server implementation supports the keep-alive + ;; mechanism. However, as we run our own modified version of the http-write + ;; procedure, we need to access a few server implementation details to keep + ;; it functional. + (define *error-events* + (logior POLLHUP POLLERR)) + + (define *read-events* + POLLIN) + + (define *events* + (logior *error-events* *read-events*)) + + ;; Access the server poll set variable. + (define http-poll-set + (@@ (web server http) http-poll-set)) + + ;; Copied from (web server http). + (define (keep-alive? response) + (let ((v (response-version response))) + (and (or (< (response-code response) 400) + (= (response-code response) 404)) + (case (car v) + ((1) + (case (cdr v) + ((1) (not (memq 'close (response-connection response)))) + ((0) (memq 'keep-alive (response-connection response))))) + (else #f))))) + + (define (keep-alive port) + "Add the given PORT the server poll set." + (force-output port) + (poll-set-add! (http-poll-set server) port *events*)) + + (define compression + (assoc-ref (response-headers response) 'x-nar-compression)) + (match (response-content-type response) (('application/x-nix-archive . _) - ;; Sending the the whole archive can take time so do it in a separate - ;; thread so that the main thread can keep working in the meantime. - (call-with-new-thread - (lambda () - (set-thread-name "publish nar") - (let* ((compression (assoc-ref (response-headers response) - 'x-nar-compression)) - (response (write-response (sans-content-length response) - client)) - (port (begin - (force-output client) - (configure-socket client) - (nar-response-port response compression)))) - ;; XXX: Given our ugly workaround for <http://bugs.gnu.org/21093> in - ;; 'render-nar', BODY here is just the file name of the store item. - ;; We call 'write-file' from here because we know that's the only - ;; way to avoid building the whole nar in memory, which could - ;; quickly become a real problem. As a bonus, we even do - ;; sendfile(2) directly from the store files to the socket. - (swallow-zlib-error - (swallow-EPIPE - (write-file (utf8->string body) port))) - (swallow-zlib-error - (close-port port)) - (values))))) + ;; When compressing the NAR on the go, we cannot announce its size + ;; beforehand to the client. Hence, the keep-alive mechanism cannot work + ;; here. + (let ((keep-alive? (and (eq? (compression-type compression) 'none) + (keep-alive? response)))) + ;; Add the client to the server poll set, so that we can receive + ;; further requests without closing the connection. + (when keep-alive? + (keep-alive client)) + ;; Sending the the whole archive can take time so do it in a separate + ;; thread so that the main thread can keep working in the meantime. + (call-with-new-thread + (lambda () + (set-thread-name "publish nar") + (let* ((response (write-response (sans-content-length response) + client)) + (port (begin + (force-output client) + (configure-socket client) + ;; Duplicate the response port, so that it is + ;; not automatically closed when closing the + ;; returned port. This is needed for the + ;; keep-alive mechanism. + (nar-compressed-port + (duplicate-port + (response-port response) "w+0b") + compression)))) + ;; XXX: Given our ugly workaround for <http://bugs.gnu.org/21093> + ;; in 'render-nar', BODY here is just the file name of the store + ;; item. We call 'write-file' from here because we know that's + ;; the only way to avoid building the whole nar in memory, which + ;; could quickly become a real problem. As a bonus, we even do + ;; sendfile(2) directly from the store files to the socket. + (swallow-zlib-error + (swallow-EPIPE + (write-file (utf8->string body) port))) + (swallow-zlib-error + (close-port port) + (unless keep-alive? + (close-port client))) + (values)))))) (_ (match (assoc-ref (response-headers response) 'x-raw-file) ((? string? file) + (when (keep-alive? response) + (keep-alive client)) ;; Send a raw file in a separate thread. (call-with-new-thread (lambda () @@ -923,19 +978,20 @@ blocking." (call-with-input-file file (lambda (input) (let* ((size (stat:size (stat input))) - (response (write-response (with-content-length response - size) - client)) + (response (write-response + (with-content-length response size) + client)) (output (response-port response))) (configure-socket client) (if (file-port? output) (sendfile output input size) (dump-port input output)) - (close-port output) + (unless (keep-alive? response) + (close-port output)) (values))))) (lambda args - ;; If the file was GC'd behind our back, that's fine. Likewise if - ;; the client closes the connection. + ;; If the file was GC'd behind our back, that's fine. Likewise + ;; if the client closes the connection. (unless (memv (system-error-errno args) (list ENOENT EPIPE ECONNRESET)) (apply throw args)) @@ -971,10 +1027,22 @@ methods, return the applicable compression." compressions) (default-compression requested-type))) +(define (preserve-connection-headers request response) + "Add REQUEST's 'connection' header, if any, to HEADERS, a list of response +headers." + (if (pair? response) + (let ((connection + (assq 'connection (request-headers request)))) + (append response + (if connection + (list connection) + '()))) + response)) + (define* (make-request-handler store #:key cache pool - narinfo-ttl + narinfo-ttl narinfo-negative-ttl (nar-path "nar") (compressions (list %no-compression))) (define compression-type? @@ -984,7 +1052,7 @@ methods, return the applicable compression." (let ((expected (split-and-decode-uri-path nar-path))) (cut equal? expected <>))) - (lambda (request body) + (define (handle request body) (format #t "~a ~a~%" (request-method request) (uri-path (request-uri request))) @@ -1006,10 +1074,12 @@ methods, return the applicable compression." #:cache cache #:pool pool #:ttl narinfo-ttl + #:negative-ttl narinfo-negative-ttl #:nar-path nar-path #:compressions compressions) (render-narinfo store request hash #:ttl narinfo-ttl + #:negative-ttl narinfo-negative-ttl #:nar-path nar-path #:compressions compressions))) ;; /nar/file/NAME/sha256/HASH @@ -1054,7 +1124,15 @@ methods, return the applicable compression." (not-found request))) (x (not-found request))) - (not-found request)))) + (not-found request))) + + ;; Preserve the request's 'connection' header in the response, so that the + ;; server can close the connection if this is requested by the client. + (lambda (request body) + (let-values (((response response-body) + (handle request body))) + (values (preserve-connection-headers request response) + response-body)))) (define (service-name) "Return the Avahi service name of the server." @@ -1068,7 +1146,7 @@ methods, return the applicable compression." #:key advertise? port (compressions (list %no-compression)) - (nar-path "nar") narinfo-ttl + (nar-path "nar") narinfo-ttl narinfo-negative-ttl cache pool) (when advertise? (let ((name (service-name))) @@ -1084,6 +1162,7 @@ methods, return the applicable compression." #:pool pool #:nar-path nar-path #:narinfo-ttl narinfo-ttl + #:narinfo-negative-ttl narinfo-negative-ttl #:compressions compressions) concurrent-http-server `(#:socket ,socket))) @@ -1127,6 +1206,7 @@ methods, return the applicable compression." (user (assoc-ref opts 'user)) (port (assoc-ref opts 'port)) (ttl (assoc-ref opts 'narinfo-ttl)) + (negative-ttl (assoc-ref opts 'narinfo-negative-ttl)) (compressions (match (filter-map (match-lambda (('compression . compression) compression) @@ -1192,6 +1272,7 @@ consider using the '--user' option!~%"))) "publish worker")) #:nar-path nar-path #:compressions compressions + #:narinfo-negative-ttl negative-ttl #:narinfo-ttl ttl)))))) ;;; Local Variables: diff --git a/guix/scripts/pull.scm b/guix/scripts/pull.scm index 07613240a8..fb8ce50fa7 100644 --- a/guix/scripts/pull.scm +++ b/guix/scripts/pull.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2015, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013, 2014, 2015, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Marius Bakke <mbakke@fastmail.com> ;;; Copyright © 2020, 2021 Tobias Geerinckx-Rice <me@tobias.gr> ;;; @@ -44,20 +44,18 @@ #:select (with-file-lock/no-wait)) #:use-module (guix git) #:use-module (git) - #:use-module (gnu packages) - #:use-module ((guix scripts package) #:select (build-and-use-profile - delete-matching-generations)) - #:use-module ((gnu packages base) #:select (canonical-package)) - #:use-module (gnu packages guile) - #:use-module ((gnu packages bootstrap) - #:select (%bootstrap-guile)) - #:use-module ((gnu packages certs) #:select (le-certs)) + #:autoload (gnu packages) (fold-available-packages) + #:autoload (guix scripts package) (build-and-use-profile + delete-matching-generations) + #:autoload (gnu packages base) (canonical-package) + #:autoload (gnu packages bootstrap) (%bootstrap-guile) + #:autoload (gnu packages certs) (le-certs) #:use-module (srfi srfi-1) - #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) #:use-module (srfi srfi-34) #:use-module (srfi srfi-35) #:use-module (srfi srfi-37) + #:use-module (srfi srfi-71) #:use-module (ice-9 match) #:use-module (ice-9 vlist) #:use-module (ice-9 format) @@ -603,7 +601,7 @@ Return true when there is more package info to display." (string-join lst ", "))) (cut string-join <> ", "))) - (let-values (((new upgraded) (new/upgraded-packages alist1 alist2))) + (let ((new upgraded (new/upgraded-packages alist1 alist2))) (define new-count (length new)) (define upgraded-count (length upgraded)) diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 48309f9b3a..c044e1d47a 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -45,7 +45,7 @@ #:select (uri-abbreviation nar-uri-abbreviation (open-connection-for-uri . guix:open-connection-for-uri))) - #:autoload (gnutls) (error/invalid-session) + #:autoload (gnutls) (error/invalid-session error/again error/interrupted) #:use-module (guix progress) #:use-module ((guix build syscalls) #:select (set-thread-name)) @@ -163,7 +163,9 @@ if file doesn't exist, and the narinfo otherwise." (define (lookup-narinfo caches path authorized?) "Return the narinfo for PATH in CACHES, or #f when no substitute for PATH was found." - (match (lookup-narinfos/diverse caches (list path) authorized?) + (match (lookup-narinfos/diverse + caches (list path) authorized? + #:open-connection open-connection-for-uri/cached) ((answer) answer) (_ #f))) @@ -417,7 +419,14 @@ server certificates." (if (or (and (eq? key 'system-error) (= EPIPE (system-error-errno `(,key ,@args)))) (and (eq? key 'gnutls-error) - (eq? (first args) error/invalid-session)) + (memq (first args) + (list error/invalid-session + + ;; XXX: These two are not properly handled in + ;; GnuTLS < 3.7.3, in + ;; 'write_to_session_record_port'; see + ;; <https://bugs.gnu.org/47867>. + error/again error/interrupted))) (memq key '(bad-response bad-header bad-header-component))) (proc (open-connection-for-uri/cached uri #:verify-certificate? #f @@ -511,8 +520,11 @@ PORT." (current-error-port) #:abbreviation nar-uri-abbreviation)))) ;; Keep RAW open upon completion so we can later reuse - ;; the underlying connection. - (progress-report-port reporter raw #:close? #f))) + ;; the underlying connection. Pass the download size so + ;; that this procedure won't block reading from RAW. + (progress-report-port reporter raw + #:close? #f + #:download-size dl-size))) ((input pids) ;; NOTE: This 'progress' port of current process will be ;; closed here, while the child process doing the @@ -631,7 +643,8 @@ found." (#f ;; This can only happen when this script is not invoked by the ;; daemon. - '("http://ci.guix.gnu.org")))) + '("http://ci.guix.gnu.org" + "http://bordeaux.guix.gnu.org")))) ;; In order to prevent using large number of discovered local substitute ;; servers, limit the local substitute urls list size. @@ -764,7 +777,7 @@ default value." (loop)))))) ((or ("-V") ("--version")) (show-version-and-exit "guix substitute")) - (("--help") + ((or ("-h") ("--help")) (show-help)) (opts (leave (G_ "~a: unrecognized options~%") opts)))))) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index 0a051ee4e3..65eb98e4b2 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -9,6 +9,7 @@ ;;; Copyright © 2020 Julien Lepiller <julien@lepiller.eu> ;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2021 Brice Waegeneire <brice@waegenei.re> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -253,7 +254,7 @@ the ownership of '~a' may be incorrect!~%") #:target target) (return (info (G_ "bootloader successfully installed on '~a'~%") - (bootloader-configuration-target bootloader)))))))) + (bootloader-configuration-targets bootloader)))))))) ;;; @@ -717,6 +718,7 @@ checking this by themselves in their 'check' procedure." (lower-object (system-image image))) ((docker-image) (system-docker-image os + #:memory-size 1024 #:shared-network? container-shared-network?))))) (define (maybe-suggest-running-guix-pull) @@ -767,14 +769,13 @@ and TARGET arguments." skip-safety-checks? install-bootloader? dry-run? derivations-only? - use-substitutes? bootloader-target target + use-substitutes? target full-boot? container-shared-network? (mappings '()) (gc-root #f)) "Perform ACTION for IMAGE. INSTALL-BOOTLOADER? specifies whether to install -bootloader; BOOTLOADER-TAGET is the target for the bootloader; TARGET is the -target root directory. +bootloader; TARGET is the target root directory. FULL-BOOT? is used for the 'vm' action; it determines whether to boot directly to the kernel or to the bootloader. CONTAINER-SHARED-NETWORK? @@ -855,13 +856,13 @@ static checks." #:target (or target "/")) (return (info (G_ "bootloader successfully installed on '~a'~%") - (bootloader-configuration-target bootloader)))) + (bootloader-configuration-targets bootloader)))) (with-shepherd-error-handling - (upgrade-shepherd-services local-eval os) - (return (format #t (G_ "\ + (upgrade-shepherd-services local-eval os) + (return (format #t (G_ "\ To complete the upgrade, run 'herd restart SERVICE' to stop, upgrade, and restart each service that was not automatically restarted.\n"))) - (return (format #t (G_ "\ + (return (format #t (G_ "\ Run 'herd status' to view the list of services on your system.\n")))))) ((init) (newline) @@ -1152,6 +1153,13 @@ Some ACTIONS support additional ARGS.\n")) ;;; Entry point. ;;; +(define actions '("build" "container" "vm" "vm-image" "image" "disk-image" + "reconfigure" "init" + "extension-graph" "shepherd-graph" + "list-generations" "describe" + "delete-generations" "roll-back" + "switch-generation" "search" "docker-image")) + (define (process-action action args opts) "Process ACTION, a sub-command, with the arguments are listed in ARGS. ACTION must be one of the sub-commands that takes an operating system @@ -1217,9 +1225,9 @@ resulting from command-line parsing." (target-file (match args ((first second) second) (_ #f))) - (bootloader-target + (bootloader-targets (and bootloader? - (bootloader-configuration-target + (bootloader-configuration-targets (operating-system-bootloader os))))) (define (graph-backend) @@ -1268,7 +1276,6 @@ resulting from command-line parsing." opts) #:install-bootloader? bootloader? #:target target-file - #:bootloader-target bootloader-target #:gc-root (assoc-ref opts 'gc-root))))) #:target target #:system system))) @@ -1336,17 +1343,18 @@ argument list and OPTS is the option alist." (define (parse-sub-command arg result) ;; Parse sub-command ARG and augment RESULT accordingly. - (if (assoc-ref result 'action) - (alist-cons 'argument arg result) - (let ((action (string->symbol arg))) - (case action - ((build container vm vm-image image disk-image reconfigure init - extension-graph shepherd-graph - list-generations describe - delete-generations roll-back - switch-generation search docker-image) - (alist-cons 'action action result)) - (else (leave (G_ "~a: unknown action~%") action)))))) + (cond ((assoc-ref result 'action) + (alist-cons 'argument arg result)) + ((member arg actions) + (let ((action (string->symbol arg))) + (alist-cons 'action action result))) + (else + (let ((hint (string-closest arg actions #:threshold 3))) + (report-error (G_ "~a: unknown action~%") arg) + (when hint + (display-hint + (format #f (G_ "Did you mean @code{~a}?~%") hint))) + (exit 1))))) (define (match-pair car) ;; Return a procedure that matches a pair with CAR. diff --git a/guix/scripts/system/reconfigure.scm b/guix/scripts/system/reconfigure.scm index 39a818dd0b..bf23fb06af 100644 --- a/guix/scripts/system/reconfigure.scm +++ b/guix/scripts/system/reconfigure.scm @@ -207,10 +207,10 @@ services as defined by OS." (define (install-bootloader-program installer disk-installer bootloader-package bootcfg - bootcfg-file device target) + bootcfg-file devices target) "Return an executable store item that, upon being evaluated, will install -BOOTCFG to BOOTCFG-FILE, a target file name, on DEVICE, a file system device, -at TARGET, a mount point, and subsequently run INSTALLER from +BOOTCFG to BOOTCFG-FILE, a target file name, on DEVICES, a list of file system +devices, at TARGET, a mount point, and subsequently run INSTALLER from BOOTLOADER-PACKAGE." (program-file "install-bootloader.scm" @@ -254,11 +254,17 @@ BOOTLOADER-PACKAGE." ;; The bootloader might not support installation on a ;; mounted directory using the BOOTLOADER-INSTALLER ;; procedure. In that case, fallback to installing the - ;; bootloader directly on DEVICE using the + ;; bootloader directly on DEVICES using the ;; BOOTLOADER-DISK-IMAGE-INSTALLER procedure. (if #$installer - (#$installer #$bootloader-package #$device #$target) - (#$disk-installer #$bootloader-package 0 #$device))) + (for-each (lambda (device) + (#$installer #$bootloader-package device + #$target)) + '#$devices) + (for-each (lambda (device) + (#$disk-installer #$bootloader-package + 0 device)) + '#$devices))) (lambda args (delete-file new-gc-root) (match args @@ -284,7 +290,7 @@ additional configurations specified by MENU-ENTRIES can be selected." (disk-installer (and run-installer? (bootloader-disk-image-installer bootloader))) (package (bootloader-package bootloader)) - (device (bootloader-configuration-target configuration)) + (devices (bootloader-configuration-targets configuration)) (bootcfg-file (bootloader-configuration-file bootloader))) (eval #~(parameterize ((current-warning-port (%make-void-port "w"))) (primitive-load #$(install-bootloader-program installer @@ -292,7 +298,7 @@ additional configurations specified by MENU-ENTRIES can be selected." package bootcfg bootcfg-file - device + devices target)))))) @@ -308,12 +314,11 @@ ancestor of COMMIT, unless CHANNEL specifies a commit." ('self #t) (_ (raise (make-compound-condition - (condition - (&message (message - (format #f (G_ "\ + (formatted-message (G_ "\ aborting reconfiguration because commit ~a of channel '~a' is not a descendant of ~a") - commit (channel-name channel) - start))) + commit (channel-name channel) + start) + (condition (&fix-hint (hint (G_ "Use @option{--allow-downgrades} to force this downgrade."))))))))) diff --git a/guix/scripts/time-machine.scm b/guix/scripts/time-machine.scm index 4aafd432e8..5179ea035f 100644 --- a/guix/scripts/time-machine.scm +++ b/guix/scripts/time-machine.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2019 Konrad Hinsen <konrad.hinsen@fastmail.net> -;;; Copyright © 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; ;;; This file is part of GNU Guix. @@ -141,13 +141,19 @@ Execute COMMAND ARGS... in an older version of Guix.\n")) (let* ((opts (parse-args args)) (channels (channel-list opts)) (command-line (assoc-ref opts 'exec)) + (substitutes? (assoc-ref opts 'substitutes?)) (authenticate? (assoc-ref opts 'authenticate-channels?))) (when command-line (let* ((directory (with-store store (with-status-verbosity (assoc-ref opts 'verbosity) - (set-build-options-from-command-line store opts) - (cached-channel-instance store channels - #:authenticate? authenticate?)))) + (with-build-handler (build-notifier #:use-substitutes? + substitutes? + #:verbosity + (assoc-ref opts 'verbosity) + #:dry-run? #f) + (set-build-options-from-command-line store opts) + (cached-channel-instance store channels + #:authenticate? authenticate?))))) (executable (string-append directory "/bin/guix"))) (apply execl (cons* executable executable command-line)))))))) diff --git a/guix/scripts/weather.scm b/guix/scripts/weather.scm index 5164fe0494..60a697d1ac 100644 --- a/guix/scripts/weather.scm +++ b/guix/scripts/weather.scm @@ -1,8 +1,9 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net> ;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> ;;; Copyright © 2020 Simon Tournier <zimon.toutoune@gmail.com> +;;; Copyright © 2021 Maxime Devos <maximedevos@telenet.be> ;;; ;;; This file is part of GNU Guix. ;;; @@ -53,16 +54,18 @@ (define (all-packages) "Return the list of public packages we are going to query." - (fold-packages (lambda (package result) - (match (package-replacement package) - ((? package? replacement) - (cons* replacement package result)) - (#f - (cons package result)))) - '() + (delete-duplicates + (fold-packages (lambda (package result) + (match (package-replacement package) + ((? package? replacement) + (cons* replacement package result)) + (#f + (cons package result)))) + '() - ;; Dismiss deprecated packages but keep hidden packages. - #:select? (negate package-superseded))) + ;; Dismiss deprecated packages but keep hidden packages. + #:select? (negate package-superseded)) + eq?)) (define (call-with-progress-reporter reporter proc) "This is a variant of 'call-with-progress-reporter' that works with monadic @@ -171,13 +174,26 @@ about the derivations queued, as is the case with Hydra." #f ;no derivation information (lset-intersection string=? queued items))) +(define (store-item-system store item) + "Return the system (a string such as \"aarch64-linux\")) ITEM targets, +or #f if it could not be determined." + (match (valid-derivers store item) + ((drv . _) + (and=> (false-if-exception (read-derivation-from-file drv)) + derivation-system)) + (() + #f))) + (define* (report-server-coverage server items #:key display-missing?) "Report the subset of ITEMS available as substitutes on SERVER. When DISPLAY-MISSING? is true, display the list of missing substitutes. -Return the coverage ratio, an exact number between 0 and 1." +Return the coverage ratio, an exact number between 0 and 1. +In case ITEMS is an empty list, return 1 instead." (define MiB (* (expt 2 20) 1.)) + ;; TRANSLATORS: it is quite possible zero store items are + ;; looked for. (format #t (G_ "looking for ~h store items on ~a...~%") (length items) server) @@ -198,9 +214,10 @@ Return the coverage ratio, an exact number between 0 and 1." narinfos)) (time (+ (time-second time) (/ (time-nanosecond time) 1e9)))) - (format #t (G_ " ~,1f% substitutes available (~h out of ~h)~%") - (* 100. (/ obtained requested 1.)) - obtained requested) + (when (> requested 0) + (format #t (G_ " ~,1f% substitutes available (~h out of ~h)~%") + (* 100. (/ obtained requested 1.)) + obtained requested)) (let ((total (/ (reduce + 0 sizes) MiB))) (match (length sizes) ((? zero?) @@ -270,11 +287,28 @@ are queued~%") (when (and display-missing? (not (null? missing))) (newline) (format #t (G_ "Substitutes are missing for the following items:~%")) - (format #t "~{ ~a~%~}" missing)) + + ;; Display two columns: store items, and their system type. + (format #t "~:{ ~a ~a~%~}" + (zip (map (let ((width (max (- (current-terminal-columns) + 20) + 0))) + (lambda (item) + (if (> (string-length item) width) + item + (string-pad-right item width)))) + missing) + (with-store store + (map (lambda (item) + (or (store-item-system store item) + (G_ "unknown system"))) + missing))))) ;; Return the coverage ratio. (let ((total (length items))) - (/ (- total (length missing)) total))))) + (if (> total 0) + (/ (- total (length missing)) total) + 1))))) ;;; diff --git a/guix/self.scm b/guix/self.scm index 3154d180ac..5922ea6aa1 100644 --- a/guix/self.scm +++ b/guix/self.scm @@ -50,7 +50,7 @@ (let ((ref (lambda (module variable) (module-ref (resolve-interface module) variable)))) (match-lambda - ("guile" (ref '(gnu packages guile) 'guile-3.0/libgc-7)) + ("guile" (ref '(gnu packages guile) 'guile-3.0-latest)) ("guile-avahi" (ref '(gnu packages guile-xyz) 'guile-avahi)) ("guile-json" (ref '(gnu packages guile) 'guile-json-4)) ("guile-ssh" (ref '(gnu packages ssh) 'guile-ssh)) @@ -63,6 +63,7 @@ ("guile-zstd" (ref '(gnu packages guile) 'guile-zstd)) ("guile-gcrypt" (ref '(gnu packages gnupg) 'guile-gcrypt)) ("gnutls" (ref '(gnu packages tls) 'gnutls)) + ("disarchive" (ref '(gnu packages backup) 'disarchive)) ("gzip" (ref '(gnu packages compression) 'gzip)) ("bzip2" (ref '(gnu packages compression) 'bzip2)) ("xz" (ref '(gnu packages compression) 'xz)) @@ -406,9 +407,8 @@ a list of extra files, such as '(\"contributing\")." "\\.[a-z]{2}(_[A-Z]{2})?\\.po$"))) (define parallel-jobs - ;; Limit thread creation by 'n-par-for-each'. Going beyond can - ;; lead libgc 8.0.4 to abort with: - ;; mmap(PROT_NONE) failed + ;; Limit thread creation by 'n-par-for-each', mostly to put an + ;; upper bound on memory usage. (min (parallel-job-count) 4)) (mkdir #$output) @@ -718,7 +718,9 @@ load path." ("share/guix/ci.guix.gnu.org.pub" ;alias ,(file-append* source "/etc/substitutes/berlin.guix.gnu.org.pub")) ("share/guix/ci.guix.info.pub" ;alias - ,(file-append* source "/etc/substitutes/berlin.guix.gnu.org.pub"))))) + ,(file-append* source "/etc/substitutes/berlin.guix.gnu.org.pub")) + ("share/guix/bordeaux.guix.gnu.org.pub" + ,(file-append* source "/etc/substitutes/bordeaux.guix.gnu.org.pub"))))) (define* (whole-package name modules dependencies #:key @@ -842,6 +844,9 @@ itself." (define gnutls (specification->package "gnutls")) + (define disarchive + (specification->package "disarchive")) + (define dependencies (append-map transitive-package-dependencies (list guile-gcrypt gnutls guile-git guile-avahi @@ -867,7 +872,9 @@ itself." ;; rebuilt when the version changes, which in turn means we ;; can have substitutes for it. #:extra-modules - `(((guix config) => ,(make-config.scm))) + `(((guix config) + => ,(make-config.scm + #:config-variables %default-config-variables))) ;; (guix man-db) is needed at build-time by (guix profiles) ;; but we don't need to compile it; not compiling it allows @@ -878,7 +885,8 @@ itself." ("guix/store/schema.sql" ,(local-file "../guix/store/schema.sql"))) - #:extensions (list guile-gcrypt) + #:extensions (list guile-gcrypt + guile-json) ;for (guix swh) #:guile-for-build guile-for-build)) (define *extra-modules* @@ -950,13 +958,23 @@ itself." #:guile-for-build guile-for-build)) + (define *home-modules* + (scheme-node "guix-home" + `((gnu home) + (gnu home-services) + ,@(scheme-modules* source "gnu/home-services")) + (list *core-package-modules* *package-modules* + *extra-modules* *core-modules* *system-modules*) + #:extensions dependencies + #:guile-for-build guile-for-build)) + (define *cli-modules* (scheme-node "guix-cli" (append (scheme-modules* source "/guix/scripts") `((gnu ci))) (list *core-modules* *extra-modules* *core-package-modules* *package-modules* - *system-modules*) + *system-modules* *home-modules*) #:extensions dependencies #:guile-for-build guile-for-build)) @@ -968,6 +986,8 @@ itself." (list *core-package-modules* *package-modules* *extra-modules* *system-modules* *core-modules* *cli-modules*) ;for (guix scripts pack), etc. + #:extra-files (file-imports source "gnu/tests/data" + (const #t)) #:extensions dependencies #:guile-for-build guile-for-build)) @@ -1002,6 +1022,7 @@ itself." *cli-modules* *system-test-modules* *system-modules* + *home-modules* *package-modules* *core-package-modules* *extra-modules* @@ -1025,7 +1046,8 @@ itself." (let* ((modules (built-modules (compose list node-source+compiled))) (command (guix-command modules #:source source - #:dependencies dependencies + #:dependencies + (cons disarchive dependencies) #:guile guile-for-build #:guile-version guile-version))) (whole-package name modules dependencies @@ -1075,10 +1097,17 @@ itself." (variables rest ...)))))) (variables %localstatedir %storedir %sysconfdir))) +(define %default-config-variables + ;; Default values of the configuration variables above. + `((%localstatedir . "/var") + (%storedir . "/gnu/store") + (%sysconfdir . "/etc"))) + (define* (make-config.scm #:key gzip xz bzip2 (package-name "GNU Guix") (package-version "0") (channel-metadata #f) + (config-variables %config-variables) (bug-report-address "bug-guix@gnu.org") (home-page-url "https://guix.gnu.org")) @@ -1108,7 +1137,7 @@ itself." #$@(map (match-lambda ((name . value) #~(define-public #$name #$value))) - %config-variables) + config-variables) (define %store-directory (or (and=> (getenv "NIX_STORE_DIR") canonicalize-path) diff --git a/guix/ssh.scm b/guix/ssh.scm index 457d1890f9..232b6bfe94 100644 --- a/guix/ssh.scm +++ b/guix/ssh.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2016, 2017, 2018, 2019, 2020, 2021, 2021 Ludovic Courtès <ludo@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -253,7 +253,22 @@ EXP never returns or calls 'primitive-exit' when it's done." (use-modules (ice-9 match) (rnrs io ports) (rnrs bytevectors)) - (let ((sock (socket AF_UNIX SOCK_STREAM 0)) + (define connect-to-daemon + ;; XXX: 'connect-to-daemon' used to be private and before that it + ;; didn't even exist, hence these shenanigans. + (let ((connect-to-daemon + (false-if-exception (module-ref (resolve-module '(guix store)) + 'connect-to-daemon)))) + (lambda (uri) + (if connect-to-daemon + (connect-to-daemon uri) + (let ((sock (socket AF_UNIX SOCK_STREAM 0))) + (connect sock AF_UNIX ,socket-name) + sock))))) + + ;; Use 'connect-to-daemon' to honor GUIX_DAEMON_SOCKET. + (let ((sock (connect-to-daemon (or (getenv "GUIX_DAEMON_SOCKET") + ,socket-name))) (stdin (current-input-port)) (stdout (current-output-port)) (select* (lambda (read write except) @@ -272,8 +287,6 @@ EXP never returns or calls 'primitive-exit' when it's done." (setvbuf stdin 'block 65536) (setvbuf sock 'block 65536) - (connect sock AF_UNIX ,socket-name) - (let loop () (match (select* (list stdin sock) '() '()) ((reads () ()) @@ -302,8 +315,13 @@ EXP never returns or calls 'primitive-exit' when it's done." "/var/guix/daemon-socket/socket")) "Connect to the remote build daemon listening on SOCKET-NAME over SESSION, an SSH session. Return a <store-connection> object." - (open-connection #:port (remote-daemon-channel session socket-name))) - + (guard (c ((store-connection-error? c) + ;; Raise a more focused error condition. + (raise (formatted-message + (G_ "failed to connect over SSH to daemon at '~a', socket ~a") + (session-get session 'host) + socket-name)))) + (open-connection #:port (remote-daemon-channel session socket-name)))) (define (store-import-channel session) "Return an output port to which archives to be exported to SESSION's store diff --git a/guix/status.scm b/guix/status.scm index 362ae2882c..f351a56d92 100644 --- a/guix/status.scm +++ b/guix/status.scm @@ -379,6 +379,8 @@ the current build phase." (G_ "building GHC package cache...")) ('ca-certificate-bundle (G_ "building CA certificate bundle...")) + ('emacs-subdirs + (G_ "listing Emacs sub-directories...")) ('glib-schemas (G_ "generating GLib schema cache...")) ('gtk-icon-themes @@ -552,12 +554,17 @@ substitutes being downloaded." (download-start download) #:transferred transferred)))))) (('substituter-succeeded item _ ...) - ;; If there are no jobs running, we already reported download completion - ;; so there's nothing left to do. - (unless (and (zero? (simultaneous-jobs status)) - (extended-build-trace-supported?)) - (format port (success (G_ "substitution of ~a complete")) item) - (newline port))) + (when (extended-build-trace-supported?) + ;; If there are no jobs running, we already reported download completion + ;; so there's nothing left to do. + (unless (zero? (simultaneous-jobs status)) + (format port (success (G_ "substitution of ~a complete")) item) + (newline port)) + + (when (and print-urls? (zero? (simultaneous-jobs status))) + ;; Leave a blank line after the "downloading ..." line and the + ;; progress bar (that's three lines in total). + (newline port)))) (('substituter-failed item _ ...) (format port (failure (G_ "substitution of ~a failed")) item) (newline port)) diff --git a/guix/store.scm b/guix/store.scm index 37ae6cfedd..89a719bcfc 100644 --- a/guix/store.scm +++ b/guix/store.scm @@ -36,6 +36,7 @@ #:use-module (rnrs bytevectors) #:use-module (ice-9 binary-ports) #:use-module ((ice-9 control) #:select (let/ec)) + #:use-module (ice-9 atomic) #:use-module (srfi srfi-1) #:use-module (srfi srfi-9) #:use-module (srfi srfi-9 gnu) @@ -47,7 +48,7 @@ #:use-module (ice-9 match) #:use-module (ice-9 vlist) #:use-module (ice-9 popen) - #:use-module (ice-9 threads) + #:autoload (ice-9 threads) (current-processor-count) #:use-module (ice-9 format) #:use-module (web uri) #:export (%daemon-socket-uri @@ -68,6 +69,7 @@ nix-server-socket current-store-protocol-version ;for internal use + cache-lookup-recorder ;for internal use mcached &store-error store-error? @@ -87,9 +89,15 @@ nix-protocol-error-message nix-protocol-error-status + allocate-store-connection-cache + store-connection-cache + set-store-connection-cache + set-store-connection-cache! + hash-algo build-mode + connect-to-daemon open-connection port->connection close-connection @@ -140,7 +148,6 @@ built-in-builders references references/cached - references/substitutes references* query-path-info* requisites @@ -382,8 +389,8 @@ ;; the session. (ats-cache store-connection-add-to-store-cache) (atts-cache store-connection-add-text-to-store-cache) - (object-cache store-connection-object-cache - (default vlist-null)) ;vhash + (caches store-connection-caches + (default '#())) ;vector (built-in-builders store-connection-built-in-builders (default (delay '())))) ;promise @@ -501,7 +508,10 @@ (define (connect-to-daemon uri) "Connect to the daemon at URI, a string that may be an actual URI or a file -name." +name, and return an input/output port. + +This is a low-level procedure that does not perform the initial handshake with +the daemon. Use 'open-connection' for that." (define (not-supported) (raise (condition (&store-connection-error (file uri) @@ -548,13 +558,16 @@ space on the file system so that the garbage collector can still operate, should the disk become full. When CPU-AFFINITY is true, it must be an integer corresponding to an OS-level CPU number to which the daemon's worker process for this connection will be pinned. Return a server object." + (define (handshake-error) + (raise (condition + (&store-connection-error (file (or port uri)) + (errno EPROTO)) + (&message (message "build daemon handshake failed"))))) + (guard (c ((nar-error? c) ;; One of the 'write-' or 'read-' calls below failed, but this is ;; really a connection error. - (raise (condition - (&store-connection-error (file (or port uri)) - (errno EPROTO)) - (&message (message "build daemon handshake failed")))))) + (handshake-error))) (let*-values (((port) (or port (connect-to-daemon uri))) ((output flush) @@ -562,32 +575,39 @@ for this connection will be pinned. Return a server object." (make-bytevector 8192)))) (write-int %worker-magic-1 port) (let ((r (read-int port))) - (and (= r %worker-magic-2) - (let ((v (read-int port))) - (and (= (protocol-major %protocol-version) - (protocol-major v)) - (begin - (write-int %protocol-version port) - (when (>= (protocol-minor v) 14) - (write-int (if cpu-affinity 1 0) port) - (when cpu-affinity - (write-int cpu-affinity port))) - (when (>= (protocol-minor v) 11) - (write-int (if reserve-space? 1 0) port)) - (letrec* ((built-in-builders - (delay (%built-in-builders conn))) - (conn - (%make-store-connection port - (protocol-major v) - (protocol-minor v) - output flush - (make-hash-table 100) - (make-hash-table 100) - vlist-null - built-in-builders))) - (let loop ((done? (process-stderr conn))) - (or done? (process-stderr conn))) - conn))))))))) + (unless (= r %worker-magic-2) + (handshake-error)) + + (let ((v (read-int port))) + (unless (= (protocol-major %protocol-version) + (protocol-major v)) + (handshake-error)) + + (write-int %protocol-version port) + (when (>= (protocol-minor v) 14) + (write-int (if cpu-affinity 1 0) port) + (when cpu-affinity + (write-int cpu-affinity port))) + (when (>= (protocol-minor v) 11) + (write-int (if reserve-space? 1 0) port)) + (letrec* ((built-in-builders + (delay (%built-in-builders conn))) + (caches + (make-vector + (atomic-box-ref %store-connection-caches) + vlist-null)) + (conn + (%make-store-connection port + (protocol-major v) + (protocol-minor v) + output flush + (make-hash-table 100) + (make-hash-table 100) + caches + built-in-builders))) + (let loop ((done? (process-stderr conn))) + (or done? (process-stderr conn))) + conn)))))) (define* (port->connection port #:key (version %protocol-version)) @@ -606,7 +626,9 @@ connection. Use with care." output flush (make-hash-table 100) (make-hash-table 100) - vlist-null + (make-vector + (atomic-box-ref %store-connection-caches) + vlist-null) (delay (%built-in-builders connection)))) connection)) @@ -638,18 +660,10 @@ connection. Use with care." (close-connection store) (apply values results))))) - (cond-expand - (guile-3 - (with-exception-handler (lambda (exception) - (close-connection store) - (raise-exception exception)) - thunk)) - (else ;Guile 2.2 - (catch #t - thunk - (lambda (key . args) - (close-connection store) - (apply throw key args))))))) + (with-exception-handler (lambda (exception) + (close-connection store) + (raise-exception exception)) + thunk))) (define-syntax-rule (with-store store exp ...) "Bind STORE to an open connection to the store and evaluate EXPs; @@ -773,7 +787,8 @@ encoding conversion errors." (map (if (false-if-exception (resolve-interface '(gnutls))) (cut string-append "https://" <>) (cut string-append "http://" <>)) - '("ci.guix.gnu.org"))) + '("ci.guix.gnu.org" + "bordeaux.guix.gnu.org"))) (define (current-user-name) "Return the name of the calling user." @@ -1340,14 +1355,33 @@ on the build output of a previous derivation." (unresolved things continue) (continue #t))) -(define (map/accumulate-builds store proc lst) +(define* (map/accumulate-builds store proc lst + #:key (cutoff 30)) "Apply PROC over each element of LST, accumulating 'build-things' calls and -coalescing them into a single call." - (define result - (map (lambda (obj) - (with-build-handler build-accumulator - (proc obj))) - lst)) +coalescing them into a single call. + +CUTOFF is the threshold above which we stop accumulating unresolved nodes." + + ;; The CUTOFF parameter helps avoid pessimal behavior where we keep + ;; stumbling upon the same .drv build requests with many incoming edges. + ;; See <https://bugs.gnu.org/49439>. + + (define-values (result rest) + (let loop ((lst lst) + (result '()) + (unresolved 0)) + (match lst + ((head . tail) + (match (with-build-handler build-accumulator + (proc head)) + ((? unresolved? obj) + (if (>= unresolved cutoff) + (values (reverse (cons obj result)) tail) + (loop tail (cons obj result) (+ 1 unresolved)))) + (obj + (loop tail (cons obj result) unresolved)))) + (() + (values (reverse result) lst))))) (match (append-map (lambda (obj) (if (unresolved? obj) @@ -1355,19 +1389,23 @@ coalescing them into a single call." '())) result) (() + ;; REST is necessarily empty. result) (to-build - ;; We've accumulated things TO-BUILD. Actually build them and resume the - ;; corresponding continuations. + ;; We've accumulated things TO-BUILD; build them. (build-things store (delete-duplicates to-build)) - (map/accumulate-builds store - (lambda (obj) - (if (unresolved? obj) - ;; Pass #f because 'build-things' is now - ;; unnecessary. - ((unresolved-continuation obj) #f) - obj)) - result)))) + + ;; Resume the continuations corresponding to TO-BUILD, and then process + ;; REST. + (append (map/accumulate-builds store + (lambda (obj) + (if (unresolved? obj) + ;; Pass #f because 'build-things' is now + ;; unnecessary. + ((unresolved-continuation obj) #f) + obj)) + result #:cutoff cutoff) + (map/accumulate-builds store proc rest #:cutoff cutoff))))) (define build-things (let ((build (operation (build-things (string-list things) @@ -1462,73 +1500,6 @@ error if there is no such root." "Return the list of references of PATH." store-path-list)) -(define %reference-cache - ;; Brute-force cache mapping store items to their list of references. - ;; Caching matters because when building a profile in the presence of - ;; grafts, we keep calling 'graft-derivation', which in turn calls - ;; 'references/substitutes' many times with the same arguments. Ideally we - ;; would use a cache associated with the daemon connection instead (XXX). - (make-hash-table 100)) - -(define (references/cached store item) - "Like 'references', but cache results." - (or (hash-ref %reference-cache item) - (let ((references (references store item))) - (hash-set! %reference-cache item references) - references))) - -(define (references/substitutes store items) - "Return the list of list of references of ITEMS; the result has the same -length as ITEMS. Query substitute information for any item missing from the -store at once. Raise a '&store-protocol-error' exception if reference -information for one of ITEMS is missing." - (let* ((requested items) - (local-refs (map (lambda (item) - (or (hash-ref %reference-cache item) - (guard (c ((store-protocol-error? c) #f)) - (references store item)))) - items)) - (missing (fold-right (lambda (item local-ref result) - (if local-ref - result - (cons item result))) - '() - items local-refs)) - - ;; Query all the substitutes at once to minimize the cost of - ;; launching 'guix substitute' and making HTTP requests. - (substs (if (null? missing) - '() - (substitutable-path-info store missing)))) - (when (< (length substs) (length missing)) - (raise (condition (&store-protocol-error - (message "cannot determine \ -the list of references") - (status 1))))) - - ;; Intersperse SUBSTS and LOCAL-REFS. - (let loop ((items items) - (local-refs local-refs) - (result '())) - (match items - (() - (let ((result (reverse result))) - (for-each (cut hash-set! %reference-cache <> <>) - requested result) - result)) - ((item items ...) - (match local-refs - ((#f tail ...) - (loop items tail - (cons (any (lambda (subst) - (and (string=? (substitutable-path subst) item) - (substitutable-references subst))) - substs) - result))) - ((head tail ...) - (loop items tail - (cons head result))))))))) - (define* (fold-path store proc seed paths #:optional (relatives (cut references store <>))) "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the @@ -1799,6 +1770,77 @@ This makes sense only when the daemon was started with '--cache-failures'." ;;; +;;; Per-connection caches. +;;; + +;; Number of currently allocated store connection caches--things that go in +;; the 'caches' vector of <store-connection>. +(define %store-connection-caches (make-atomic-box 0)) + +(define (allocate-store-connection-cache name) + "Allocate a new cache for store connections and return its identifier. Said +identifier can be passed as an argument to " + (let loop ((current (atomic-box-ref %store-connection-caches))) + (let ((previous (atomic-box-compare-and-swap! %store-connection-caches + current (+ current 1)))) + (if (= previous current) + current + (loop current))))) + +(define %object-cache-id + ;; The "object cache", mapping lowerable objects such as <package> records + ;; to derivations. + (allocate-store-connection-cache 'object-cache)) + +(define (vector-set vector index value) + (let ((new (vector-copy vector))) + (vector-set! new index value) + new)) + +(define (store-connection-cache store cache) + "Return the cache of STORE identified by CACHE, an identifier as returned by +'allocate-store-connection-cache'." + (vector-ref (store-connection-caches store) cache)) + +(define (set-store-connection-cache store cache value) + "Return a copy of STORE where CACHE has the given VALUE. CACHE must be a +value returned by 'allocate-store-connection-cache'." + (store-connection + (inherit store) + (caches (vector-set (store-connection-caches store) cache value)))) + +(define set-store-connection-caches! ;private + (record-modifier <store-connection> 'caches)) + +(define (set-store-connection-cache! store cache value) + "Set STORE's CACHE to VALUE. + +This is a mutating version that should be avoided. Prefer the functional +'set-store-connection-cache' instead, together with using %STORE-MONAD." + (vector-set! (store-connection-caches store) cache value)) + + +(define %reference-cache-id + ;; Cache mapping store items to their list of references. Caching matters + ;; because when building a profile in the presence of grafts, we keep + ;; calling 'graft-derivation', which in turn calls 'references/cached' many + ;; times with the same arguments. + (allocate-store-connection-cache 'reference-cache)) + +(define (references/cached store item) + "Like 'references', but cache results." + (let ((cache (store-connection-cache store %reference-cache-id))) + (match (vhash-assoc item cache) + ((_ . references) + references) + (#f + (let* ((references (references store item)) + (cache (vhash-cons item references cache))) + (set-store-connection-cache! store %reference-cache-id cache) + references))))) + + +;;; ;;; Store monad. ;;; @@ -1817,7 +1859,9 @@ This makes sense only when the daemon was started with '--cache-failures'." (template-directory instantiations %store-monad) (define* (cache-object-mapping object keys result - #:key (vhash-cons vhash-consq)) + #:key + (cache %object-cache-id) + (vhash-cons vhash-consq)) "Augment the store's object cache with a mapping from OBJECT/KEYS to RESULT. KEYS is a list of additional keys to match against, for instance a (SYSTEM TARGET) tuple. Use VHASH-CONS to insert OBJECT into the cache. @@ -1826,26 +1870,29 @@ OBJECT is typically a high-level object such as a <package> or an <origin>, and RESULT is typically its derivation." (lambda (store) (values result - (store-connection - (inherit store) - (object-cache (vhash-cons object (cons result keys) - (store-connection-object-cache store))))))) + (set-store-connection-cache + store cache + (vhash-cons object (cons result keys) + (store-connection-cache store cache)))))) -(define record-cache-lookup! - (if (profiled? "object-cache") +(define (cache-lookup-recorder component title) + "Return a procedure of two arguments to record cache lookups, hits, and +misses for COMPONENT. The procedure must be passed a Boolean indicating +whether the cache lookup was a hit, and the actual cache (a vhash)." + (if (profiled? component) (let ((fresh 0) (lookups 0) (hits 0) (size 0)) (register-profiling-hook! - "object-cache" + component (lambda () - (format (current-error-port) "Store object cache: + (format (current-error-port) "~a: fresh caches: ~5@a lookups: ~5@a hits: ~5@a (~,1f%) cache size: ~5@a entries~%" - fresh lookups hits + title fresh lookups hits (if (zero? lookups) 100. (* 100. (/ hits lookups))) @@ -1853,9 +1900,9 @@ and RESULT is typically its derivation." (lambda (hit? cache) (set! fresh - (if (eq? cache vlist-null) - (+ 1 fresh) - fresh)) + (if (eq? cache vlist-null) + (+ 1 fresh) + fresh)) (set! lookups (+ 1 lookups)) (set! hits (if hit? (+ hits 1) hits)) (set! size (+ (if hit? 0 1) @@ -1863,13 +1910,16 @@ and RESULT is typically its derivation." (lambda (x y) #t))) +(define record-cache-lookup! + (cache-lookup-recorder "object-cache" "Store object cache")) + (define-inlinable (lookup-cached-object object keys vhash-fold*) "Return the cached object in the store connection corresponding to OBJECT and KEYS; use VHASH-FOLD* to look for OBJECT in the cache. KEYS is a list of additional keys to match against, and which are compared with 'equal?'. Return #f on failure and the cached result otherwise." (lambda (store) - (let* ((cache (store-connection-object-cache store)) + (let* ((cache (store-connection-cache store %object-cache-id)) ;; Escape as soon as we find the result. This avoids traversing ;; the whole vlist chain and significantly reduces the number of @@ -2046,9 +2096,6 @@ the store." ;; when using 'gexp->derivation' and co. (make-parameter #f)) -(define set-store-connection-object-cache! - (record-modifier <store-connection> 'object-cache)) - (define* (run-with-store store mval #:key (guile-for-build (%guile-for-build)) @@ -2068,8 +2115,8 @@ connection, and return the result." (when (and store new-store) ;; Copy the object cache from NEW-STORE so we don't fully discard ;; the state. - (let ((cache (store-connection-object-cache new-store))) - (set-store-connection-object-cache! store cache))) + (let ((caches (store-connection-caches new-store))) + (set-store-connection-caches! store caches))) result)))) diff --git a/guix/substitutes.scm b/guix/substitutes.scm index 08f8c24efd..a5c554acff 100644 --- a/guix/substitutes.scm +++ b/guix/substitutes.scm @@ -1,5 +1,5 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2014 Nikita Karetnikov <nikita@karetnikov.org> ;;; Copyright © 2018 Kyle Meyer <kyle@kyleam.com> ;;; Copyright © 2020 Christopher Baines <mail@cbaines.net> @@ -37,7 +37,8 @@ #:use-module ((guix build utils) #:select (mkdir-p dump-port)) #:use-module ((guix build download) #:select ((open-connection-for-uri - . guix:open-connection-for-uri))) + . guix:open-connection-for-uri) + resolve-uri-reference)) #:use-module (guix progress) #:use-module (ice-9 rdelim) #:use-module (ice-9 regex) @@ -72,11 +73,11 @@ (define %narinfo-negative-ttl ;; Likewise, but for negative lookups---i.e., cached lookup failures (404). - (* 1 3600)) + (* 10 60)) (define %narinfo-transient-error-ttl ;; Likewise, but for transient errors such as 504 ("Gateway timeout"). - (* 10 60)) + (* 5 60)) (define %narinfo-cache-directory ;; A local cache of narinfos, to avoid going to the network. Most of the @@ -155,10 +156,12 @@ indicates that PATH is unavailable at CACHE-URL." (define (narinfo-request cache-url path) "Return an HTTP request for the narinfo of PATH at CACHE-URL." - (let ((url (string-append cache-url "/" (store-path-hash-part path) - ".narinfo")) - (headers '((User-Agent . "GNU Guile")))) - (build-request (string->uri url) #:method 'GET #:headers headers))) + (let* ((base (string->uri cache-url)) + (ref (build-relative-ref + #:path (string-append (store-path-hash-part path) ".narinfo"))) + (url (resolve-uri-reference ref base)) + (headers '((User-Agent . "GNU Guile")))) + (build-request url #:method 'GET #:headers headers))) (define (narinfo-from-file file url) "Attempt to read a narinfo from FILE, using URL as the cache URL. Return #f diff --git a/guix/swh.scm b/guix/swh.scm index f11b7ea2d5..5c41685a24 100644 --- a/guix/swh.scm +++ b/guix/swh.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -54,6 +55,11 @@ visit-number visit-snapshot + snapshot? + snapshot-id + snapshot-branches + lookup-snapshot-branch + branch? branch-name branch-target @@ -98,16 +104,16 @@ vault-reply? vault-reply-id vault-reply-fetch-url - vault-reply-object-id - vault-reply-object-type vault-reply-progress-message vault-reply-status + vault-reply-swhid query-vault request-cooking vault-fetch commit-id? + swh-download-directory swh-download)) ;;; Commentary: @@ -181,6 +187,12 @@ Software Heritage." (ref 10)))))) str)) ;oops! +(define (maybe-null proc) + (match-lambda + ((? null?) #f) + ('null #f) + (obj (proc obj)))) + (define string* ;; Converts "string or #nil" coming from JSON to "string or #f". (match-lambda @@ -285,6 +297,7 @@ FALSE-IF-404? is true, return #f upon 404 responses." ;; <https://archive.softwareheritage.org/api/1/snapshot/4334c3ed4bb208604ed780d8687fe523837f1bd1/> (define-json-mapping <snapshot> make-snapshot snapshot? json->snapshot + (id snapshot-id) (branches snapshot-branches "branches" json->branches)) ;; This is used for the "branches" field of snapshots. @@ -314,10 +327,13 @@ FALSE-IF-404? is true, return #f upon 404 responses." (target-url release-target-url "target_url")) ;; <https://archive.softwareheritage.org/api/1/revision/359fdda40f754bbf1b5dc261e7427b75463b59be/> +;; Note: Some revisions, such as those for "nixguix" origins (e.g., +;; <https://archive.softwareheritage.org/api/1/revision/b8dbc65475bbedde8e015d4730ade8864c38fad3/>), +;; have their 'date' field set to null. (define-json-mapping <revision> make-revision revision? json->revision (id revision-id) - (date revision-date "date" string->date*) + (date revision-date "date" (maybe-null string->date*)) (directory revision-directory) (directory-url revision-directory-url "directory_url")) @@ -374,10 +390,9 @@ FALSE-IF-404? is true, return #f upon 404 responses." json->vault-reply (id vault-reply-id) (fetch-url vault-reply-fetch-url "fetch_url") - (object-id vault-reply-object-id "obj_id") - (object-type vault-reply-object-type "obj_type" string->symbol) (progress-message vault-reply-progress-message "progress_message") - (status vault-reply-status "status" string->symbol)) + (status vault-reply-status "status" string->symbol) + (swhid vault-reply-swhid)) ;;; @@ -424,6 +439,32 @@ available." (call (swh-url (visit-snapshot-url visit)) json->snapshot))) +(define (snapshot-url snapshot branch-count first-branch) + "Return the URL of SNAPSHOT such that it contains information for +BRANCH-COUNT branches, starting at FIRST-BRANCH." + (string-append (swh-url "/api/1/snapshot" (snapshot-id snapshot)) + "?branches_count=" (number->string branch-count) + "&branches_from=" (uri-encode first-branch))) + +(define (lookup-snapshot-branch snapshot name) + "Look up branch NAME on SNAPSHOT. Return the branch, or return #f if it +could not be found." + (or (find (lambda (branch) + (string=? (branch-name branch) name)) + (snapshot-branches snapshot)) + + ;; There's no API entry point to look up a snapshot branch by name. + ;; Work around that by using the paginated list of branches provided by + ;; the /api/1/snapshot API: ask for one branch, and start pagination at + ;; NAME. + (let ((snapshot (call (snapshot-url snapshot 1 name) + json->snapshot))) + (match (snapshot-branches snapshot) + ((branch) + (and (string=? (branch-name branch) name) + branch)) + (_ #f))))) + (define (branch-target branch) "Return the target of BRANCH, either a <revision> or a <release>." (match (branch-target-type branch) @@ -446,12 +487,21 @@ URL could not be found." (match (lookup-origin url) (#f #f) (origin - (match (filter visit-snapshot-url (origin-visits origin)) + (match (filter (lambda (visit) + ;; Return #f if (visit-snapshot VISIT) would return #f. + (and (visit-snapshot-url visit) + (eq? 'full (visit-status visit)))) + (origin-visits origin)) ((visit . _) (let ((snapshot (visit-snapshot visit))) (match (and=> (find (lambda (branch) - (string=? (string-append "refs/tags/" tag) - (branch-name branch))) + (or + ;; Git specific. + (string=? (string-append "refs/tags/" tag) + (branch-name branch)) + ;; Hg specific. + (string=? tag + (branch-name branch)))) (snapshot-branches snapshot)) branch-target) ((? release? release) @@ -488,35 +538,57 @@ directory entries; if it has type 'file, return its <content> object." (path "/api/1/origin/save" type "url" url) json->save-reply) -(define-query (query-vault id kind) - "Ask the availability of object ID and KIND to the vault, where KIND is -'directory or 'revision. Return #f if it could not be found, or a -<vault-reply> on success." - ;; <https://docs.softwareheritage.org/devel/swh-vault/api.html#vault-api-ref> - ;; There's a single format supported for directories and revisions and for - ;; now, the "/format" bit of the URL *must* be omitted. - (path "/api/1/vault" (symbol->string kind) id) - json->vault-reply) +(define* (vault-url id kind #:optional (archive-type 'flat)) + "Return the vault query/cooking URL for ID and KIND. Normally, ID is an +SWHID and KIND is #f; the deprecated convention is to set ID to a raw +directory or revision ID and KIND to 'revision or 'directory." + ;; Note: /api/1/vault/directory/ID was deprecated in favor of + ;; /api/1/vault/flat/SWHID; this procedure "converts" automatically. + (let ((id (match kind + ('directory (string-append "swh:1:dir:" id)) + ('revision (string-append "swh:1:rev:" id)) + (#f id)))) + (swh-url "/api/1/vault" (symbol->string archive-type) id))) + +(define* (query-vault id #:optional kind #:key (archive-type 'flat)) + "Ask the availability of object ID (an SWHID) to the vault. Return #f if it +could not be found, or a <vault-reply> on success. ARCHIVE-TYPE can be 'flat +for a tarball containing a directory, or 'git-bare for a tarball containing a +bare Git repository corresponding to a revision. -(define (request-cooking id kind) - "Request the cooking of object ID and KIND (one of 'directory or 'revision) -to the vault. Return a <vault-reply>." - (call (swh-url "/api/1/vault" (symbol->string kind) id) +Passing KIND (one of 'directory or 'revision) together with a raw revision or +directory identifier is deprecated." + (call (vault-url id kind archive-type) + json->vault-reply)) + +(define* (request-cooking id #:optional kind #:key (archive-type 'flat)) + "Request the cooking of object ID, an SWHID. Return a <vault-reply>. +ARCHIVE-TYPE can be 'flat for a tarball containing a directory, or 'git-bare +for a tarball containing a bare Git repository corresponding to a revision. + +Passing KIND (one of 'directory or 'revision) together with a raw revision or +directory identifier is deprecated." + (call (vault-url id kind archive-type) json->vault-reply http-post*)) -(define* (vault-fetch id kind - #:key (log-port (current-error-port))) - "Return an input port from which a bundle of the object with the given ID -and KIND (one of 'directory or 'revision) can be retrieved, or #f if the -object could not be found. +(define* (vault-fetch id + #:optional kind + #:key + (archive-type 'flat) + (log-port (current-error-port))) + "Return an input port from which a bundle of the object with the given ID, +an SWHID, or #f if the object could not be found. -For a directory, the returned stream is a gzip-compressed tarball. For a -revision, it is a gzip-compressed stream for 'git fast-import'." - (let loop ((reply (query-vault id kind))) +ARCHIVE-TYPE can be 'flat for a tarball containing a directory, or 'git-bare +for a tarball containing a bare Git repository corresponding to a revision." + (let loop ((reply (query-vault id kind + #:archive-type archive-type))) (match reply (#f - (and=> (request-cooking id kind) loop)) + (and=> (request-cooking id kind + #:archive-type archive-type) + loop)) (_ (match (vault-reply-status reply) ('done @@ -536,7 +608,8 @@ revision, it is a gzip-compressed stream for 'git fast-import'." (format log-port "SWH vault: failure: ~a~%" (vault-reply-progress-message reply)) (format log-port "SWH vault: retrying...~%") - (loop (request-cooking id kind))) + (loop (request-cooking id kind + #:archive-type archive-type))) ((and (or 'new 'pending) status) ;; Wait until the bundle shows up. (let ((message (vault-reply-progress-message reply))) @@ -551,19 +624,14 @@ requested bundle cooking, waiting for completion...~%")) ;; requests per hour per IP address.) (sleep (if (eq? status 'new) 60 30)) - (loop (query-vault id kind))))))))) + (loop (query-vault id kind + #:archive-type archive-type))))))))) ;;; ;;; High-level interface. ;;; -(define (commit-id? reference) - "Return true if REFERENCE is likely a commit ID, false otherwise---e.g., if -it is a tag name. This is based on a simple heuristic so use with care!" - (and (= (string-length reference) 40) - (string-every char-set:hex-digit reference))) - (define (call-with-temporary-directory proc) ;FIXME: factorize "Call PROC with a name of a temporary directory; close the directory and delete it when leaving the dynamic extent of this call." @@ -577,9 +645,62 @@ delete it when leaving the dynamic extent of this call." (lambda () (false-if-exception (delete-file-recursively tmp-dir)))))) +(define* (swh-download-archive swhid output + #:key + (archive-type 'flat) + (log-port (current-error-port))) + "Download from Software Heritage the directory or revision with the given +SWID, in the ARCHIVE-TYPE format (one of 'flat or 'git-bare), and unpack it to +OUTPUT. Return #t on success and #f on failure." + (call-with-temporary-directory + (lambda (directory) + (match (vault-fetch swhid + #:archive-type archive-type + #:log-port log-port) + (#f + (format log-port + "SWH: object ~a could not be fetched from the vault~%" + swhid) + #f) + ((? port? input) + (let ((tar (open-pipe* OPEN_WRITE "tar" "-C" directory + (match archive-type + ('flat "-xzvf") ;gzipped + ('git-bare "-xvf")) ;uncompressed + "-"))) + (dump-port input tar) + (close-port input) + (let ((status (close-pipe tar))) + (unless (zero? status) + (error "tar extraction failure" status))) + + (match (scandir directory) + (("." ".." sub-directory) + (copy-recursively (string-append directory "/" sub-directory) + output + #:log (%make-void-port "w")) + #t)))))))) + +(define* (swh-download-directory id output + #:key (log-port (current-error-port))) + "Download from Software Heritage the directory with the given ID, and +unpack it to OUTPUT. Return #t on success and #f on failure." + (swh-download-archive (string-append "swh:1:dir:" id) output + #:archive-type 'flat + #:log-port log-port)) + +(define (commit-id? reference) + "Return true if REFERENCE is likely a commit ID, false otherwise---e.g., if +it is a tag name. This is based on a simple heuristic so use with care!" + (and (= (string-length reference) 40) + (string-every char-set:hex-digit reference))) + (define* (swh-download url reference output - #:key (log-port (current-error-port))) - "Download from Software Heritage a checkout of the Git tag or commit + #:key + (archive-type 'flat) + (log-port (current-error-port))) + "Download from Software Heritage a checkout (if ARCHIVE-TYPE is 'flat) or a +full Git repository (if ARCHIVE-TYPE is 'git-bare) of the Git tag or commit REFERENCE originating from URL, and unpack it in OUTPUT. Return #t on success and #f on failure. @@ -593,28 +714,18 @@ wait until it becomes available, which could take several minutes." (format log-port "SWH: found revision ~a with directory at '~a'~%" (revision-id revision) (swh-url (revision-directory-url revision))) - (call-with-temporary-directory - (lambda (directory) - (match (vault-fetch (revision-directory revision) 'directory - #:log-port log-port) - (#f - (format log-port - "SWH: directory ~a could not be fetched from the vault~%" - (revision-directory revision)) - #f) - ((? port? input) - (let ((tar (open-pipe* OPEN_WRITE "tar" "-C" directory "-xzvf" "-"))) - (dump-port input tar) - (close-port input) - (let ((status (close-pipe tar))) - (unless (zero? status) - (error "tar extraction failure" status))) - - (match (scandir directory) - (("." ".." sub-directory) - (copy-recursively (string-append directory "/" sub-directory) - output - #:log (%make-void-port "w")) - #t)))))))) + (swh-download-archive (match archive-type + ('flat + (string-append + "swh:1:dir:" (revision-directory revision))) + ('git-bare + (string-append + "swh:1:rev:" (revision-id revision)))) + output + #:archive-type archive-type + #:log-port log-port)) (#f + (format log-port + "SWH: revision ~s originating from ~a could not be found~%" + reference url) #f))) diff --git a/guix/tests/git.scm b/guix/tests/git.scm index b8e5f7e643..69960284d9 100644 --- a/guix/tests/git.scm +++ b/guix/tests/git.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2019, 2020 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> ;;; ;;; This file is part of GNU Guix. ;;; @@ -53,6 +54,7 @@ Return DIRECTORY on success." (with-environment-variables `(("GIT_CONFIG_NOSYSTEM" "1") ("GIT_ATTR_NOSYSTEM" "1") + ("GIT_CONFIG_GLOBAL" ,(string-append home "/.gitconfig")) ("HOME" ,home)) (apply invoke (git-command) "-C" directory command args))))) @@ -88,6 +90,9 @@ Return DIRECTORY on success." ((('tag name) rest ...) (git "tag" name) (loop rest)) + ((('tag name text) rest ...) + (git "tag" "-m" text name) + (loop rest)) ((('branch name) rest ...) (git "branch" name) (loop rest)) diff --git a/guix/transformations.scm b/guix/transformations.scm index 4e9260350c..5ae1977cb2 100644 --- a/guix/transformations.scm +++ b/guix/transformations.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Marius Bakke <marius@gnu.org> ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,7 +26,7 @@ #:autoload (guix download) (download-to-store) #:autoload (guix git-download) (git-reference? git-reference-url) #:autoload (guix git) (git-checkout git-checkout? git-checkout-url) - #:autoload (guix upstream) (package-latest-release* + #:autoload (guix upstream) (package-latest-release upstream-source-version upstream-source-signature-urls) #:use-module (guix utils) @@ -270,6 +271,25 @@ strings like \"guile-next=stable-3.0\" meaning that packages are built using (rewrite obj) obj)))) +(define (commit->version-string commit) + "Return a string suitable for use in the 'version' field of a package based +on the given COMMIT." + (cond ((and (> (string-length commit) 1) + (string-prefix? "v" commit) + (char-set-contains? char-set:digit + (string-ref commit 1))) + ;; Probably a tag like "v1.0" or a 'git describe' identifier. + (string-drop commit 1)) + ((not (string-every char-set:hex-digit commit)) + ;; Pass through tags and 'git describe' style IDs directly. + commit) + (else + (string-append "git." + (if (< (string-length commit) 7) + commit + (string-take commit 7)))))) + + (define (transform-package-source-commit replacement-specs) "Return a procedure that, when passed a package, replaces its direct dependencies according to REPLACEMENT-SPECS. REPLACEMENT-SPECS is a list of @@ -278,15 +298,7 @@ strings like \"guile-next=cabba9e\" meaning that packages are built using (define (replace old url commit) (package (inherit old) - (version (if (and (> (string-length commit) 1) - (string-prefix? "v" commit) - (char-set-contains? char-set:digit - (string-ref commit 1))) - (string-drop commit 1) ;looks like a tag like "v1.0" - (string-append "git." - (if (< (string-length commit) 7) - commit - (string-take commit 7))))) + (version (commit->version-string commit)) (source (git-checkout (url url) (commit commit) (recursive? #t))))) @@ -460,19 +472,46 @@ to the same package but with #:strip-binaries? #f in its 'arguments' field." (rewrite obj) obj))) +(define (patched-source name source patches) + "Return a file-like object with the given NAME that applies PATCHES to +SOURCE. SOURCE must itself be a file-like object of any type, including +<git-checkout>, <local-file>, etc." + (define patch + (module-ref (resolve-interface '(gnu packages base)) 'patch)) + + (computed-file name + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (setenv "PATH" #+(file-append patch "/bin")) + + ;; XXX: Assume SOURCE is a directory. This is true in + ;; most practical cases, where it's a <git-checkout>. + (copy-recursively #+source #$output) + (chdir #$output) + (for-each (lambda (patch) + (invoke "patch" "-p1" "--batch" + "-i" patch)) + '(#+@patches)))))) + (define (transform-package-patches specs) "Return a procedure that, when passed a package, returns a package with additional patches." (define (package-with-extra-patches p patches) - (if (origin? (package-source p)) - (package/inherit p - (source (origin - (inherit (package-source p)) - (patches (append (map (lambda (file) - (local-file file)) - patches) - (origin-patches (package-source p))))))) - p)) + (let ((patches (map (lambda (file) + (local-file file)) + patches))) + (if (origin? (package-source p)) + (package/inherit p + (source (origin + (inherit (package-source p)) + (patches (append patches + (origin-patches (package-source p))))))) + (package/inherit p + (source (patched-source (string-append (package-full-name p "-") + "-source") + (package-source p) patches)))))) (define (coalesce-alist alist) ;; Coalesce multiple occurrences of the same key in ALIST. @@ -518,7 +557,7 @@ additional patches." "Return a procedure that rewrites package graphs such that those in SPECS are replaced by their latest upstream version." (define (package-with-latest-upstream p) - (let ((source (package-latest-release* p))) + (let ((source (package-latest-release p))) (cond ((not source) (warning (G_ "could not determine latest upstream release of '~a'~%") diff --git a/guix/ui.scm b/guix/ui.scm index 7fbd4c63a2..1428c254b3 100644 --- a/guix/ui.scm +++ b/guix/ui.scm @@ -16,6 +16,7 @@ ;;; Copyright © 2019, 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; Copyright © 2020 Arun Isaac <arunisaac@systemreboot.net> ;;; Copyright © 2020 Maxim Cournoyer <maxim.cournoyer@gmail.com> +;;; Copyright © 2018 Steve Sprang <scs@stevesprang.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -73,7 +74,6 @@ #:use-module (ice-9 format) #:use-module (ice-9 regex) #:autoload (ice-9 popen) (open-pipe* close-pipe) - #:autoload (system base compile) (compile-file) #:autoload (system repl repl) (start-repl) #:autoload (system repl debug) (make-debug stack->vector) #:autoload (web uri) (encode-and-join-uri-path) @@ -197,6 +197,12 @@ information, or #f if it could not be found." (stack-ref stack 1) ;skip the 'throw' frame last)))) +(define-syntax-rule (without-compiler-optimizations exp) + ;; Compile with the baseline compiler (-O1), which is much less expensive + ;; than -O2. + (parameterize (((@ (system base compile) default-optimization-level) 1)) + exp)) + (define* (load* file user-module #:key (on-error 'nothing-special)) "Load the user provided Scheme source code FILE." @@ -211,17 +217,7 @@ information, or #f if it could not be found." (catch #t (lambda () ;; XXX: Force a recompilation to avoid ABI issues. - ;; - ;; In 2.2.3, the bogus answer to <https://bugs.gnu.org/29226> was to - ;; ignore all available .go, not just those from ~/.cache, which in turn - ;; meant that we had to rebuild *everything*. Since this is too costly, - ;; we have to turn off '%fresh-auto-compile' with that version, so to - ;; avoid ABI breakage in the user's config file, we explicitly compile - ;; it (the problem remains if the user's config is spread on several - ;; modules.) See <https://bugs.gnu.org/29881>. - (unless (string=? (version) "2.2.3") - (set! %fresh-auto-compile #t)) - + (set! %fresh-auto-compile #t) (set! %load-should-auto-compile #t) (save-module-excursion @@ -232,17 +228,12 @@ information, or #f if it could not be found." (parameterize ((current-warning-port (%make-void-port "w"))) (call-with-prompt tag (lambda () - (when (string=? (version) "2.2.3") - (catch 'system-error - (lambda () - (compile-file file #:env user-module)) - (const #f))) ;EACCES maybe, let's interpret it - ;; Give 'load' an absolute file name so that it doesn't try to ;; search for FILE in %LOAD-PATH. Note: use 'load', not ;; 'primitive-load', so that FILE is compiled, which then allows ;; us to provide better error reporting with source line numbers. - (load (canonicalize-path file))) + (without-compiler-optimizations + (load (canonicalize-path file)))) (const #f)))))) (lambda _ ;; XXX: Errors are reported from the pre-unwind handler below, but @@ -376,12 +367,14 @@ ARGS is the list of arguments received by the 'throw' handler." (('system-error . rest) (let ((err (system-error-errno args))) (report-error (G_ "failed to load '~a': ~a~%") file (strerror err)))) - (('read-error "scm_i_lreadparen" message _ ...) + (('read-error _ message args ...) ;; Guile's missing-paren messages are obscure so we make them more ;; intelligible here. - (if (string-suffix? "end of file" message) - (let ((location (string-drop-right message - (string-length "end of file")))) + (if (or (string-suffix? "end of file" message) ;Guile < 3.0.6 + (and (string-contains message "unexpected end of input") + (member '(#\)) args))) + (let ((location (string-take message + (+ 2 (string-contains message ": "))))) (format (current-error-port) (G_ "~amissing closing parenthesis~%") location)) (apply throw args))) @@ -490,12 +483,11 @@ part." (lambda _ (setlocale LC_ALL "")) (lambda args - (display-hint (G_ "Consider installing the @code{glibc-utf8-locales} or -@code{glibc-locales} package and defining @code{GUIX_LOCPATH}, along these -lines: + (display-hint (G_ "Consider installing the @code{glibc-locales} package +and defining @code{GUIX_LOCPATH}, along these lines: @example -guix install glibc-utf8-locales +guix install glibc-locales export GUIX_LOCPATH=\"$HOME/.guix-profile/lib/locale\" @end example @@ -677,22 +669,17 @@ or variants of @code{~a} in the same profile.") or remove one of them from the profile.") name1 name2))))) -(cond-expand - (guile-3 - ;; On Guile 3.0, in 'call-with-error-handling' we need to re-raise. To - ;; preserve useful backtraces in case of unhandled errors, we want that to - ;; happen before the stack has been unwound, hence 'guard*'. - (define-syntax-rule (guard* (var clauses ...) exp ...) - "This variant of SRFI-34 'guard' does not unwind the stack before +;; On Guile 3.0, in 'call-with-error-handling' we need to re-raise. To +;; preserve useful backtraces in case of unhandled errors, we want that to +;; happen before the stack has been unwound, hence 'guard*'. +(define-syntax-rule (guard* (var clauses ...) exp ...) + "This variant of SRFI-34 'guard' does not unwind the stack before evaluating the tests and bodies of CLAUSES." - (with-exception-handler - (lambda (var) - (cond clauses ... (else (raise var)))) - (lambda () exp ...) - #:unwind? #f))) - (else - (define-syntax-rule (guard* (var clauses ...) exp ...) - (guard (var clauses ...) exp ...)))) + (with-exception-handler + (lambda (var) + (cond clauses ... (else (raise var)))) + (lambda () exp ...) + #:unwind? #f)) (define (call-with-error-handling thunk) "Call THUNK within a user-friendly error handler." @@ -825,11 +812,13 @@ directories:~{ ~a~}~%") ;; Furthermore, use of 'guard*' ensures that the stack has not ;; been unwound when we re-raise, since that would otherwise show ;; useless backtraces. - ((cond-expand - (guile-3 - ((exception-predicate &exception-with-kind-and-args) c)) - (else #f)) - (raise c)) + (((exception-predicate &exception-with-kind-and-args) c) + (if (eq? 'system-error (exception-kind c)) ;EPIPE & co. + (match (exception-args c) + ((proc format-string format-args . _) + (leave (G_ "~a: ~a~%") proc + (apply format #f format-string format-args)))) + (raise c))) ((message-condition? c) ;; Normally '&message' error conditions have an i18n'd message. @@ -839,12 +828,7 @@ directories:~{ ~a~}~%") (when (fix-hint? c) (display-hint (condition-fix-hint c))) (exit 1))) - ;; Catch EPIPE and the likes. - (catch 'system-error - thunk - (lambda (key proc format-string format-args . rest) - (leave (G_ "~a: ~a~%") proc - (apply format #f format-string format-args)))))) + (thunk))) (define-syntax-rule (leave-on-EPIPE exp ...) "Run EXP... in a context where EPIPE errors are caught and lead to 'exit' @@ -1906,10 +1890,10 @@ DURATION-RELATION with the current time." (define (equal-entry? first second) (string= (manifest-entry-item first) (manifest-entry-item second))) - (define (display-entry entry prefix) + (define (make-row entry prefix) (match entry (($ <manifest-entry> name version output location _) - (format #t " ~a ~a\t~a\t~a\t~a~%" prefix name version output location)))) + (list (format #f " ~a ~a" prefix name) version output location)))) (define (list-entries number) (manifest-entries (profile-manifest (generation-file-name profile number)))) @@ -1920,8 +1904,8 @@ DURATION-RELATION with the current time." equal-entry? (list-entries new) (list-entries old))) (removed (lset-difference equal-entry? (list-entries old) (list-entries new)))) - (for-each (cut display-entry <> "+") added) - (for-each (cut display-entry <> "-") removed) + (pretty-print-table (append (map (cut make-row <> "+") added) + (map (cut make-row <> "-") removed))) (newline))) (display-diff profile gen1 gen2)) @@ -1949,15 +1933,17 @@ already taken." (define (display-profile-content profile number) "Display the packages in PROFILE, generation NUMBER, in a human-readable way." - (for-each (match-lambda - (($ <manifest-entry> name version output location _) - (format #t " ~a\t~a\t~a\t~a~%" - name version output location))) - ;; Show most recently installed packages last. - (reverse - (manifest-entries - (profile-manifest (generation-file-name profile number)))))) + (define entry->row + (match-lambda + (($ <manifest-entry> name version output location _) + (list (string-append " " name) version output location)))) + + (let* ((manifest (profile-manifest (generation-file-name profile number))) + (entries (manifest-entries manifest)) + (rows (map entry->row entries))) + ;; Show most recently installed packages last. + (pretty-print-table (reverse rows)))) (define (display-generation-change previous current) (format #t (G_ "switched from generation ~a to ~a~%") previous current)) @@ -2156,16 +2142,14 @@ found." (let ((command-main (module-ref module (symbol-append 'guix- command)))) (parameterize ((program-name command)) - ;; Disable canonicalization so we don't don't stat unreasonably. - (with-fluids ((%file-port-name-canonicalization #f)) - (dynamic-wind - (const #f) - (lambda () - (apply command-main args)) - (lambda () - ;; Abuse 'exit-hook' (which is normally meant to be used by the - ;; REPL) to run things like profiling hooks upon completion. - (run-hook exit-hook))))))) + (dynamic-wind + (const #f) + (lambda () + (apply command-main args)) + (lambda () + ;; Abuse 'exit-hook' (which is normally meant to be used by the + ;; REPL) to run things like profiling hooks upon completion. + (run-hook exit-hook)))))) (define (run-guix . args) "Run the 'guix' command defined by command line ARGS. @@ -2177,28 +2161,30 @@ and signal handling have already been set up." ;; number of 'stat' calls per entry in %LOAD-PATH. Shamelessly remove it. (set! %load-extensions '(".scm")) - (match args - (() - (format (current-error-port) - (G_ "guix: missing command name~%")) - (show-guix-usage)) - ((or ("-h") ("--help")) - (leave-on-EPIPE (show-guix-help))) - ((or ("-V") ("--version")) - (show-version-and-exit "guix")) - (((? option? o) args ...) - (format (current-error-port) - (G_ "guix: unrecognized option '~a'~%") o) - (show-guix-usage)) - (("help" command) - (apply run-guix-command (string->symbol command) - '("--help"))) - (("help" args ...) - (leave-on-EPIPE (show-guix-help))) - ((command args ...) - (apply run-guix-command - (string->symbol command) - args)))) + ;; Disable canonicalization so we don't don't stat unreasonably. + (with-fluids ((%file-port-name-canonicalization #f)) + (match args + (() + (format (current-error-port) + (G_ "guix: missing command name~%")) + (show-guix-usage)) + ((or ("-h") ("--help")) + (leave-on-EPIPE (show-guix-help))) + ((or ("-V") ("--version")) + (show-version-and-exit "guix")) + (((? option? o) args ...) + (format (current-error-port) + (G_ "guix: unrecognized option '~a'~%") o) + (show-guix-usage)) + (("help" command) + (apply run-guix-command (string->symbol command) + '("--help"))) + (("help" args ...) + (leave-on-EPIPE (show-guix-help))) + ((command args ...) + (apply run-guix-command + (string->symbol command) + args))))) (define (guix-main arg0 . args) (initialize-guix) diff --git a/guix/utils.scm b/guix/utils.scm index 05af86fc37..2920fa7684 100644 --- a/guix/utils.scm +++ b/guix/utils.scm @@ -10,6 +10,8 @@ ;;; Copyright © 2020 Efraim Flashner <efraim@flashner.co.il> ;;; Copyright © 2021 Simon Tournier <zimon.toutoune@gmail.com> ;;; Copyright © 2021 Chris Marusich <cmmarusich@gmail.com> +;;; Copyright © 2018 Steve Sprang <scs@stevesprang.com> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -123,7 +125,9 @@ canonical-newline-port string-distance - string-closest)) + string-closest + + pretty-print-table)) ;;; @@ -935,6 +939,32 @@ according to THRESHOLD, then #f is returned." #f +inf.0 tests))) + +;;; +;;; Prettified output. +;;; + +(define* (pretty-print-table rows #:key (max-column-width 20)) + "Print ROWS in neat columns. All rows should be lists of strings and each +row should have the same length. The columns are separated by a tab +character, and aligned using spaces. The maximum width of each column is +bound by MAX-COLUMN-WIDTH." + (let* ((number-of-columns-to-pad (if (null? rows) + 0 + (1- (length (first rows))))) + ;; Ignore the last column as it is left aligned and doesn't need + ;; padding; this prevents printing extraneous trailing spaces. + (column-widths (fold (lambda (row maximums) + (map max (map string-length row) maximums)) + ;; Initial max width is 0 for each column. + (make-list number-of-columns-to-pad 0) + (map (cut drop-right <> 1) rows))) + (column-formats (map (cut format #f "~~~da" <>) + (map (cut min <> max-column-width) + column-widths))) + (fmt (string-append (string-join column-formats "\t") "\t~a"))) + (for-each (cut format #t "~?~%" fmt <>) rows))) + ;;; Local Variables: ;;; eval: (put 'call-with-progress-reporter 'scheme-indent-function 1) ;;; End: |