diff options
Diffstat (limited to 'guix')
| -rw-r--r-- | guix/build-system/go.scm | 24 | ||||
| -rw-r--r-- | guix/build/qt-build-system.scm | 93 | ||||
| -rw-r--r-- | guix/channels.scm | 17 | ||||
| -rw-r--r-- | guix/git.scm | 23 | ||||
| -rw-r--r-- | guix/gnu-maintenance.scm | 61 | ||||
| -rw-r--r-- | guix/import/go.scm | 458 | ||||
| -rw-r--r-- | guix/import/utils.scm | 47 | ||||
| -rw-r--r-- | guix/ipfs.scm | 183 | ||||
| -rw-r--r-- | guix/scripts/import.scm | 3 | ||||
| -rw-r--r-- | guix/scripts/import/go.scm | 75 | ||||
| -rwxr-xr-x | guix/scripts/substitute.scm | 183 | ||||
| -rw-r--r-- | guix/scripts/system.scm | 2 |
12 files changed, 803 insertions, 366 deletions
diff --git a/guix/build-system/go.scm b/guix/build-system/go.scm index 0e2c1cd2ee..8f55796e86 100644 --- a/guix/build-system/go.scm +++ b/guix/build-system/go.scm @@ -31,6 +31,7 @@ go-build go-build-system + go-pseudo-version? go-version->git-ref)) ;; Commentary: @@ -40,17 +41,19 @@ ;; ;; Code: -(define %go-version-rx +(define %go-pseudo-version-rx + ;; Match only the end of the version string; this is so that matching the + ;; more complex leading semantic version pattern is not required. (make-regexp (string-append - "(v?[0-9]\\.[0-9]\\.[0-9])" ;"v" prefix can be omitted in version prefix - "(-|-pre\\.0\\.|-0\\.)" ;separator - "([0-9]{14})-" ;timestamp - "([0-9A-Fa-f]{12})"))) ;commit hash + "([0-9]{14}-)" ;timestamp + "([0-9A-Fa-f]{12})" ;commit hash + "(\\+incompatible)?$"))) ;optional +incompatible tag (define (go-version->git-ref version) "Parse VERSION, a \"pseudo-version\" as defined at <https://golang.org/ref/mod#pseudo-versions>, and extract the commit hash from -it, defaulting to full VERSION if a pseudo-version pattern is not recognized." +it, defaulting to full VERSION (stripped from the \"+incompatible\" suffix if +present) if a pseudo-version pattern is not recognized." ;; A module version like v1.2.3 is introduced by tagging a revision in the ;; underlying source repository. Untagged revisions can be referred to ;; using a "pseudo-version" like v0.0.0-yyyymmddhhmmss-abcdefabcdef, where @@ -65,11 +68,16 @@ it, defaulting to full VERSION if a pseudo-version pattern is not recognized." (if (string-suffix? "+incompatible" version) (string-drop-right version 13) version)) - (match (regexp-exec %go-version-rx version))) + (match (regexp-exec %go-pseudo-version-rx version))) (if match - (match:substring match 4) + (match:substring match 2) version))) +(define (go-pseudo-version? version) + "True if VERSION is a Go pseudo-version, i.e., a version string made of a +commit hash and its date rather than a proper release tag." + (regexp-exec %go-pseudo-version-rx version)) + (define %go-build-system-modules ;; Build-side modules imported and used by default. `((guix build go-build-system) diff --git a/guix/build/qt-build-system.scm b/guix/build/qt-build-system.scm index a6955ce4c2..b0d6ddafac 100644 --- a/guix/build/qt-build-system.scm +++ b/guix/build/qt-build-system.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2014 Federico Beffa <beffa@fbengineering.ch> -;;; Copyright © 2014, 2015 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2014, 2015, 2021 Ludovic Courtès <ludo@gnu.org> ;;; Copyright © 2018 Mark H Weaver <mhw@netris.org> ;;; Copyright © 2019, 2020, 2021 Hartmut Goebel <h.goebel@crazy-compilers.com> ;;; @@ -48,6 +48,97 @@ (setenv "CTEST_OUTPUT_ON_FAILURE" "1") #t) +(define (variables-for-wrapping base-directories) + + (define (collect-sub-dirs base-directories file-type subdirectory + selectors) + ;; Append SUBDIRECTORY and each of BASE-DIRECTORIES, and return the subset + ;; that exists and has at least one of the SELECTORS sub-directories, + ;; unless SELECTORS is the empty list. FILE-TYPE should by 'directory or + ;; 'regular file. For the later, it allows searching for plain files + ;; rather than directories. + (define exists? (match file-type + ('directory directory-exists?) + ('regular file-exists?))) + + (filter-map (lambda (dir) + (let ((directory (string-append dir subdirectory))) + (and (exists? directory) + (or (null? selectors) + (any (lambda (selector) + (exists? + (string-append directory selector))) + selectors)) + directory))) + base-directories)) + + (filter-map + (match-lambda + ((variable file-type directory selectors ...) + (match (collect-sub-dirs base-directories file-type directory + selectors) + (() + #f) + (directories + `(,variable = ,directories))))) + + ;; These shall match the search-path-specification for Qt and KDE + ;; libraries. + (list '("XDG_DATA_DIRS" directory "/share" + + ;; These are "selectors": consider /share if and only if at least + ;; one of these sub-directories exist. This avoids adding + ;; irrelevant packages to XDG_DATA_DIRS just because they have a + ;; /share sub-directory. + "/glib-2.0/schemas" "/sounds" "/themes" + "/cursors" "/wallpapers" "/icons" "/mime") + '("XDG_CONFIG_DIRS" directory "/etc/xdg") + '("QT_PLUGIN_PATH" directory "/lib/qt5/plugins") + '("QML2_IMPORT_PATH" directory "/lib/qt5/qml") + '("QTWEBENGINEPROCESS_PATH" regular + "/lib/qt5/libexec/QtWebEngineProcess")))) + +(define* (wrap-all-programs #:key inputs outputs + (qt-wrap-excluded-outputs '()) + #:allow-other-keys) + "Implement phase \"qt-wrap\": look for GSettings schemas and +gtk+-v.0 libraries and create wrappers with suitably set environment variables +if found. + +Wrapping is not applied to outputs whose name is listed in +QT-WRAP-EXCLUDED-OUTPUTS. This is useful when an output is known not +to contain any Qt binaries, and where wrapping would gratuitously +add a dependency of that output on Qt." + (define (find-files-to-wrap directory) + (append-map + (lambda (dir) + (if (directory-exists? dir) (find-files dir ".*") (list))) + (list (string-append directory "/bin") + (string-append directory "/sbin") + (string-append directory "/libexec") + (string-append directory "/lib/libexec")))) + + (define input-directories + ;; FIXME: Filter out unwanted inputs, e.g. cmake + (match inputs + (((_ . dir) ...) + dir))) + + (define handle-output + (match-lambda + ((output . directory) + (unless (member output qt-wrap-excluded-outputs) + (let ((bin-list (find-files-to-wrap directory)) + (vars-to-wrap (variables-for-wrapping + (append (list directory) + input-directories)))) + (when (not (null? vars-to-wrap)) + (for-each (cut apply wrap-program <> vars-to-wrap) + bin-list))))))) + + (for-each handle-output outputs) + #t) + (define %standard-phases (modify-phases cmake:%standard-phases (add-before 'check 'check-setup check-setup) diff --git a/guix/channels.scm b/guix/channels.scm index b812c1b6e5..c40fc0c507 100644 --- a/guix/channels.scm +++ b/guix/channels.scm @@ -597,9 +597,24 @@ to '%package-module-path'." (string-append #$output "/share/guile/site/" (effective-version))) + (define optimizations-for-level + ;; Guile 3.0 provides this procedure but Guile 2.2 didn't. + ;; Since this code may be executed by either version, we can't + ;; rely on its availability. + (or (and=> (false-if-exception + (resolve-interface '(system base optimize))) + (lambda (iface) + (module-ref iface 'optimizations-for-level))) + (const '()))) + + (define -O1 + ;; Optimize for package module compilation speed. + (optimizations-for-level 1)) + (let* ((subdir #$directory) (source (string-append #$source subdir))) - (compile-files source go (find-files source "\\.scm$")) + (compile-files source go (find-files source "\\.scm$") + #:optimization-options (const -O1)) (mkdir-p (dirname scm)) (symlink (string-append #$source subdir) scm)) diff --git a/guix/git.scm b/guix/git.scm index 1820036f25..776b03f33a 100644 --- a/guix/git.scm +++ b/guix/git.scm @@ -1,6 +1,7 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017, 2020 Mathieu Othacehe <m.othacehe@gmail.com> ;;; Copyright © 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> +;;; Copyright © 2021 Kyle Meyer <kyle@kyleam.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -209,6 +210,9 @@ corresponding Git object." (let ((oid (reference-target (branch-lookup repository branch BRANCH-REMOTE)))) (object-lookup repository oid))) + (('symref . symref) + (let ((oid (reference-name->oid repository symref))) + (object-lookup repository oid))) (('commit . commit) (let ((len (string-length commit))) ;; 'object-lookup-prefix' appeared in Guile-Git in Mar. 2018, so we @@ -340,7 +344,7 @@ definitely available in REPOSITORY, false otherwise." (define* (update-cached-checkout url #:key - (ref '(branch . "master")) + (ref '()) recursive? (check-out? #t) starting-commit @@ -356,6 +360,7 @@ provided) as returned by 'commit-relation'. REF is pair whose key is [branch | commit | tag | tag-or-commit ] and value the associated data: [<branch name> | <sha1> | <tag name> | <string>]. +If REF is the empty list, the remote HEAD is used. When RECURSIVE? is true, check out submodules as well, if any. @@ -374,6 +379,7 @@ it unchanged." ;; made little sense since the cache should be transparent to them. So ;; here we append "origin/" if it's missing and otherwise keep it. (match ref + (() '(symref . "refs/remotes/origin/HEAD")) (('branch . branch) `(branch . ,(if (string-prefix? "origin/" branch) branch @@ -433,12 +439,13 @@ it unchanged." (log-port (%make-void-port "w")) (cache-directory (%repository-cache-directory)) - (ref '(branch . "master"))) + (ref '())) "Return two values: the content of the git repository at URL copied into a store directory and the sha1 of the top level commit in this directory. The reference to be checkout, once the repository is fetched, is specified by REF. REF is pair whose key is [branch | commit | tag] and value the associated -data, respectively [<branch name> | <sha1> | <tag name>]. +data, respectively [<branch name> | <sha1> | <tag name>]. If REF is the empty +list, the remote HEAD is used. When RECURSIVE? is true, check out submodules as well, if any. @@ -548,7 +555,7 @@ objects: 'ancestor (meaning that OLD is an ancestor of NEW), 'descendant, or git-checkout make-git-checkout git-checkout? (url git-checkout-url) - (branch git-checkout-branch (default "master")) + (branch git-checkout-branch (default #f)) (commit git-checkout-commit (default #f)) ;#f | tag | commit (recursive? git-checkout-recursive? (default #f))) @@ -587,9 +594,11 @@ objects: 'ancestor (meaning that OLD is an ancestor of NEW), 'descendant, or (match checkout (($ <git-checkout> url branch commit recursive?) (latest-repository-commit* url - #:ref (if commit - `(tag-or-commit . ,commit) - `(branch . ,branch)) + #:ref (cond (commit + `(tag-or-commit . ,commit)) + (branch + `(branch . ,branch)) + (else '())) #:recursive? recursive? #:log-port (current-error-port))))) diff --git a/guix/gnu-maintenance.scm b/guix/gnu-maintenance.scm index ba659c0a60..fece84b341 100644 --- a/guix/gnu-maintenance.scm +++ b/guix/gnu-maintenance.scm @@ -31,7 +31,7 @@ #:use-module (srfi srfi-34) #:use-module (rnrs io ports) #:use-module (system foreign) - #:use-module (guix http-client) + #:use-module ((guix http-client) #:hide (open-socket-for-uri)) #:use-module (guix ftp-client) #:use-module (guix utils) #:use-module (guix memoization) @@ -669,10 +669,10 @@ GNOME packages; EMMS is included though, because its releases are on gnu.org." #:host (uri-host uri) #:path (string-append (uri-path uri) extension))) - (define (valid-uri? uri) + (define (valid-uri? uri port) ;; Return true if URI is reachable. (false-if-exception - (case (response-code (http-head uri)) + (case (response-code (http-head uri #:port port #:keep-alive? #t)) ((200 302) #t) (else #f)))) @@ -680,30 +680,39 @@ GNOME packages; EMMS is included though, because its releases are on gnu.org." (base (string-append "https://sourceforge.net/projects/" name "/files")) (url (string-append base "/latest/download")) - (response (false-if-exception (http-head url)))) - (and response - (= 302 (response-code response)) - (response-location response) - (match (string-tokenize (uri-path (response-location response)) - (char-set-complement (char-set #\/))) - ((_ components ...) - (let* ((path (string-join components "/")) - (url (string-append "mirror://sourceforge/" path))) - (and (release-file? name (basename path)) + (uri (string->uri url)) + (port (false-if-exception (open-socket-for-uri uri))) + (response (and port + (http-head uri #:port port #:keep-alive? #t)))) + (dynamic-wind + (const #t) + (lambda () + (and response + (= 302 (response-code response)) + (response-location response) + (match (string-tokenize (uri-path (response-location response)) + (char-set-complement (char-set #\/))) + ((_ components ...) + (let* ((path (string-join components "/")) + (url (string-append "mirror://sourceforge/" path))) + (and (release-file? name (basename path)) - ;; Take the heavy-handed approach of probing 3 additional - ;; URLs. XXX: Would be nicer if this could be avoided. - (let* ((loc (response-location response)) - (sig (any (lambda (extension) - (let ((uri (uri-append loc extension))) - (and (valid-uri? uri) - (string-append url extension)))) - '(".asc" ".sig" ".sign")))) - (upstream-source - (package name) - (version (tarball->version (basename path))) - (urls (list url)) - (signature-urls (and sig (list sig)))))))))))) + ;; Take the heavy-handed approach of probing 3 additional + ;; URLs. XXX: Would be nicer if this could be avoided. + (let* ((loc (response-location response)) + (sig (any (lambda (extension) + (let ((uri (uri-append loc extension))) + (and (valid-uri? uri port) + (string-append url extension)))) + '(".asc" ".sig" ".sign")))) + (upstream-source + (package name) + (version (tarball->version (basename path))) + (urls (list url)) + (signature-urls (and sig (list sig))))))))))) + (lambda () + (when port + (close-port port)))))) (define (latest-xorg-release package) "Return the latest release of PACKAGE." diff --git a/guix/import/go.scm b/guix/import/go.scm index 6c0231e113..bc53f8f558 100644 --- a/guix/import/go.scm +++ b/guix/import/go.scm @@ -33,7 +33,7 @@ #:use-module (guix http-client) #:use-module ((guix licenses) #:prefix license:) #:use-module (guix memoization) - #:autoload (htmlprag) (html->sxml) ;from Guile-Lib + #:use-module (htmlprag) ;from Guile-Lib #:autoload (guix git) (update-cached-checkout) #:autoload (gcrypt hash) (open-hash-port hash-algorithm sha256) #:autoload (guix serialization) (write-file) @@ -43,20 +43,29 @@ #:use-module (ice-9 rdelim) #:use-module (ice-9 receive) #:use-module (ice-9 regex) + #:use-module (ice-9 textual-ports) #:use-module ((rnrs io ports) #:select (call-with-port)) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-2) #:use-module (srfi srfi-9) #:use-module (srfi srfi-11) #:use-module (srfi srfi-26) - #:use-module (sxml xpath) + #:use-module (srfi srfi-34) + #:use-module (sxml match) + #:use-module ((sxml xpath) #:renamer (lambda (s) + (if (eq? 'filter s) + 'xfilter + s))) #:use-module (web client) #:use-module (web response) #:use-module (web uri) - #:export (go-path-escape - go-module->guix-package + #:export (go-module->guix-package go-module-recursive-import)) +;;; Parameterize htmlprag to parse valid HTML more reliably. +(%strict-tokenizer? #t) + ;;; Commentary: ;;; ;;; (guix import go) attempts to make it easier to create Guix package @@ -84,12 +93,18 @@ ;;; assumption that there will be no collision. ;;; TODO list -;;; - get correct hash in vcs->origin -;;; - print partial result during recursive imports (need to catch -;;; exceptions) +;;; - get correct hash in vcs->origin for Mercurial and Subversion ;;; Code: +(define http-fetch* + ;; Like http-fetch, but memoized and returning the body as a string. + (memoize (lambda args + (call-with-port (apply http-fetch args) get-string-all)))) + +(define json-fetch* + (memoize json-fetch)) + (define (go-path-escape path) "Escape a module path by replacing every uppercase letter with an exclamation mark followed with its lowercase equivalent, as per the module @@ -99,54 +114,87 @@ https://godoc.org/golang.org/x/mod/module#hdr-Escaped_Paths)." (string-append "!" (string-downcase (match:substring occurrence)))) (regexp-substitute/global #f "[A-Z]" path 'pre escape 'post)) -(define (go-module-latest-version goproxy-url module-path) - "Fetch the version number of the latest version for MODULE-PATH from the -given GOPROXY-URL server." - (assoc-ref (json-fetch (format #f "~a/~a/@latest" goproxy-url - (go-path-escape module-path))) - "Version")) +;; Prevent inlining of this procedure, which is accessed by unit tests. +(set! go-path-escape go-path-escape) + +(define (go.pkg.dev-info name) + (http-fetch* (string-append "https://pkg.go.dev/" name))) + +(define* (go-module-version-string goproxy name #:key version) + "Fetch the version string of the latest version for NAME from the given +GOPROXY server, or for VERSION when specified." + (let ((file (if version + (string-append "@v/" version ".info") + "@latest"))) + (assoc-ref (json-fetch* (format #f "~a/~a/~a" + goproxy (go-path-escape name) file)) + "Version"))) +(define* (go-module-available-versions goproxy name) + "Retrieve the available versions for a given module from the module proxy. +Versions are being returned **unordered** and may contain different versioning +styles for the same package." + (let* ((url (string-append goproxy "/" (go-path-escape name) "/@v/list")) + (body (http-fetch* url)) + (versions (remove string-null? (string-split body #\newline)))) + (if (null? versions) + (list (go-module-version-string goproxy name)) ;latest version + versions))) (define (go-package-licenses name) "Retrieve the list of licenses that apply to NAME, a Go package or module -name (e.g. \"github.com/golang/protobuf/proto\"). The data is scraped from -the https://pkg.go.dev/ web site." - (let*-values (((url) (string-append "https://pkg.go.dev/" name - "?tab=licenses")) - ((response body) (http-get url)) - ;; Extract the text contained in a h2 child node of any - ;; element marked with a "License" class attribute. - ((select) (sxpath `(// (* (@ (equal? (class "License")))) - h2 // *text*)))) - (and (eq? (response-code response) 200) - (match (select (html->sxml body)) - (() #f) ;nothing selected - (licenses licenses))))) +name (e.g. \"github.com/golang/protobuf/proto\")." + (let* ((body (go.pkg.dev-info (string-append name "?tab=licenses"))) + ;; Extract the text contained in a h2 child node of any + ;; element marked with a "License" class attribute. + (select (sxpath `(// (* (@ (equal? (class "License")))) + h2 // *text*)))) + (select (html->sxml body)))) -(define (go.pkg.dev-info name) - (http-get (string-append "https://pkg.go.dev/" name))) -(define go.pkg.dev-info* - (memoize go.pkg.dev-info)) +(define (sxml->texi sxml-node) + "A very basic SXML to Texinfo converter which attempts to preserve HTML +formatting and links as text." + (sxml-match sxml-node + ((strong ,text) + (format #f "@strong{~a}" text)) + ((a (@ (href ,url)) ,text) + (format #f "@url{~a,~a}" url text)) + ((code ,text) + (format #f "@code{~a}" text)) + (,something-else something-else))) (define (go-package-description name) "Retrieve a short description for NAME, a Go package name, -e.g. \"google.golang.org/protobuf/proto\". The data is scraped from the -https://pkg.go.dev/ web site." - (let*-values (((response body) (go.pkg.dev-info* name)) - ;; Extract the text contained in a h2 child node of any - ;; element marked with a "License" class attribute. - ((select) (sxpath - `(// (section - (@ (equal? (class "Documentation-overview")))) - (p 1))))) - (and (eq? (response-code response) 200) - (match (select (html->sxml body)) - (() #f) ;nothing selected - (((p . strings)) - ;; The paragraph text is returned as a list of strings embedding - ;; newline characters. Join them and strip the newline - ;; characters. - (string-delete #\newline (string-join strings))))))) +e.g. \"google.golang.org/protobuf/proto\"." + (let* ((body (go.pkg.dev-info name)) + (sxml (html->sxml body)) + (overview ((sxpath + `(// + (* (@ (equal? (class "Documentation-overview")))) + (p 1))) sxml)) + ;; Sometimes, the first paragraph just contains images/links that + ;; has only "\n" for text. The following filter is designed to + ;; omit it. + (contains-text? (lambda (node) + (remove string-null? + (map string-trim-both + (filter (node-typeof? '*text*) + (cdr node)))))) + (select-content (sxpath + `(// + (* (@ (equal? (class "UnitReadme-content")))) + div // p ,(xfilter contains-text?)))) + ;; Fall-back to use content; this is less desirable as it is more + ;; verbose, but not every page has an overview. + (description (if (not (null? overview)) + overview + (select-content sxml))) + (description* (and (not (null? description)) + (first description)))) + (match description* + (() #f) ;nothing selected + ((p elements ...) + (apply string-append (filter string? (map sxml->texi elements))))))) (define (go-package-synopsis module-name) "Retrieve a short synopsis for a Go module named MODULE-NAME, @@ -154,17 +202,17 @@ e.g. \"google.golang.org/protobuf\". The data is scraped from the https://pkg.go.dev/ web site." ;; Note: Only the *module* (rather than package) page has the README title ;; used as a synopsis on the https://pkg.go.dev web site. - (let*-values (((response body) (go.pkg.dev-info* module-name)) - ;; Extract the text contained in a h2 child node of any - ;; element marked with a "License" class attribute. - ((select) (sxpath - `(// (div (@ (equal? (class "UnitReadme-content")))) - // h3 *text*)))) - (and (eq? (response-code response) 200) - (match (select (html->sxml body)) - (() #f) ;nothing selected - ((title more ...) ;title is the first string of the list - (string-trim-both title)))))) + (let* ((url (string-append "https://pkg.go.dev/" module-name)) + (body (http-fetch* url)) + ;; Extract the text contained in a h2 child node of any + ;; element marked with a "License" class attribute. + (select-title (sxpath + `(// (div (@ (equal? (class "UnitReadme-content")))) + // h3 *text*)))) + (match (select-title (html->sxml body)) + (() #f) ;nothing selected + ((title more ...) ;title is the first string of the list + (string-trim-both title))))) (define (list->licenses licenses) "Given a list of LICENSES mostly following the SPDX conventions, return the @@ -189,13 +237,13 @@ corresponding Guix license or 'unknown-license!" 'unknown-license!))) licenses)) -(define (fetch-go.mod goproxy-url module-path version) - "Fetches go.mod from the given GOPROXY-URL server for the given MODULE-PATH -and VERSION." - (let ((url (format #f "~a/~a/@v/~a.mod" goproxy-url +(define (fetch-go.mod goproxy module-path version) + "Fetch go.mod from the given GOPROXY server for the given MODULE-PATH +and VERSION and return an input port." + (let ((url (format #f "~a/~a/@v/~a.mod" goproxy (go-path-escape module-path) (go-path-escape version)))) - (http-fetch url))) + (http-fetch* url))) (define %go.mod-require-directive-rx ;; A line in a require directive is composed of a module path and @@ -203,118 +251,119 @@ and VERSION." ;; the end. (make-regexp (string-append - "^[[:blank:]]*" - "([^[:blank:]]+)[[:blank:]]+([^[:blank:]]+)" - "([[:blank:]]+//.*)?"))) + "^[[:blank:]]*([^[:blank:]]+)[[:blank:]]+" ;the module path + "([^[:blank:]]+)" ;the version + "([[:blank:]]+//.*)?"))) ;an optional comment (define %go.mod-replace-directive-rx ;; ReplaceSpec = ModulePath [ Version ] "=>" FilePath newline ;; | ModulePath [ Version ] "=>" ModulePath Version newline . (make-regexp (string-append - "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?" - "[[:blank:]]+" "=>" "[[:blank:]]+" - "([^[:blank:]]+)([[:blank:]]+([^[:blank:]]+))?"))) + "([^[:blank:]]+)" ;the module path + "([[:blank:]]+([^[:blank:]]+))?" ;optional version + "[[:blank:]]+=>[[:blank:]]+" + "([^[:blank:]]+)" ;the file or module path + "([[:blank:]]+([^[:blank:]]+))?"))) ;the version (if a module path) -(define (parse-go.mod port) - "Parse the go.mod file accessible via the input PORT, returning a list of -requirements." - (define-record-type <results> - (make-results requirements replacements) - results? - (requirements results-requirements) - (replacements results-replacements)) +(define (parse-go.mod content) + "Parse the go.mod file CONTENT, returning a list of requirements." ;; We parse only a subset of https://golang.org/ref/mod#go-mod-file-grammar ;; which we think necessary for our use case. - (define (toplevel results) - "Main parser, RESULTS is a pair of alist serving as accumulator for - all encountered requirements and replacements." - (let ((line (read-line port))) + (define (toplevel requirements replaced) + "This is the main parser. The results are accumulated in THE REQUIREMENTS +and REPLACED lists." + (let ((line (read-line))) (cond ((eof-object? line) ;; parsing ended, give back the result - results) + (values requirements replaced)) ((string=? line "require (") ;; a require block begins, delegate parsing to IN-REQUIRE - (in-require results)) + (in-require requirements replaced)) ((string=? line "replace (") ;; a replace block begins, delegate parsing to IN-REPLACE - (in-replace results)) + (in-replace requirements replaced)) ((string-prefix? "require " line) - ;; a standalone require directive - (let* ((stripped-line (string-drop line 8)) - (new-results (require-directive results stripped-line))) - (toplevel new-results))) + ;; a require directive by itself + (let* ((stripped-line (string-drop line 8))) + (call-with-values + (lambda () + (require-directive requirements replaced stripped-line)) + toplevel))) ((string-prefix? "replace " line) - ;; a standalone replace directive - (let* ((stripped-line (string-drop line 8)) - (new-results (replace-directive results stripped-line))) - (toplevel new-results))) + ;; a replace directive by itself + (let* ((stripped-line (string-drop line 8))) + (call-with-values + (lambda () + (replace-directive requirements replaced stripped-line)) + toplevel))) (#t ;; unrecognised line, ignore silently - (toplevel results))))) + (toplevel requirements replaced))))) - (define (in-require results) - (let ((line (read-line port))) + (define (in-require requirements replaced) + (let ((line (read-line))) (cond ((eof-object? line) ;; this should never happen here but we ignore silently - results) + (values requirements replaced)) ((string=? line ")") ;; end of block, coming back to toplevel - (toplevel results)) + (toplevel requirements replaced)) (#t - (in-require (require-directive results line)))))) + (call-with-values (lambda () + (require-directive requirements replaced line)) + in-require))))) - (define (in-replace results) - (let ((line (read-line port))) + (define (in-replace requirements replaced) + (let ((line (read-line))) (cond ((eof-object? line) ;; this should never happen here but we ignore silently - results) + (values requirements replaced)) ((string=? line ")") ;; end of block, coming back to toplevel - (toplevel results)) + (toplevel requirements replaced)) (#t - (in-replace (replace-directive results line)))))) + (call-with-values (lambda () + (replace-directive requirements replaced line)) + in-replace))))) + + (define (replace-directive requirements replaced line) + "Extract replaced modules and new requirements from the replace directive +in LINE and add them to the REQUIREMENTS and REPLACED lists." + (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line)) + (module-path (match:substring rx-match 1)) + (version (match:substring rx-match 3)) + (new-module-path (match:substring rx-match 4)) + (new-version (match:substring rx-match 6)) + (new-replaced (cons (list module-path version) replaced)) + (new-requirements + (if (string-match "^\\.?\\./" new-module-path) + requirements + (cons (list new-module-path new-version) requirements)))) + (values new-requirements new-replaced))) - (define (replace-directive results line) - "Extract replaced modules and new requirements from replace directive - in LINE and add to RESULTS." - (match results - (($ <results> requirements replaced) - (let* ((rx-match (regexp-exec %go.mod-replace-directive-rx line)) - (module-path (match:substring rx-match 1)) - (version (match:substring rx-match 3)) - (new-module-path (match:substring rx-match 4)) - (new-version (match:substring rx-match 6)) - (new-replaced (alist-cons module-path version replaced)) - (new-requirements - (if (string-match "^\\.?\\./" new-module-path) - requirements - (alist-cons new-module-path new-version requirements)))) - (make-results new-requirements new-replaced))))) - (define (require-directive results line) - "Extract requirement from LINE and add it to RESULTS." + (define (require-directive requirements replaced line) + "Extract requirement from LINE and augment the REQUIREMENTS and REPLACED +lists." (let* ((rx-match (regexp-exec %go.mod-require-directive-rx line)) (module-path (match:substring rx-match 1)) - ;; we saw double-quoted string in the wild without escape - ;; sequences so we just trim the quotes + ;; Double-quoted strings were seen in the wild without escape + ;; sequences; trim the quotes to be on the safe side. (module-path (string-trim-both module-path #\")) (version (match:substring rx-match 2))) - (match results - (($ <results> requirements replaced) - (make-results (alist-cons module-path version requirements) replaced))))) + (values (cons (list module-path version) requirements) replaced))) - (let ((results (toplevel (make-results '() '())))) - (match results - (($ <results> requirements replaced) - ;; At last we remove replaced modules from the requirements list - (fold - (lambda (replacedelem requirements) - (alist-delete! (car replacedelem) requirements)) - requirements - replaced))))) + (with-input-from-string content + (lambda () + (receive (requirements replaced) + (toplevel '() '()) + ;; At last remove the replaced modules from the requirements list. + (remove (lambda (r) + (assoc (car r) replaced)) + requirements))))) ;; Prevent inlining of this procedure, which is accessed by unit tests. (set! parse-go.mod parse-go.mod) @@ -325,8 +374,10 @@ requirements." (url-prefix vcs-url-prefix) (root-regex vcs-root-regex) (type vcs-type)) + (define (make-vcs prefix regexp type) - (%make-vcs prefix (make-regexp regexp) type)) + (%make-vcs prefix (make-regexp regexp) type)) + (define known-vcs ;; See the following URL for the official Go equivalent: ;; https://github.com/golang/go/blob/846dce9d05f19a1f53465e62a304dea21b99f910/src/cmd/go/internal/vcs/vcs.go#L1026-L1087 @@ -377,15 +428,27 @@ hence the need to derive this information." (vcs-qualified-module-path->root-repo-url module-path) module-path)) -(define (go-module->guix-package-name module-path) - "Converts a module's path to the canonical Guix format for Go packages." - (string-downcase (string-append "go-" (string-replace-substring - (string-replace-substring - (string-replace-substring - module-path - "." "-") - "/" "-") - "_" "-")))) +(define* (go-module->guix-package-name module-path #:optional version) + "Converts a module's path to the canonical Guix format for Go packages. +Optionally include a VERSION string to append to the name." + ;; Map dot, slash and underscore characters to hyphens. + (let ((module-path* (string-map (lambda (c) + (if (member c '(#\. #\/ #\_)) + #\- + c)) + module-path))) + (string-downcase (string-append "go-" module-path* + (if version + (string-append "-" version) + ""))))) + +(define (strip-.git-suffix/maybe repo-url) + "Strip a repository URL '.git' suffix from REPO-URL if hosted at GitHub." + (match repo-url + ((and (? (cut string-prefix? "https://github.com" <>)) + (? (cut string-suffix? ".git" <>))) + (string-drop-right repo-url 4)) + (_ repo-url))) (define-record-type <module-meta> (make-module-meta import-prefix vcs repo-root) @@ -399,21 +462,22 @@ hence the need to derive this information." because goproxy servers don't currently provide all the information needed to build a package." ;; <meta name="go-import" content="import-prefix vcs repo-root"> - (let* ((port (http-fetch (format #f "https://~a?go-get=1" module-path))) + (let* ((meta-data (http-fetch* (format #f "https://~a?go-get=1" module-path))) (select (sxpath `(// head (meta (@ (equal? (name "go-import")))) // content)))) - (match (select (call-with-port port html->sxml)) - (() #f) ;nothing selected + (match (select (html->sxml meta-data)) + (() #f) ;nothing selected (((content content-text)) (match (string-split content-text #\space) ((root-path vcs repo-url) - (make-module-meta root-path (string->symbol vcs) repo-url))))))) + (make-module-meta root-path (string->symbol vcs) + (strip-.git-suffix/maybe repo-url)))))))) -(define (module-meta-data-repo-url meta-data goproxy-url) +(define (module-meta-data-repo-url meta-data goproxy) "Return the URL where the fetcher which will be used can download the source." (if (member (module-meta-vcs meta-data) '(fossil mod)) - goproxy-url + goproxy (module-meta-repo-root meta-data))) ;; XXX: Copied from (guix scripts hash). @@ -466,6 +530,9 @@ control system is being used." (method git-fetch) (uri (git-reference (url ,vcs-repo-url) + ;; This is done because the version field of the package, + ;; which the generated quoted expression refers to, has been + ;; stripped of any 'v' prefixed. (commit ,(if (and plain-version? v-prefixed?) '(string-append "v" version) '(go-version->git-ref version))))) @@ -503,48 +570,95 @@ control system is being used." vcs-type vcs-repo-url))))) (define* (go-module->guix-package module-path #:key - (goproxy-url "https://proxy.golang.org")) - (let* ((latest-version (go-module-latest-version goproxy-url module-path)) - (port (fetch-go.mod goproxy-url module-path latest-version)) - (dependencies (map car (call-with-port port parse-go.mod))) + (goproxy "https://proxy.golang.org") + version + pin-versions?) + "Return the package S-expression corresponding to MODULE-PATH at VERSION, a Go package. +The meta-data is fetched from the GOPROXY server and https://pkg.go.dev/. +When VERSION is unspecified, the latest version available is used." + (let* ((available-versions (go-module-available-versions goproxy module-path)) + (version* (or version + (go-module-version-string goproxy module-path))) ;latest + ;; Elide the "v" prefix Go uses. + (strip-v-prefix (cut string-trim <> #\v)) + ;; Pseudo-versions do not appear in the versions list; skip the + ;; following check. + (_ (unless (or (go-pseudo-version? version*) + (member version* available-versions)) + (error (format #f "error: version ~s is not available +hint: use one of the following available versions ~a\n" + version* available-versions)))) + (content (fetch-go.mod goproxy module-path version*)) + (dependencies+versions (parse-go.mod content)) + (dependencies (if pin-versions? + dependencies+versions + (map car dependencies+versions))) (guix-name (go-module->guix-package-name module-path)) (root-module-path (module-path->repository-root module-path)) ;; The VCS type and URL are not included in goproxy information. For ;; this we need to fetch it from the official module page. (meta-data (fetch-module-meta-data root-module-path)) (vcs-type (module-meta-vcs meta-data)) - (vcs-repo-url (module-meta-data-repo-url meta-data goproxy-url)) + (vcs-repo-url (module-meta-data-repo-url meta-data goproxy)) (synopsis (go-package-synopsis root-module-path)) (description (go-package-description module-path)) (licenses (go-package-licenses module-path))) (values `(package (name ,guix-name) - ;; Elide the "v" prefix Go uses - (version ,(string-trim latest-version #\v)) + (version ,(strip-v-prefix version*)) (source - ,(vcs->origin vcs-type vcs-repo-url latest-version)) + ,(vcs->origin vcs-type vcs-repo-url version*)) (build-system go-build-system) (arguments '(#:import-path ,root-module-path)) - ,@(maybe-inputs (map go-module->guix-package-name dependencies)) + ,@(maybe-propagated-inputs + (map (match-lambda + ((name version) + (go-module->guix-package-name name (strip-v-prefix version))) + (name + (go-module->guix-package-name name))) + dependencies)) (home-page ,(format #f "https://~a" root-module-path)) (synopsis ,synopsis) - (description ,description) - (license ,(match (and=> licenses list->licenses) - ((license) license) - ((licenses ...) `(list ,@licenses)) - (x x)))) - dependencies))) + (description ,(and=> description beautify-description)) + (license ,(match (list->licenses licenses) + (() #f) ;unknown license + ((license) ;a single license + license) + ((license ...) ;a list of licenses + `(list ,@license))))) + (if pin-versions? + dependencies+versions + dependencies)))) (define go-module->guix-package* (memoize go-module->guix-package)) (define* (go-module-recursive-import package-name - #:key (goproxy-url "https://proxy.golang.org")) + #:key (goproxy "https://proxy.golang.org") + version + pin-versions?) + (recursive-import package-name - #:repo->guix-package (lambda* (name . _) - (go-module->guix-package* - name - #:goproxy-url goproxy-url)) - #:guix-name go-module->guix-package-name)) + #:repo->guix-package + (lambda* (name #:key version repo) + ;; Disable output buffering so that the following warning gets printed + ;; consistently. + (setvbuf (current-error-port) 'none) + (guard (c ((http-get-error? c) + (warning (G_ "Failed to import package ~s. +reason: ~s could not be fetched: HTTP error ~a (~s). +This package and its dependencies won't be imported.~%") + name + (uri->string (http-get-error-uri c)) + (http-get-error-code c) + (http-get-error-reason c)) + (values '() '()))) + (receive (package-sexp dependencies) + (go-module->guix-package* name #:goproxy goproxy + #:version version + #:pin-versions? pin-versions?) + (values package-sexp dependencies)))) + #:guix-name go-module->guix-package-name + #:version version)) diff --git a/guix/import/utils.scm b/guix/import/utils.scm index 6b85b3aa1d..d817318a91 100644 --- a/guix/import/utils.scm +++ b/guix/import/utils.scm @@ -7,6 +7,7 @@ ;;; Copyright © 2019 Robert Vollmert <rob@vllmrt.net> ;;; Copyright © 2020 Helio Machado <0x2b3bfa0+guix@googlemail.com> ;;; Copyright © 2020 Martin Becze <mjbecze@riseup.net> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -57,6 +58,7 @@ package-names->package-inputs maybe-inputs maybe-native-inputs + maybe-propagated-inputs package->definition spdx-string->license @@ -247,27 +249,34 @@ use in an 'inputs' field of a package definition." (input (make-input input #f))) names)) -(define* (maybe-inputs package-names #:optional (output #f)) +(define* (maybe-inputs package-names #:optional (output #f) + #:key (type #f)) "Given a list of PACKAGE-NAMES, tries to generate the 'inputs' field of a -package definition." - (match (package-names->package-inputs package-names output) - (() - '()) - ((package-inputs ...) - `((inputs (,'quasiquote ,package-inputs)))))) +package definition. TYPE can be used to specify the type of the inputs; +either the 'native or 'propagated symbols are accepted. Left unspecified, the +snippet generated is for regular inputs." + (let ((field-name (match type + ('native 'native-inputs) + ('propagated 'propagated-inputs) + (_ 'inputs)))) + (match (package-names->package-inputs package-names output) + (() + '()) + ((package-inputs ...) + `((,field-name (,'quasiquote ,package-inputs))))))) (define* (maybe-native-inputs package-names #:optional (output #f)) - "Given a list of PACKAGE-NAMES, tries to generate the 'inputs' field of a -package definition." - (match (package-names->package-inputs package-names output) - (() - '()) - ((package-inputs ...) - `((native-inputs (,'quasiquote ,package-inputs)))))) + "Same as MAYBE-INPUTS, but for native inputs." + (maybe-inputs package-names output #:type 'native)) + +(define* (maybe-propagated-inputs package-names #:optional (output #f)) + "Same as MAYBE-INPUTS, but for propagated inputs." + (maybe-inputs package-names output #:type 'propagated)) (define* (package->definition guix-package #:optional append-version?/string) - "If APPEND-VERSION?/STRING is #t, append the package's major+minor -version. If APPEND-VERSION?/string is a string, append this string." + "If APPEND-VERSION?/STRING is #t, append the package's major+minor version. +If it is the symbol 'full, append the package's complete version. If +APPEND-VERSION?/string is a string, append this string." (match guix-package ((or ('package ('name name) ('version version) . rest) @@ -279,6 +288,8 @@ version. If APPEND-VERSION?/string is a string, append this string." (string-append name "-" append-version?/string)) ((eq? append-version?/string #t) (string-append name "-" (version-major+minor version))) + ((eq? 'full append-version?/string) + (string-append name "-" version)) (else name))) ,guix-package)))) @@ -438,8 +449,8 @@ obtain a node's uniquely identifying \"key\"." "Return a list of package expressions for PACKAGE-NAME and all its dependencies, sorted in topological order. For each package, call (REPO->GUIX-PACKAGE NAME :KEYS version repo), which should return a -package expression and a list of dependencies; call (GUIX-NAME NAME) to -obtain the Guix package name corresponding to the upstream name." +package expression and a list of dependencies; call (GUIX-NAME PACKAGE-NAME) +to obtain the Guix package name corresponding to the upstream name." (define-record-type <node> (make-node name version package dependencies) node? diff --git a/guix/ipfs.scm b/guix/ipfs.scm new file mode 100644 index 0000000000..31a89888a7 --- /dev/null +++ b/guix/ipfs.scm @@ -0,0 +1,183 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2018 Ludovic Courtès <ludo@gnu.org> +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. + +(define-module (guix ipfs) + #:use-module (json) + #:use-module (guix base64) + #:use-module ((guix build utils) #:select (dump-port)) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-11) + #:use-module (srfi srfi-26) + #:use-module (rnrs io ports) + #:use-module (rnrs bytevectors) + #:use-module (ice-9 match) + #:use-module (ice-9 ftw) + #:use-module (web uri) + #:use-module (web client) + #:use-module (web response) + #:export (%ipfs-base-url + add-data + add-file + + content? + content-name + content-hash + content-size + + add-empty-directory + add-to-directory + read-contents + publish-name)) + +;;; Commentary: +;;; +;;; This module implements bindings for the HTTP interface of the IPFS +;;; gateway, documented here: <https://docs.ipfs.io/reference/api/http/>. It +;;; allows you to add and retrieve files over IPFS, and a few other things. +;;; +;;; Code: + +(define %ipfs-base-url + ;; URL of the IPFS gateway. + (make-parameter "http://localhost:5001")) + +(define* (call url decode #:optional (method http-post) + #:key body (false-if-404? #t) (headers '())) + "Invoke the endpoint at URL using METHOD. Decode the resulting JSON body +using DECODE, a one-argument procedure that takes an input port; when DECODE +is false, return the input port. When FALSE-IF-404? is true, return #f upon +404 responses." + (let*-values (((response port) + (method url #:streaming? #t + #:body body + + ;; Always pass "Connection: close". + #:keep-alive? #f + #:headers `((connection close) + ,@headers)))) + (cond ((= 200 (response-code response)) + (if decode + (let ((result (decode port))) + (close-port port) + result) + port)) + ((and false-if-404? + (= 404 (response-code response))) + (close-port port) + #f) + (else + (close-port port) + (throw 'ipfs-error url response))))) + +;; Result of a file addition. +(define-json-mapping <content> make-content content? + json->content + (name content-name "Name") + (hash content-hash "Hash") + (bytes content-bytes "Bytes") + (size content-size "Size" string->number)) + +;; Result of a 'patch/add-link' operation. +(define-json-mapping <directory> make-directory directory? + json->directory + (hash directory-hash "Hash") + (links directory-links "Links" json->links)) + +;; A "link". +(define-json-mapping <link> make-link link? + json->link + (name link-name "Name") + (hash link-hash "Hash") + (size link-size "Size" string->number)) + +;; A "binding", also known as a "name". +(define-json-mapping <binding> make-binding binding? + json->binding + (name binding-name "Name") + (value binding-value "Value")) + +(define (json->links json) + (match json + (#f '()) + (links (map json->link links)))) + +(define %multipart-boundary + ;; XXX: We might want to find a more reliable boundary. + (string-append (make-string 24 #\-) "2698127afd7425a6")) + +(define (bytevector->form-data bv port) + "Write to PORT a 'multipart/form-data' representation of BV." + (display (string-append "--" %multipart-boundary "\r\n" + "Content-Disposition: form-data\r\n" + "Content-Type: application/octet-stream\r\n\r\n") + port) + (put-bytevector port bv) + (display (string-append "\r\n--" %multipart-boundary "--\r\n") + port)) + +(define* (add-data data #:key (name "file.txt") recursive?) + "Add DATA, a bytevector, to IPFS. Return a content object representing it." + (call (string-append (%ipfs-base-url) + "/api/v0/add?arg=" (uri-encode name) + "&recursive=" + (if recursive? "true" "false")) + json->content + #:headers + `((content-type + . (multipart/form-data + (boundary . ,%multipart-boundary)))) + #:body + (call-with-bytevector-output-port + (lambda (port) + (bytevector->form-data data port))))) + +(define (not-dot? entry) + (not (member entry '("." "..")))) + +(define* (add-file file #:key (name (basename file))) + "Add FILE under NAME to the IPFS and return a content object for it." + (add-data (match (call-with-input-file file get-bytevector-all) + ((? eof-object?) #vu8()) + (bv bv)) + #:name name)) + +(define* (add-empty-directory #:key (name "directory")) + "Return a content object for an empty directory." + (add-data #vu8() #:recursive? #t #:name name)) + +(define* (add-to-directory directory file name) + "Add FILE to DIRECTORY under NAME, and return the resulting directory. +DIRECTORY and FILE must be hashes identifying objects in the IPFS store." + (call (string-append (%ipfs-base-url) + "/api/v0/object/patch/add-link?arg=" + (uri-encode directory) + "&arg=" (uri-encode name) "&arg=" (uri-encode file) + "&create=true") + json->directory)) + +(define* (read-contents object #:key offset length) + "Return an input port to read the content of OBJECT from." + (call (string-append (%ipfs-base-url) + "/api/v0/cat?arg=" object) + #f)) + +(define* (publish-name object) + "Publish OBJECT under the current peer ID." + (call (string-append (%ipfs-base-url) + "/api/v0/name/publish?arg=" object) + json->binding)) diff --git a/guix/scripts/import.scm b/guix/scripts/import.scm index 1d2b45d942..98554ef79b 100644 --- a/guix/scripts/import.scm +++ b/guix/scripts/import.scm @@ -119,7 +119,8 @@ Run IMPORTER with ARGS.\n")) (current-output-port)))))) (match (apply (resolve-importer importer) args) ((and expr (or ('package _ ...) - ('let _ ...))) + ('let _ ...) + ('define-public _ ...))) (print expr)) ((? list? expressions) (for-each (lambda (expr) diff --git a/guix/scripts/import/go.scm b/guix/scripts/import/go.scm index afdba4e8f1..04b07f80cc 100644 --- a/guix/scripts/import/go.scm +++ b/guix/scripts/import/go.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU -;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com> +;;; Copyright © 2020 Katherine Cox-Buday <cox.katherine.e@gmail.com> +;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com> ;;; ;;; This file is part of GNU Guix. ;;; @@ -21,34 +22,38 @@ #:use-module (guix utils) #:use-module (guix scripts) #:use-module (guix import go) + #:use-module (guix import utils) #:use-module (guix scripts import) #:use-module (srfi srfi-1) #:use-module (srfi srfi-11) + #:use-module (srfi srfi-26) #:use-module (srfi srfi-37) #:use-module (ice-9 match) #:use-module (ice-9 format) + #:use-module (ice-9 receive) #:export (guix-import-go)) - + ;;; ;;; Command-line options. ;;; (define %default-options - '()) + '((goproxy . "https://proxy.golang.org"))) (define (show-help) - (display (G_ "Usage: guix import go PACKAGE-PATH -Import and convert the Go module for PACKAGE-PATH.\n")) + (display (G_ "Usage: guix import go PACKAGE-PATH[@VERSION] +Import and convert the Go module for PACKAGE-PATH. Optionally, a version +can be specified after the arobas (@) character.\n")) (display (G_ " -h, --help display this help and exit")) (display (G_ " - -V, --version display version information and exit")) - (display (G_ " - -r, --recursive generate package expressions for all Go modules\ - that are not yet in Guix")) + -r, --recursive generate package expressions for all Go modules +that are not yet in Guix")) (display (G_ " -p, --goproxy=GOPROXY specify which goproxy server to use")) + (display (G_ " + --pin-versions use the exact versions of a module's dependencies")) (newline) (show-bug-report-information)) @@ -58,9 +63,6 @@ Import and convert the Go module for PACKAGE-PATH.\n")) (lambda args (show-help) (exit 0))) - (option '(#\V "version") #f #f - (lambda args - (show-version-and-exit "guix import go"))) (option '(#\r "recursive") #f #f (lambda (opt name arg result) (alist-cons 'recursive #t result))) @@ -69,9 +71,12 @@ Import and convert the Go module for PACKAGE-PATH.\n")) (alist-cons 'goproxy (string->symbol arg) (alist-delete 'goproxy result)))) + (option '("pin-versions") #f #f + (lambda (opt name arg result) + (alist-cons 'pin-versions? #t result))) %standard-import-options)) - + ;;; ;;; Entry point. ;;; @@ -91,27 +96,31 @@ Import and convert the Go module for PACKAGE-PATH.\n")) (('argument . value) value) (_ #f)) - (reverse opts)))) + (reverse opts))) + ;; Append the full version to the package symbol name when using + ;; pinned versions. + (package->definition* (if (assoc-ref opts 'pin-versions?) + (cut package->definition <> 'full) + package->definition))) (match args - ((module-name) - (if (assoc-ref opts 'recursive) - (map (match-lambda - ((and ('package ('name name) . rest) pkg) - `(define-public ,(string->symbol name) - ,pkg)) - (_ #f)) - (go-module-recursive-import module-name - #:goproxy-url - (or (assoc-ref opts 'goproxy) - "https://proxy.golang.org"))) - (let ((sexp (go-module->guix-package module-name - #:goproxy-url - (or (assoc-ref opts 'goproxy) - "https://proxy.golang.org")))) - (unless sexp - (leave (G_ "failed to download meta-data for module '~a'~%") - module-name)) - sexp))) + ((spec) ;e.g., github.com/golang/protobuf@v1.3.1 + (receive (name version) + (package-name->name+version spec) + (let ((arguments (list name + #:goproxy (assoc-ref opts 'goproxy) + #:version version + #:pin-versions? + (assoc-ref opts 'pin-versions?)))) + (if (assoc-ref opts 'recursive) + ;; Recursive import. + (map package->definition* + (apply go-module-recursive-import arguments)) + ;; Single import. + (let ((sexp (apply go-module->guix-package arguments))) + (unless sexp + (leave (G_ "failed to download meta-data for module '~a'~%") + module-name)) + (package->definition* sexp)))))) (() (leave (G_ "too few arguments~%"))) ((many ...) diff --git a/guix/scripts/substitute.scm b/guix/scripts/substitute.scm index 79eaabd8fd..48309f9b3a 100755 --- a/guix/scripts/substitute.scm +++ b/guix/scripts/substitute.scm @@ -63,7 +63,7 @@ #:use-module (web uri) #:use-module (guix http-client) #:export (%allow-unauthenticated-substitutes? - %error-to-file-descriptor-4? + %reply-file-descriptor substitute-urls guix-substitute)) @@ -279,29 +279,29 @@ Internal tool to substitute a pre-built binary to a local build.\n")) "Evaluate EXP... Return its CPU usage as a fraction between 0 and 1." (call-with-cpu-usage-monitoring (lambda () exp ...))) -(define (display-narinfo-data narinfo) - "Write to the current output port the contents of NARINFO in the format -expected by the daemon." - (format #t "~a\n~a\n~a\n" +(define (display-narinfo-data port narinfo) + "Write to PORT the contents of NARINFO in the format expected by the +daemon." + (format port "~a\n~a\n~a\n" (narinfo-path narinfo) (or (and=> (narinfo-deriver narinfo) (cute string-append (%store-prefix) "/" <>)) "") (length (narinfo-references narinfo))) - (for-each (cute format #t "~a/~a~%" (%store-prefix) <>) + (for-each (cute format port "~a/~a~%" (%store-prefix) <>) (narinfo-references narinfo)) (let-values (((uri compression file-size) (narinfo-best-uri narinfo #:fast-decompression? %prefer-fast-decompression?))) - (format #t "~a\n~a\n" + (format port "~a\n~a\n" (or file-size 0) (or (narinfo-size narinfo) 0)))) -(define* (process-query command +(define* (process-query port command #:key cache-urls acl) - "Reply to COMMAND, a query as written by the daemon to this process's + "Reply on PORT to COMMAND, a query as written by the daemon to this process's standard input. Use ACL as the access-control list against which to check authorized substitutes." (define valid? @@ -338,17 +338,17 @@ authorized substitutes." #:open-connection open-connection-for-uri/cached #:make-progress-reporter make-progress-reporter))) (for-each (lambda (narinfo) - (format #t "~a~%" (narinfo-path narinfo))) + (format port "~a~%" (narinfo-path narinfo))) substitutable) - (newline))) + (newline port))) (("info" paths ..1) ;; Reply info about PATHS if it's in CACHE-URLS. (let ((substitutable (lookup-narinfos/diverse cache-urls paths valid? #:open-connection open-connection-for-uri/cached #:make-progress-reporter make-progress-reporter))) - (for-each display-narinfo-data substitutable) - (newline))) + (for-each (cut display-narinfo-data port <>) substitutable) + (newline port))) (wtf (error "unknown `--query' command" wtf)))) @@ -428,14 +428,14 @@ server certificates." "Bind PORT with EXP... to a socket connected to URI." (call-with-cached-connection uri (lambda (port) exp ...))) -(define* (process-substitution store-item destination +(define* (process-substitution port store-item destination #:key cache-urls acl deduplicate? print-build-trace?) "Substitute STORE-ITEM (a store file name) from CACHE-URLS, and write it to DESTINATION as a nar file. Verify the substitute against ACL, and verify its hash against what appears in the narinfo. When DEDUPLICATE? is true, and if -DESTINATION is in the store, deduplicate its files. Print a status line on -the current output port." +DESTINATION is in the store, deduplicate its files. Print a status line to +PORT." (define narinfo (lookup-narinfo cache-urls store-item (if (%allow-unauthenticated-substitutes?) @@ -565,10 +565,10 @@ the current output port." (let ((actual (get-hash))) (if (bytevector=? actual expected) ;; Tell the daemon that we're done. - (format (current-output-port) "success ~a ~a~%" + (format port "success ~a ~a~%" (narinfo-hash narinfo) (narinfo-size narinfo)) ;; The actual data has a different hash than that in NARINFO. - (format (current-output-port) "hash-mismatch ~a ~a ~a~%" + (format port "hash-mismatch ~a ~a ~a~%" (hash-algorithm-name algorithm) (bytevector->nix-base32-string expected) (bytevector->nix-base32-string actual))))))) @@ -682,28 +682,10 @@ default value." (unless (string->uri uri) (leave (G_ "~a: invalid URI~%") uri))) -(define %error-to-file-descriptor-4? - ;; Whether to direct 'current-error-port' to file descriptor 4 like - ;; 'guix-daemon' expects. - (make-parameter #t)) - -;; The daemon's agent code opens file descriptor 4 for us and this is where -;; stderr should go. -(define-syntax-rule (with-redirected-error-port exp ...) - "Evaluate EXP... with the current error port redirected to file descriptor 4 -if needed, as expected by the daemon's agent." - (let ((thunk (lambda () exp ...))) - (if (%error-to-file-descriptor-4?) - (parameterize ((current-error-port (fdopen 4 "wl"))) - ;; Redirect diagnostics to file descriptor 4 as well. - (guix-warning-port (current-error-port)) - - ;; 'with-continuation-barrier' captures the initial value of - ;; 'current-error-port' to report backtraces in case of uncaught - ;; exceptions. Without it, backtraces would be printed to FD 2, - ;; thereby confusing the daemon. - (with-continuation-barrier thunk)) - (thunk)))) +(define %reply-file-descriptor + ;; The file descriptor where replies to the daemon must be sent, or #f to + ;; use the current output port instead. + (make-parameter 4)) (define-command (guix-substitute . args) (category internal) @@ -719,68 +701,73 @@ if needed, as expected by the daemon's agent." (define deduplicate? (find-daemon-option "deduplicate")) - (with-redirected-error-port - (mkdir-p %narinfo-cache-directory) - (maybe-remove-expired-cache-entries %narinfo-cache-directory - cached-narinfo-files - #:entry-expiration - cached-narinfo-expiration-time - #:cleanup-period - %narinfo-expired-cache-entry-removal-delay) - (check-acl-initialized) + (define reply-port + ;; Port used to reply to the daemon. + (if (%reply-file-descriptor) + (fdopen (%reply-file-descriptor) "wl") + (current-output-port))) - ;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error - ;; message. - (for-each validate-uri (substitute-urls)) + (mkdir-p %narinfo-cache-directory) + (maybe-remove-expired-cache-entries %narinfo-cache-directory + cached-narinfo-files + #:entry-expiration + cached-narinfo-expiration-time + #:cleanup-period + %narinfo-expired-cache-entry-removal-delay) + (check-acl-initialized) - ;; Attempt to install the client's locale so that messages are suitably - ;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default - ;; so don't change it. - (match (or (find-daemon-option "untrusted-locale") - (find-daemon-option "locale")) - (#f #f) - (locale (false-if-exception (setlocale LC_MESSAGES locale)))) + ;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error + ;; message. + (for-each validate-uri (substitute-urls)) - (catch 'system-error - (lambda () - (set-thread-name "guix substitute")) - (const #t)) ;GNU/Hurd lacks 'prctl' + ;; Attempt to install the client's locale so that messages are suitably + ;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default + ;; so don't change it. + (match (or (find-daemon-option "untrusted-locale") + (find-daemon-option "locale")) + (#f #f) + (locale (false-if-exception (setlocale LC_MESSAGES locale)))) - (with-networking - (with-error-handling ; for signature errors - (match args - (("--query") - (let ((acl (current-acl))) - (let loop ((command (read-line))) - (or (eof-object? command) - (begin - (process-query command - #:cache-urls (substitute-urls) - #:acl acl) - (loop (read-line))))))) - (("--substitute") - ;; Download STORE-PATH and store it as a Nar in file DESTINATION. - ;; Specify the number of columns of the terminal so the progress - ;; report displays nicely. - (parameterize ((current-terminal-columns (client-terminal-columns))) - (let loop () - (match (read-line) - ((? eof-object?) - #t) - ((= string-tokenize ("substitute" store-path destination)) - (process-substitution store-path destination - #:cache-urls (substitute-urls) - #:acl (current-acl) - #:deduplicate? deduplicate? - #:print-build-trace? - print-build-trace?) - (loop)))))) - ((or ("-V") ("--version")) - (show-version-and-exit "guix substitute")) - (("--help") - (show-help)) - (opts - (leave (G_ "~a: unrecognized options~%") opts))))))) + (catch 'system-error + (lambda () + (set-thread-name "guix substitute")) + (const #t)) ;GNU/Hurd lacks 'prctl' + + (with-networking + (with-error-handling ; for signature errors + (match args + (("--query") + (let ((acl (current-acl))) + (let loop ((command (read-line))) + (or (eof-object? command) + (begin + (process-query reply-port command + #:cache-urls (substitute-urls) + #:acl acl) + (loop (read-line))))))) + (("--substitute") + ;; Download STORE-PATH and store it as a Nar in file DESTINATION. + ;; Specify the number of columns of the terminal so the progress + ;; report displays nicely. + (parameterize ((current-terminal-columns (client-terminal-columns))) + (let loop () + (match (read-line) + ((? eof-object?) + #t) + ((= string-tokenize ("substitute" store-path destination)) + (process-substitution reply-port store-path destination + #:cache-urls (substitute-urls) + #:acl (current-acl) + #:deduplicate? deduplicate? + #:print-build-trace? + print-build-trace?) + (loop)))))) + ((or ("-V") ("--version")) + (show-version-and-exit "guix substitute")) + (("--help") + (show-help)) + (opts + (leave (G_ "~a: unrecognized options~%") opts)))))) ;;; Local Variables: ;;; eval: (put 'with-timeout 'scheme-indent-function 1) diff --git a/guix/scripts/system.scm b/guix/scripts/system.scm index c226f08371..0a051ee4e3 100644 --- a/guix/scripts/system.scm +++ b/guix/scripts/system.scm @@ -1145,7 +1145,7 @@ Some ACTIONS support additional ARGS.\n")) "Return the verbosity level based on OPTS, the alist of parsed options." (or (assoc-ref opts 'verbosity) (if (eq? (assoc-ref opts 'action) 'build) - 2 1))) + 3 1))) ;;; |