Merge branch 'master' into gnome-team

1 files changed, 249 insertions, 220 deletions

diff --git a/tests/pypi.scm b/tests/pypi.scm
index 1ddcc542ff..42b39cde73 100644
--- a/tests/pypi.scm
+++ b/tests/pypi.scm
@@ -25,11 +25,19 @@
   #:use-module (guix base32)
   #:use-module (guix memoization)
   #:use-module (guix utils)
+  #:use-module ((guix base16) #:select (base16-string->bytevector))
+  #:use-module (guix upstream)
   #:use-module (gcrypt hash)
   #:use-module (guix tests)
+  #:use-module (guix tests http)
+  #:use-module ((guix download) #:select (url-fetch))
   #:use-module (guix build-system python)
-  #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p))
+  #:use-module ((guix build utils)
+                #:select (delete-file-recursively
+                          which mkdir-p dump-port
+                          with-directory-excursion))
   #:use-module ((guix diagnostics) #:select (guix-warning-port))
+  #:use-module ((guix build syscalls) #:select (mkdtemp!))
   #:use-module (json)
   #:use-module (srfi srfi-26)
   #:use-module (srfi srfi-34)
@@ -38,6 +46,12 @@
   #:use-module (ice-9 match)
   #:use-module (ice-9 optargs))
 
+(define default-sha256
+  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+(define default-sha256/base32
+  (bytevector->nix-base32-string
+   (base16-string->bytevector default-sha256)))
+
 (define* (foo-json #:key (name "foo") (name-in-url #f))
   "Create a JSON description of an example pypi package, named @var{name},
 optionally using a different @var{name in its URL}."
@@ -53,25 +67,20 @@ optionally using a different @var{name in its URL}."
      (urls . #())
      (releases
       . ((1.0.0
-          . #(((url . ,(format #f "https://example.com/~a-1.0.0.egg"
+          . #(((url . ,(format #f "~a/~a-1.0.0.egg"
+                               (%local-url #:path "")
                                (or name-in-url name)))
                (packagetype . "bdist_egg"))
-              ((url . ,(format #f "https://example.com/~a-1.0.0.tar.gz"
+              ((url . ,(format #f "~a/~a-1.0.0.tar.gz"
+                               (%local-url #:path "")
                                (or name-in-url name)))
-               (packagetype . "sdist"))
-              ((url . ,(format #f "https://example.com/~a-1.0.0-py2.py3-none-any.whl"
+               (packagetype . "sdist")
+               (digests . (("sha256" . ,default-sha256))))
+              ((url . ,(format #f "~a/~a-1.0.0-py2.py3-none-any.whl"
+                               (%local-url #:path "")
                                (or name-in-url name)))
                (packagetype . "bdist_wheel")))))))))
 
-(define test-json-1
-  (foo-json))
-
-(define test-json-2
-  (foo-json #:name "foo-99"))
-
-(define test-source-hash
-  "")
-
 (define test-specifications
   '("Fizzy [foo, bar]"
     "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1"
@@ -131,6 +140,70 @@ Provides-Extra: testing
 Requires-Dist: pytest (>=3.1.0); extra == 'testing'
 ")
 
+(define sample-directory
+  ;; Directory containing tarballs and .whl files for this test.
+  (let ((template (string-append (or (getenv "TMPDIR") "/tmp")
+                                 "/guix-pypi-test-XXXXXX")))
+    (mkdtemp! template)))
+
+(define (pypi-tarball name specs)
+  "Return a PyPI tarball called NAME suffixed with '.tar.gz' and containing
+the files specified in SPECS.  Return its file name."
+  (let ((directory (in-vicinity sample-directory name))
+        (tarball (in-vicinity sample-directory (string-append name ".tar.gz"))))
+    (false-if-exception (delete-file tarball))
+    (mkdir-p directory)
+    (for-each (match-lambda
+                ((file content)
+                 (mkdir-p (in-vicinity directory (dirname file)))
+                 (call-with-output-file (in-vicinity directory file)
+                   (lambda (port)
+                     (display content port)))))
+              specs)
+    (parameterize ((current-output-port (%make-void-port "w0")))
+      (system* "tar" "-C" sample-directory "-czvf" tarball
+               (basename directory)))
+    (delete-file-recursively directory)
+    tarball))
+
+(define (wheel-file name specs)
+  "Return a Wheel file called NAME suffixed with '.whl' and containing the
+files specified by SPECS.  Return its file name."
+  (let* ((directory (in-vicinity sample-directory
+                                 (string-append name ".dist-info")))
+         (zip-file (in-vicinity sample-directory
+                                (string-append name ".zip")))
+         (whl-file (in-vicinity sample-directory
+                                (string-append name ".whl"))))
+    (false-if-exception (delete-file whl-file))
+    (mkdir-p directory)
+    (for-each (match-lambda
+                ((file content)
+                 (mkdir-p (in-vicinity directory (dirname file)))
+                 (call-with-output-file (in-vicinity directory file)
+                   (lambda (port)
+                     (display content port)))))
+              specs)
+    ;; zip always adds a "zip" extension to the file it creates,
+    ;; so we need to rename it.
+    (with-directory-excursion (dirname directory)
+      (system* "zip" "-qr" zip-file (basename directory)))
+    (rename-file zip-file whl-file)
+    (delete-file-recursively directory)
+    whl-file))
+
+(define (file-dump file)
+  "Return a procedure that dumps FILE to the given port."
+  (lambda (output)
+    (call-with-input-file file
+      (lambda (input)
+        (dump-port input output)))))
+
+(define-syntax-rule (with-pypi responses body ...)
+  (with-http-server responses
+    (parameterize ((%pypi-base-url (%local-url #:path "/")))
+      body ...)))
+
 
 (test-begin "pypi")
 
@@ -219,218 +292,174 @@ Requires-Dist: pytest (>=3.1.0); extra == 'testing'
    "https://files.pythonhosted.org/packages/f0/f00/goo-0.0.0.tar.gz"))
 
 (test-assert "pypi->guix-package, no wheel"
-  ;; Replace network resources with sample data.
-    (mock ((guix import utils) url-fetch
-           (lambda (url file-name)
-             (match url
-               ("https://example.com/foo-1.0.0.tar.gz"
-                (begin
-                  ;; Unusual requires.txt location should still be found.
-                  (mkdir-p "foo-1.0.0/src/bizarre.egg-info")
-                  (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt"
-                    (lambda ()
-                      (display test-requires.txt)))
-                  (parameterize ((current-output-port (%make-void-port "rw+")))
-                    (system* "tar" "czvf" file-name "foo-1.0.0/"))
-                  (delete-file-recursively "foo-1.0.0")
-                  (set! test-source-hash
-                    (call-with-input-file file-name port-sha256))))
-               ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
-               (_ (error "Unexpected URL: " url)))))
-          (mock ((guix http-client) http-fetch
-                 (lambda (url . rest)
-                   (match url
-                     ("https://pypi.org/pypi/foo/json"
-                      (values (open-input-string test-json-1)
-                              (string-length test-json-1)))
-                     ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
-                     (_ (error "Unexpected URL: " url)))))
-                (match (pypi->guix-package "foo")
-                  (('package
-                     ('name "python-foo")
-                     ('version "1.0.0")
-                     ('source ('origin
-                                ('method 'url-fetch)
-                                ('uri ('pypi-uri "foo" 'version))
-                                ('sha256
-                                 ('base32
-                                  (? string? hash)))))
-                     ('build-system 'pyproject-build-system)
-                     ('propagated-inputs ('list 'python-bar 'python-foo))
-                     ('native-inputs ('list 'python-pytest))
-                     ('home-page "http://example.com")
-                     ('synopsis "summary")
-                     ('description "summary")
-                     ('license 'license:lgpl2.0))
-                   (and (string=? (bytevector->nix-base32-string
-                                   test-source-hash)
-                                  hash)
-                        (equal? (pypi->guix-package "foo" #:version "1.0.0")
-                                (pypi->guix-package "foo"))
-                        (guard (c ((error? c) #t))
-                          (pypi->guix-package "foo" #:version "42"))))
-                  (x
-                   (pk 'fail x #f))))))
+  (let ((tarball (pypi-tarball
+                  "foo-1.0.0"
+                  `(("src/bizarre.egg-info/requires.txt"
+                     ,test-requires.txt))))
+        (twice (lambda (lst) (append lst lst))))
+    (with-pypi (twice `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                        ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
+                        ("/foo/json" 200 ,(lambda (port)
+                                            (display (foo-json) port)))))
+      (match (pypi->guix-package "foo")
+        (`(package
+            (name "python-foo")
+            (version "1.0.0")
+            (source (origin
+                      (method url-fetch)
+                      (uri (pypi-uri "foo" version))
+                      (sha256
+                       (base32 ,(? string? hash)))))
+            (build-system pyproject-build-system)
+            (propagated-inputs (list python-bar python-foo))
+            (native-inputs (list python-pytest))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary")
+            (license license:lgpl2.0))
+         (and (string=? default-sha256/base32 hash)
+              (equal? (pypi->guix-package "foo" #:version "1.0.0")
+                      (pypi->guix-package "foo"))
+              (guard (c ((error? c) #t))
+                (pypi->guix-package "foo" #:version "42"))))
+        (x
+         (pk 'fail x #f))))))
 
 (test-skip (if (which "zip") 0 1))
 (test-assert "pypi->guix-package, wheels"
-  ;; Replace network resources with sample data.
-  (mock ((guix import utils) url-fetch
-         (lambda (url file-name)
-           (match url
-             ("https://example.com/foo-1.0.0.tar.gz"
-              (begin
-                (mkdir-p "foo-1.0.0/foo.egg-info/")
-                (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
-                  (lambda ()
-                    (display "wrong data to make sure we're testing wheels ")))
-                (parameterize ((current-output-port (%make-void-port "rw+")))
-                  (system* "tar" "czvf" file-name "foo-1.0.0/"))
-                (delete-file-recursively "foo-1.0.0")
-                (set! test-source-hash
-                  (call-with-input-file file-name port-sha256))))
-             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
-              (begin
-                (mkdir "foo-1.0.0.dist-info")
-                (with-output-to-file "foo-1.0.0.dist-info/METADATA"
-                  (lambda ()
-                    (display test-metadata)))
-                (let ((zip-file (string-append file-name ".zip")))
-                  ;; zip always adds a "zip" extension to the file it creates,
-                  ;; so we need to rename it.
-                  (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA")
-                  (rename-file zip-file file-name))
-                (delete-file-recursively "foo-1.0.0.dist-info")))
-             (_ (error "Unexpected URL: " url)))))
-        (mock ((guix http-client) http-fetch
-               (lambda (url . rest)
-                 (match url
-                   ("https://pypi.org/pypi/foo/json"
-                    (values (open-input-string test-json-1)
-                            (string-length test-json-1)))
-                   ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
-                   (_ (error "Unexpected URL: " url)))))
-              ;; Not clearing the memoization cache here would mean returning the value
-              ;; computed in the previous test.
-              (invalidate-memoization! pypi->guix-package)
-              (match (pypi->guix-package "foo")
-                (('package
-                   ('name "python-foo")
-                   ('version "1.0.0")
-                   ('source ('origin
-                              ('method 'url-fetch)
-                              ('uri ('pypi-uri "foo" 'version))
-                              ('sha256
-                               ('base32
-                                (? string? hash)))))
-                   ('build-system 'pyproject-build-system)
-                   ('propagated-inputs ('list 'python-bar 'python-baz))
-                   ('native-inputs ('list 'python-pytest))
-                   ('home-page "http://example.com")
-                   ('synopsis "summary")
-                   ('description "summary")
-                   ('license 'license:lgpl2.0))
-                 (string=? (bytevector->nix-base32-string
-                            test-source-hash)
-                           hash))
-                (x
-                 (pk 'fail x #f))))))
+  (let ((tarball (pypi-tarball
+                  "foo-1.0.0"
+                  '(("foo-1.0.0/foo.egg-info/requires.txt"
+                     "wrong data \
+to make sure we're testing wheels"))))
+        (wheel (wheel-file "foo-1.0.0"
+                           `(("METADATA" ,test-metadata)))))
+    (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                 ("/foo-1.0.0-py2.py3-none-any.whl"
+                  200 ,(file-dump wheel))
+                 ("/foo/json" 200 ,(lambda (port)
+                                     (display (foo-json) port))))
+      ;; Not clearing the memoization cache here would mean returning the value
+      ;; computed in the previous test.
+      (invalidate-memoization! pypi->guix-package)
+      (match (pypi->guix-package "foo")
+        (`(package
+            (name "python-foo")
+            (version "1.0.0")
+            (source (origin
+                      (method url-fetch)
+                      (uri (pypi-uri "foo" version))
+                      (sha256
+                       (base32 ,(? string? hash)))))
+            (build-system pyproject-build-system)
+            (propagated-inputs (list python-bar python-baz))
+            (native-inputs (list python-pytest))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary")
+            (license license:lgpl2.0))
+         (string=? default-sha256/base32 hash))
+        (x
+         (pk 'fail x #f))))))
 
 (test-assert "pypi->guix-package, no usable requirement file."
-  ;; Replace network resources with sample data.
-  (mock ((guix import utils) url-fetch
-         (lambda (url file-name)
-           (match url
-             ("https://example.com/foo-1.0.0.tar.gz"
-              (mkdir-p "foo-1.0.0/foo.egg-info/")
-              (parameterize ((current-output-port (%make-void-port "rw+")))
-                (system* "tar" "czvf" file-name "foo-1.0.0/"))
-              (delete-file-recursively "foo-1.0.0")
-              (set! test-source-hash
-                (call-with-input-file file-name port-sha256)))
-             ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
-             (_ (error "Unexpected URL: " url)))))
-        (mock ((guix http-client) http-fetch
-               (lambda (url . rest)
-                 (match url
-                   ("https://pypi.org/pypi/foo/json"
-                    (values (open-input-string test-json-1)
-                            (string-length test-json-1)))
-                   ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
-                   (_ (error "Unexpected URL: " url)))))
-              ;; Not clearing the memoization cache here would mean returning the value
-              ;; computed in the previous test.
-              (invalidate-memoization! pypi->guix-package)
-              (match (pypi->guix-package "foo")
-                (('package
-                   ('name "python-foo")
-                   ('version "1.0.0")
-                   ('source ('origin
-                              ('method 'url-fetch)
-                              ('uri ('pypi-uri "foo" 'version))
-                              ('sha256
-                               ('base32
-                                (? string? hash)))))
-                   ('build-system 'pyproject-build-system)
-                   ('home-page "http://example.com")
-                   ('synopsis "summary")
-                   ('description "summary")
-                   ('license 'license:lgpl2.0))
-                 (string=? (bytevector->nix-base32-string
-                            test-source-hash)
-                           hash))
-                (x
-                 (pk 'fail x #f))))))
+  (let ((tarball (pypi-tarball "foo-1.0.0"
+                               '(("foo.egg-info/.empty" "")))))
+    (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
+                 ("/foo/json" 200 ,(lambda (port)
+                                     (display (foo-json) port))))
+      ;; Not clearing the memoization cache here would mean returning the
+      ;; value computed in the previous test.
+      (invalidate-memoization! pypi->guix-package)
+      (match (pypi->guix-package "foo")
+        (`(package
+            (name "python-foo")
+            (version "1.0.0")
+            (source (origin
+                       (method url-fetch)
+                       (uri (pypi-uri "foo" version))
+                       (sha256
+                        (base32 ,(? string? hash)))))
+            (build-system pyproject-build-system)
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary")
+            (license license:lgpl2.0))
+         (string=? default-sha256/base32 hash))
+        (x
+         (pk 'fail x #f))))))
 
 (test-assert "pypi->guix-package, package name contains \"-\" followed by digits"
-  ;; Replace network resources with sample data.
-  (mock ((guix import utils) url-fetch
-         (lambda (url file-name)
-           (match url
-             ("https://example.com/foo-99-1.0.0.tar.gz"
-              (begin
-                ;; Unusual requires.txt location should still be found.
-                (mkdir-p "foo-99-1.0.0/src/bizarre.egg-info")
-                (with-output-to-file "foo-99-1.0.0/src/bizarre.egg-info/requires.txt"
-                  (lambda ()
-                    (display test-requires.txt)))
-                (parameterize ((current-output-port (%make-void-port "rw+")))
-                  (system* "tar" "czvf" file-name "foo-99-1.0.0/"))
-                (delete-file-recursively "foo-99-1.0.0")
-                (set! test-source-hash
-                  (call-with-input-file file-name port-sha256))))
-             ("https://example.com/foo-99-1.0.0-py2.py3-none-any.whl" #f)
-             (_ (error "Unexpected URL: " url)))))
-        (mock ((guix http-client) http-fetch
-               (lambda (url . rest)
-                 (match url
-                   ("https://pypi.org/pypi/foo-99/json"
-                    (values (open-input-string test-json-2)
-                            (string-length test-json-2)))
-                   ("https://example.com/foo-99-1.0.0-py2.py3-none-any.whl" #f)
-                   (_ (error "Unexpected URL: " url)))))
-              (match (pypi->guix-package "foo-99")
-                (('package
-                   ('name "python-foo-99")
-                   ('version "1.0.0")
-                   ('source ('origin
-                              ('method 'url-fetch)
-                              ('uri ('pypi-uri "foo-99" 'version))
-                              ('sha256
-                               ('base32
-                                (? string? hash)))))
-                   ('properties ('quote (("upstream-name" . "foo-99"))))
-                   ('build-system 'pyproject-build-system)
-                   ('propagated-inputs ('list 'python-bar 'python-foo))
-                   ('native-inputs ('list 'python-pytest))
-                   ('home-page "http://example.com")
-                   ('synopsis "summary")
-                   ('description "summary")
-                   ('license 'license:lgpl2.0))
-                 (string=? (bytevector->nix-base32-string
-                            test-source-hash)
-                           hash))
-                (x
-                 (pk 'fail x #f))))))
+  (let ((tarball (pypi-tarball "foo-99-1.0.0"
+                               `(("src/bizarre.egg-info/requires.txt"
+                                  ,test-requires.txt)))))
+    (with-pypi `(("/foo-99-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                 ("/foo-99-1.0.0-py2.py3-none-any.whl" 404 "")
+                 ("/foo-99/json" 200 ,(lambda (port)
+                                        (display (foo-json #:name "foo-99")
+                                                 port))))
+      (match (pypi->guix-package "foo-99")
+        (`(package
+            (name "python-foo-99")
+            (version "1.0.0")
+            (source (origin
+                      (method url-fetch)
+                      (uri (pypi-uri "foo-99" version))
+                      (sha256
+                       (base32 ,(? string? hash)))))
+            (properties (quote (("upstream-name" . "foo-99"))))
+            (build-system pyproject-build-system)
+            (propagated-inputs (list python-bar python-foo))
+            (native-inputs (list python-pytest))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary")
+            (license license:lgpl2.0))
+         (string=? default-sha256/base32 hash))
+        (x
+         (pk 'fail x #f))))))
+
+(test-equal "package-latest-release"
+  (list '("foo-1.0.0.tar.gz")
+        '("foo-1.0.0.tar.gz.asc")
+        (list (upstream-input
+               (name "bar")
+               (downstream-name "python-bar")
+               (type 'propagated))
+              (upstream-input
+               (name "foo")
+               (downstream-name "python-foo")
+               (type 'propagated))
+              (upstream-input
+               (name "pytest")
+               (downstream-name "python-pytest")
+               (type 'native))))
+  (let ((tarball (pypi-tarball
+                  "foo-1.0.0"
+                  `(("src/bizarre.egg-info/requires.txt"
+                     ,test-requires.txt)))))
+    (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
+                 ("/foo/json" 200 ,(lambda (port)
+                                     (display (foo-json) port))))
+      (define source
+        (package-latest-release
+         (dummy-package "python-foo"
+                        (version "0.1.2")
+                        (source (dummy-origin
+                                 (method url-fetch)
+                                 (uri (pypi-uri "foo" version))))
+                        (build-system python-build-system))
+         (list %pypi-updater)))
+
+      (list (map basename (upstream-source-urls source))
+            (map basename (upstream-source-signature-urls source))
+            (upstream-source-inputs source)))))
 
 (test-end "pypi")
+(delete-file-recursively sample-directory)
+
+;; Local Variables:
+;; eval: (put 'with-pypi 'scheme-indent-function 1)
+;; End: