gnu: xnnpack: Update to commit 08f1489.

More tests fail when building the shared library, so they are disabled.  The
scripts to generate the autogenerated files run all commands at once, so they
are converted to Makefiles first.

* gnu/packages/machine-learning.scm (xnnpack): Update to commit 08f1489.
[source]: Remove patch.  Adjust snippet.
[arguments]<#:configure-flags>: Add "-DXNNPACK_BUILD_TESTS=OFF".
<#:tests?>: Disable tests.
<#:phases>: Adjust 'fix-cmake' phase.  Convert code generation scripts to
Makefiles and run them.  Run different missing scripts.
* gnu/packages/patches/xnnpack-remove-broken-tests.patch: Remove file.
* gnu/packages/patches/xnnpack-system-libraries.patch: Remove file.
* gnu/local.mk (dist_patch_DATA): Remove them.

1 files changed, 65 insertions, 37 deletions

diff --git a/gnu/packages/machine-learning.scm b/gnu/packages/machine-learning.scm
index 4f47334a80..b72c442fc0 100644
--- a/gnu/packages/machine-learning.scm
+++ b/gnu/packages/machine-learning.scm
@@ -4106,7 +4106,7 @@ on quantized 8-bit tensors.")
 (define-public xnnpack
   ;; There's currently no tag on this repo.
   (let ((version "0.0")
-        (commit "51a987591a6fc9f0fc0707077f53d763ac132cbf")
+        (commit "08f1489fc815e8f121d4d2676c4863d2b51bfe73")
         (revision "3"))
     (package
       (name "xnnpack")
@@ -4119,34 +4119,14 @@ on quantized 8-bit tensors.")
          (file-name (git-file-name name version))
          (sha256
           (base32
-           "1rzby82xq8d0rl1d148yz88jh9cpsw5c8b2yw7yg39mi7qmr55rm"))
-         ;; Some tests fail to link as they use internal symbols, which are
-         ;; not included in the shared library.
-         ;; XXX: Additionally, these tests fail on i686 due to incorrect results:
-         ;; 171 - f32-vlrelu-test (Failed)
-         ;; 211 - qs8-gavgpool-minmax-fp32-test (Failed)
-         ;; 224 - qu8-avgpool-minmax-fp32-test (Failed)
-         ;; 228 - qu8-gavgpool-minmax-fp32-test (Failed)
-         ;; 263 - x32-packx-test (Failed)
-         (patches (search-patches "xnnpack-remove-broken-tests.patch"))
+           "00jjhz0nfggbdnqqvcznba03pcyy7gssd24yhhzjhincnz9qh8jr"))
          (modules '((guix build utils)
                     (ice-9 ftw)
                     (ice-9 textual-ports)
                     (srfi srfi-26)))
          (snippet
           '(begin
-             ;; Remove autogenerated files
-             (for-each
-              (lambda (dir)
-                (let ((gendir (string-append "src/" dir "/gen")))
-                  (when (file-exists? gendir)
-                    (delete-file-recursively gendir)
-                    ;; Needed for the scripts generating the files
-                    (mkdir gendir))))
-              (scandir "src" (negate (cut member <> '("." "..")))))
-             (delete-file-recursively "google3")
-             (delete-file "cmake/microkernels.cmake")
-             ;; Additional autogenerated files which contain the string
+             ;; Remove autogenerated files, which contain the string
              ;; "Auto-generated file"
              (for-each
               (lambda (dir)
@@ -4159,8 +4139,18 @@ on quantized 8-bit tensors.")
                                 (get-string-all port)
                                 "Auto-generated file")))
                        (delete-file path))))
-                 (scandir dir (negate (cut member <> '("." ".."))))))
-              '("test" "bench" "eval" "models" "src/enums" "src/xnnpack"))))))
+                 (scandir dir (negate (cut member <> '("." ".." "simd"))))))
+              (cons*
+               "test" "bench" "eval" "models" "src/enums" "src/xnnpack"
+               "gen" "cmake/gen"
+               (filter
+                identity
+                (map
+                 (lambda (dir)
+                   (let ((path
+                          (string-append "src/" dir "/gen")))
+                     (and (file-exists? path) path)))
+                 (scandir "src" (negate (cut member <> '("." ".."))))))))))))
       (build-system cmake-build-system)
       (arguments
        (list
@@ -4169,7 +4159,11 @@ on quantized 8-bit tensors.")
                              "-DBUILD_SHARED_LIBS=ON"
                              "-DCMAKE_POSITION_INDEPENDENT_CODE=ON"
                              "-DXNNPACK_LIBRARY_TYPE=shared"
-                             "-DXNNPACK_BUILD_BENCHMARKS=FALSE")
+                             "-DXNNPACK_BUILD_BENCHMARKS=FALSE"
+                             ;; Tests fail to build with -DXNNPACK_LIBRARY_TYPE=shared:
+                             ;; https://github.com/google/XNNPACK/issues/6285
+                             "-DXNNPACK_BUILD_TESTS=OFF")
+        #:tests? #f
         #:modules '((ice-9 ftw)
                     (guix build cmake-build-system)
                     (guix build utils))
@@ -4179,27 +4173,61 @@ on quantized 8-bit tensors.")
               (lambda _
                 (substitute* "CMakeLists.txt"
                   (("TARGET_INCLUDE_DIRECTORIES\\((pthreadpool|cpuinfo).*") "")
-                  ((".*IF\\(NOT TARGET gtest\\).*")
-                   "IF(FALSE)\n")
-                  (("SET\\(CMAKE_CXX_STANDARD 11\\)")
-                   "SET(CMAKE_CXX_STANDARD 14)")
                   (("AMD64") "x86_64"))))
             (add-after 'patch-source-shebangs 'generate-files
               (lambda _
+                ;; This script just calls two other scripts.
+                (delete-file "scripts/generate-tests-and-benchmarks.sh")
+                ;; The bash scripts run all jobs at once and then wait, so we
+                ;; convert them to Makefiles.
                 (for-each
                  (lambda (name)
+                   (define counter 0) ; For the targets
+                   (define target-deps "")
                    (when (and (string-prefix? "generate" name)
-                              (string-suffix? ".sh" name)
-                              (not (equal? "generate-amalgamation.sh" name)))
-                     (display (string-append name "\n"))
-                     (invoke "bash" (string-append "scripts/" name))))
+                              (string-suffix? ".sh" name))
+                     (let ((file (string-append "scripts/" name)))
+                       (substitute* file
+                         ;; Turn the commands into targets and remove trailing
+                         ;; '&' characters
+                         (("(.*(\\.sh|\\.py|-o |--output)[^&]*)&?[[:space:]]*$" _ command)
+                          (begin
+                            (set! counter (+ counter 1))
+                            (string-append "target" (number->string counter)
+                                           ":" target-deps
+                                           "\n\t" command "\n")))
+                         (("[[:space:]]*wait[[:space:]]*") "")
+                         ;; The commands after this line depend on the
+                         ;; previous commands in the file.
+                         (("JIT requires assembly files to be generated first.*" all)
+                          (begin
+                            (set! target-deps
+                                  (string-append
+                                   target-deps " target"
+                                   (string-join
+                                    (map number->string (iota counter 1)) " target")))
+                            all)))
+                       (display (string-append "Running " name "\n"))
+                       (apply invoke "make" "-s" "-f" file "-j"
+                              (number->string (parallel-job-count))
+                              (map
+                               (lambda (i)
+                                 (string-append "target" (number->string i)))
+                               (iota counter 1))))))
                  (scandir "scripts"))
                 ;; These need to run after the above scripts
                 (display "Remaining files\n")
                 (invoke "python3" "tools/update-microkernels.py")
-                (substitute* "tools/amalgamate-microkernels.py"
-                  (("BUILD") "BUILD.bazel"))
-                (invoke "bash" "scripts/generate-amalgamation.sh"))))))
+                (invoke "python3" "tools/update-microkernels.py" "-a")
+                (invoke "python3" "tools/generate-lut-norm-test.py"
+                        "--spec" "test/u8-lut32norm.yaml"
+                        "--output" "test/u8-lut32norm.cc")
+                (invoke "python3" "tools/generate-gemm-test.py"
+                        "--spec" "test/qd8-f16-qb4w-gemm-minmax.yaml"
+                        "--output-test" "test/qd8-f16-qb4w-gemm-minmax.cc")
+                (invoke "python3" "tools/generate-gemm-test.py"
+                        "--spec" "test/qd8-f32-qb4w-gemm-minmax.yaml"
+                        "--output-test" "test/qd8-f32-qb4w-gemm-minmax.cc"))))))
       (inputs
        (list clog
              cpuinfo