1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
Upstream status: https://gitlab.com/kernel-firmware/linux-firmware/-/merge_requests/396
From 5d8341be46374c5b4c3e34af0c9fd4837f25ad67 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 2 Jan 2025 00:31:43 +0900
Subject: [PATCH 3/4] Add support to install files/symlinks in parallel.
This reduces the install-zst target time from 100 s to 25 s on my test
system, a 400% speed improvement.
* Makefile (NUM_JOBS): New variable.
(install, install-xz, install-zst): Use it.
* copy-firmware.sh (num_jobs): New variable.
(has_gnu_parallel): New procedure.
<-j>: Parse new option, and use it along GNU parallel to parallelize
firmware copying and compression/symlink creation.
Signed-off-by: Maxim Cournoyer <maxim.cournoyer@gmail.com>
---
Makefile | 8 +++++---
copy-firmware.sh | 51 ++++++++++++++++++++++++++++++++++++++++++++----
2 files changed, 52 insertions(+), 7 deletions(-)
diff --git a/Makefile b/Makefile
index 1f507bfd..0719f874 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,6 @@
FIRMWAREDIR = /lib/firmware
+NUM_JOBS := $(or $(patsubst -j%,%,$(filter -j%,$(MAKEFLAGS))),\
+ 1)
all:
@@ -33,17 +35,17 @@ install:
false; \
fi
install -d $(DESTDIR)$(FIRMWAREDIR)
- ./copy-firmware.sh $(DESTDIR)$(FIRMWAREDIR)
+ ./copy-firmware.sh -j$(NUM_JOBS) $(DESTDIR)$(FIRMWAREDIR)
@echo "Now run \"make dedup\" to de-duplicate any firmware files"
install-xz:
install -d $(DESTDIR)$(FIRMWAREDIR)
- ./copy-firmware.sh --xz $(DESTDIR)$(FIRMWAREDIR)
+ ./copy-firmware.sh -j$(NUM_JOBS) --xz $(DESTDIR)$(FIRMWAREDIR)
@echo "Now run \"make dedup\" to de-duplicate any firmware files"
install-zst:
install -d $(DESTDIR)$(FIRMWAREDIR)
- ./copy-firmware.sh --zstd $(DESTDIR)$(FIRMWAREDIR)
+ ./copy-firmware.sh -j$(NUM_JOBS) --zstd $(DESTDIR)$(FIRMWAREDIR)
@echo "Now run \"make dedup\" to de-duplicate any firmware files"
clean:
diff --git a/copy-firmware.sh b/copy-firmware.sh
index 1a1094ae..f703443e 100755
--- a/copy-firmware.sh
+++ b/copy-firmware.sh
@@ -9,6 +9,7 @@ verbose=:
compress=cat
compext=
destdir=
+num_jobs=1
err() {
printf "ERROR: %s\n" "$*"
@@ -19,6 +20,15 @@ warn() {
printf "WARNING: %s\n" "$*"
}
+has_gnu_parallel() {
+ if command -v parallel > /dev/null; then
+ if parallel --version | grep -Fq 'GNU Parallel'; then
+ return 0
+ fi
+ fi
+ return 1
+}
+
while test $# -gt 0; do
case $1 in
-v | --verbose)
@@ -27,6 +37,16 @@ while test $# -gt 0; do
shift
;;
+ -j*)
+ num_jobs=$(echo "$1" | sed 's/-j//')
+ if [ "$num_jobs" -gt 1 ] && ! has_gnu_parallel; then
+ err "the GNU parallel command is required to use -j"
+ fi
+ parallel_args_file=$(mktemp)
+ trap 'rm -f $parallel_args_file' EXIT INT QUIT TERM
+ shift
+ ;;
+
--xz)
if test "$compext" = ".zst"; then
err "cannot mix XZ and ZSTD compression"
@@ -76,12 +96,24 @@ grep -E '^(RawFile|File):' WHENCE | sed -E -e 's/^(RawFile|File): */\1 /;s/"//g'
$verbose "copying/compressing file $f$compext"
if test "$compress" != "cat" && test "$k" = "RawFile"; then
$verbose "compression will be skipped for file $f"
- cat "$f" > "$destdir/$f"
+ if [ "$num_jobs" -gt 1 ]; then
+ echo "cat \"$f\" > \"$destdir/$f\"" >> "$parallel_args_file"
+ else
+ cat "$f" > "$destdir/$f"
+ fi
else
- $compress "$f" > "$destdir/$f$compext"
+ if [ "$num_jobs" -gt 1 ]; then
+ echo "$compress \"$f\" > \"$destdir/$f$compext\"" >> "$parallel_args_file"
+ else
+ $compress "$f" > "$destdir/$f$compext"
+ fi
fi
done
+if [ "$num_jobs" -gt 1 ]; then
+ parallel -j"$num_jobs" -a "$parallel_args_file"
+fi
+echo > "$parallel_args_file"
# shellcheck disable=SC2162 # file/folder name can include escaped symbols
grep -E '^Link:' WHENCE | sed -e 's/^Link: *//g;s/-> //g' | while read l t; do
directory="$destdir/$(dirname "$l")"
@@ -89,12 +121,23 @@ grep -E '^Link:' WHENCE | sed -e 's/^Link: *//g;s/-> //g' | while read l t; do
target="$(cd "$directory" && realpath -m -s "$t")"
if test -e "$target"; then
$verbose "creating link $l -> $t"
- ln -s "$t" "$destdir/$l"
+ if [ "$num_jobs" -gt 1 ]; then
+ echo "ln -s \"$t\" \"$destdir/$l\"" >> "$parallel_args_file"
+ else
+ ln -s "$t" "$destdir/$l"
+ fi
else
$verbose "creating link $l$compext -> $t$compext"
- ln -s "$t$compext" "$destdir/$l$compext"
+ if [ "$num_jobs" -gt 1 ]; then
+ echo "ln -s \"$t$compext\" \"$destdir/$l$compext\"" >> "$parallel_args_file"
+ else
+ ln -s "$t$compext" "$destdir/$l$compext"
+ fi
fi
done
+if [ "$num_jobs" -gt 1 ]; then
+ parallel -j"$num_jobs" -a "$parallel_args_file"
+fi
# Verify no broken symlinks
if test "$(find "$destdir" -xtype l | wc -l)" -ne 0 ; then
--
GitLab
|
