Skip to content

Commit f11aad0

Browse files
pmonkspmonks
committed
⬆️ Upgrade dependency and switch to more resource-conscious concurrency
1 parent b925d83 commit f11aad0

File tree

5 files changed

+47
-29
lines changed

5 files changed

+47
-29
lines changed

‎deps.edn

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
miikka/clj-base62 {:mvn/version "0.1.1"}
2929
com.github.pmonks/clj-spdx {:mvn/version "1.0.176"}
3030
com.github.pmonks/rencg {:mvn/version "1.0.51"}
31-
com.github.pmonks/embroidery {:mvn/version "1.0.41"}}
31+
com.github.pmonks/embroidery {:mvn/version "1.0.44"}}
3232
:aliases
3333
{:build {:deps {com.github.pmonks/pbr {:mvn/version "RELEASE"}}
3434
:ns-default pbr.build}}}

‎src/lice_comb/deps.clj

+13-11
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@
2121
license information."
2222
(:require [clojure.string :as s]
2323
[clojure.tools.logging :as log]
24-
[embroidery.api :as e]
2524
[lice-comb.maven :as lcmvn]
2625
[lice-comb.files :as lcf]
27-
[lice-comb.impl.expressions-info :as lciei]))
26+
[lice-comb.impl.expressions-info :as lciei]
27+
[lice-comb.impl.utils :as lciu]))
2828

2929
(defn- normalise-dep
3030
"Normalises a dep, by removing any classifier suffixes from the artifact-id
@@ -60,15 +60,17 @@
6060
nil))]
6161
gav-expressions
6262
; If we didn't find any licenses in the dep's POM, check the dep's JAR(s)
63-
(into {} (filter identity (e/pmap* #(try
64-
(lcf/zip->expressions-info %)
65-
(catch javax.xml.stream.XMLStreamException xse
66-
(log/warn (str "Failed to parse pom inside " % " - ignoring") xse)
67-
nil)
68-
(catch java.util.zip.ZipException ze
69-
(log/warn (str "Failed to unzip " % " - ignoring") ze)
70-
nil))
71-
(:paths info))))))))
63+
(into {} (filter identity
64+
(lciu/file-handle-bounded-pmap
65+
#(try
66+
(lcf/zip->expressions-info %)
67+
(catch javax.xml.stream.XMLStreamException xse
68+
(log/warn (str "Failed to parse pom inside " % " - ignoring") xse)
69+
nil)
70+
(catch java.util.zip.ZipException ze
71+
(log/warn (str "Failed to unzip " % " - ignoring") ze)
72+
nil))
73+
(:paths info))))))))
7274

7375
(defmulti dep->expressions-info
7476
"Returns an expressions-info map for `dep` (a `MapEntry` or two-element vector

‎src/lice_comb/files.clj

+16-13
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
(:require [clojure.string :as s]
2323
[clojure.java.io :as io]
2424
[clojure.tools.logging :as log]
25-
[embroidery.api :as e]
2625
[lice-comb.matching :as lcm]
2726
[lice-comb.maven :as lcmvn]
2827
[lice-comb.impl.expressions-info :as lciei]
@@ -160,19 +159,23 @@
160159
([dir] (dir->expressions-info dir nil))
161160
([dir {:keys [include-hidden-dirs? include-zips?] :or {include-hidden-dirs? false include-zips? false} :as opts}]
162161
(when (lciu/readable-dir? dir)
163-
(let [file-expressions (into {} (filter identity (e/pmap* #(try
164-
(file->expressions-info %)
165-
(catch Exception e
166-
(log/warn (str "Unexpected exception while processing " % " - ignoring") e)
167-
nil))
168-
(probable-license-files dir opts))))]
162+
(let [file-expressions (into {} (filter identity
163+
(lciu/file-handle-bounded-pmap
164+
#(try
165+
(file->expressions-info %)
166+
(catch Exception e
167+
(log/warn (str "Unexpected exception while processing " % " - ignoring") e)
168+
nil))
169+
(probable-license-files dir opts))))]
169170
(if include-zips?
170-
(let [zip-expressions (into {} (filter identity (e/pmap* #(try
171-
(zip->expressions-info %)
172-
(catch Exception e
173-
(log/warn (str "Unexpected exception while processing " % " - ignoring") e)
174-
nil))
175-
(zip-compressed-files dir opts))))]
171+
(let [zip-expressions (into {} (filter identity
172+
(lciu/file-handle-bounded-pmap
173+
#(try
174+
(zip->expressions-info %)
175+
(catch Exception e
176+
(log/warn (str "Unexpected exception while processing " % " - ignoring") e)
177+
nil))
178+
(zip-compressed-files dir opts))))]
176179
(lciei/prepend-source (lciu/filepath dir) (merge file-expressions zip-expressions)))
177180
(lciei/prepend-source (lciu/filepath dir) file-expressions))))))
178181

‎src/lice_comb/impl/utils.clj

+13-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
lice-comb and may change without notice."
2323
(:require [clojure.string :as s]
2424
[clojure.java.io :as io]
25-
[clj-base62.core :as base62]))
25+
[clj-base62.core :as base62]
26+
[embroidery.api :as e]))
2627

2728
(defn mapfonk
2829
"Returns a new map where f has been applied to all of the keys of m."
@@ -326,3 +327,14 @@
326327
(if-not (s/blank? val)
327328
val
328329
default))))
330+
331+
; Note: we could use OSHI to determine the actual number of possible open file
332+
; handles on the runtime environment, but it seems like overkill to bring in
333+
; such a large dependency for this one feature, especially when lice-comb
334+
; typically won't get close to opening this many files.
335+
(defn file-handle-bounded-pmap
336+
"bounded-pmap* hardcoded to no more than 8192 virtual threads. This size is
337+
determined conservatively from macOS, since it's the least common denominator
338+
of the major OSes in terms of number of possible open file handles."
339+
[f coll]
340+
(e/bounded-pmap* 8192 f coll))

‎src/lice_comb/lein.clj

+4-3
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
information."
2222
(:require [embroidery.api :as e]
2323
[lice-comb.deps :as lcd]
24-
[lice-comb.impl.expressions-info :as lciei]))
24+
[lice-comb.impl.expressions-info :as lciei]
25+
[lice-comb.impl.utils :as lciu]))
2526

2627
(defn- lein-dep->toolsdeps-dep
2728
"Converts a leiningen style dependency vector into a (partial) tools.deps style
@@ -54,14 +55,14 @@
5455
expressions were found)."
5556
[deps]
5657
(when deps
57-
(into {} (e/pmap* #(vec [% (dep->expressions-info %)]) deps))))
58+
(into {} (lciu/file-handle-bounded-pmap #(vec [% (dep->expressions-info %)]) deps))))
5859

5960
(defn deps->expressions
6061
"Returns a map of sets of SPDX expressions (`String`s) for each Leiningen
6162
style dep in `deps`. See [[deps->expressions-info]] for details."
6263
[deps]
6364
(when deps
64-
(into {} (e/pmap* #(vec [% (dep->expressions %)]) deps))))
65+
(into {} (lciu/file-handle-bounded-pmap #(vec [% (dep->expressions %)]) deps))))
6566

6667
(defn init!
6768
"Initialises this namespace upon first call (and does nothing on subsequent

0 commit comments

Comments
 (0)