Merge pull request #53 from pmonks/dev

Release 2.0.292

Author: pmonks

README.md

@@ -15,7 +15,7 @@ This library leverages, and is inspired by, the *excellent* [SPDX project](https
 
 ## Disclaimer
 
-**The author and contributors to `lice-comb` are not lawyers, and neither they nor `lice-comb` itself provides legal advice. This is nothing more than logic to assist in finding license information.**
+**The author and contributors to `lice-comb` are not lawyers, and neither they nor `lice-comb` itself provides legal advice. This is nothing more than logic to assist in finding license information.**  If you need a primer on the legal aspects of open source software, the author has found the [Blue Oak Council](https://blueoakcouncil.org/) to be a useful resource.
 
 ## System Requirements
 

deps.edn

@@ -26,7 +26,7 @@
     hato/hato                             {:mvn/version "0.9.0"}
     dev.weavejester/medley                {:mvn/version "1.7.0"}
     miikka/clj-base62                     {:mvn/version "0.1.1"}
-    com.github.pmonks/clj-spdx            {:mvn/version "1.0.145"}
+    com.github.pmonks/clj-spdx            {:mvn/version "1.0.152"}
     com.github.pmonks/rencg               {:mvn/version "1.0.51"}
     com.github.pmonks/embroidery          {:mvn/version "0.1.20"}}
  :aliases

src/lice_comb/files.clj

@@ -45,11 +45,16 @@
 (defn probable-license-files
   "Returns all probable license files in the given directory, recursively, as a
   set of java.io.File objects. dir may be a String or a java.io.File, either of
-  which must refer to a readable directory."
-  [dir]
-  (when (lciu/readable-dir? dir)
-    (some-> (seq (filter #(and (.isFile ^java.io.File %) (probable-license-file? %)) (file-seq (io/file dir))))
-            set)))
+  which must refer to a readable directory. By default ignores hidden
+  directories."
+  ([dir] (probable-license-files dir false))
+  ([dir include-hidden-dirs?]
+   (when (lciu/readable-dir? dir)
+     (some-> (lciu/filter-file-only-seq (io/file dir)
+                                        (fn [^java.io.File d] (and (not= (.getCanonicalFile d) (.getCanonicalFile (io/file (lcmvn/local-maven-repo))))  ; Make sure to exclude the Maven local repo, just in case it happens to be nested within dir
+                                                                   (or include-hidden-dirs? (not (.isHidden d)))))
+                                        probable-license-file?)
+             set))))
 
 (defn file->expressions-info
   "Returns an expressions-info map for the given file (an InputStream or
@@ -123,14 +128,18 @@
 (defn- zip-compressed-files
   "Returns a set of all probable ZIP compressed files (Files) in the given
   directory, recursively, or nil if there are none. dir may be a String or a
-  java.io.File, and must refer to a readable directory."
-  [dir]
-  (when (lciu/readable-dir? dir)
-    (some-> (seq (filter #(and (.isFile ^java.io.File %)
-                               (or (s/ends-with? (str %) ".zip")
-                                   (s/ends-with? (str %) ".jar")))
-                         (file-seq (io/file dir))))
-            set)))
+  java.io.File, and must refer to a readable directory. By default ignores
+  hidden directories."
+  ([dir] (zip-compressed-files dir false))
+  ([dir include-hidden-dirs?]
+   (when (lciu/readable-dir? dir)
+      (some-> (lciu/filter-file-only-seq (io/file dir)
+                                         (fn [^java.io.File d] (or include-hidden-dirs? (not (.isHidden d))))
+                                         (fn [^java.io.File f]
+                                           (let [lname (s/lower-case (.getName f))]
+                                             (or (s/ends-with? lname ".zip")
+                                                 (s/ends-with? lname ".jar")))))
+              set))))
 
 (defn dir->expressions-info
   "Returns an expressions-info map for the given dir (a String or a File,

src/lice_comb/impl/utils.clj

@@ -252,6 +252,46 @@
   [_]
   (throw (ex-info "Cannot determine filename of an InputStream - did you forget to provide it separately?" {})))
 
+(defn filter-file-seq*
+  "As for clojure.core/file-seq, but with support for filtering.  pred must be
+  a predicate that accepts one argument of type java.io.File.  Files that don't
+  meet pred will not be included in the result, and directories that don't meet
+  pred will not be recursed into (so pred must be able to handle both distinct
+  cases).
+
+  Note also that dir is always returned, even if it does not meet pred."
+  [^java.io.File dir pred]
+  (let [pred   (or pred (constantly true))
+        filter (reify java.io.FileFilter (accept [_ f] (boolean (pred (.getCanonicalFile ^java.io.File f)))))]  ; Use the canonical file, otherwise we will get tripped up by "." being "hidden" according to the JVM when running on a Unix 🤡
+    (tree-seq
+      (fn [^java.io.File f] (.isDirectory f))
+      (fn [^java.io.File d] (seq (.listFiles d filter)))
+      dir)))
+
+(defn filter-file-seq
+  "As for clojure.core/file-seq, but with support for filtering.  dir-pred
+  controls which directories will be included in the result and recursed into.
+  file-pred controls which files will be included in the result.  Both must be
+  a predicate of one argument of type java.io.File.
+
+  Note also that dir is always returned, even if it does not meet dir-pred."
+  [dir dir-pred file-pred]
+  (let [dir-pred  (or dir-pred  (constantly true))
+        file-pred (or file-pred (constantly true))
+        pred      (fn [^java.io.File f]
+                    (or (and (.isDirectory f) (dir-pred f))
+                        (file-pred f)))]
+    (filter-file-seq* dir pred)))
+
+(defn filter-file-only-seq
+  "As for clojure.core/file-seq, with support for filtering and only returns
+  files (but not any directories that were traversed during the seq).  dir-pred
+  controls which directories will be recursed into.  file-pred controls which
+  files will be included in the result.  Both must be a predicate of one
+  argument of type java.io.File."
+  [dir dir-pred file-pred]
+  (seq (filter #(.isFile ^java.io.File %) (filter-file-seq dir dir-pred file-pred))))
+
 (defn getenv
   "Obtain the given environment variable, returning default (or nil, if default
   is not provided) if it isn't set."

src/lice_comb/maven.clj

@@ -119,11 +119,11 @@
   "Attempts to determine the license(s) (a map) from a POM license name/URL
   pair. Returns nil if no matches were found."
   [{:keys [name url]}]
-  ; 1. Look in the name field(s)
+  ; 1. Look in the name field
   (let [name-expressions (lcm/name->expressions-info name)]
     (if (or (empty? name-expressions)
             (and (= 1 (count name-expressions)) (lcm/unidentified? (first (keys name-expressions)))))
-      ; 2. If the names didn't give us any identified licenses, look in the url field(s) (this can be slower and less accurate, which is why it has lower priority)
+      ; 2. If the name didn't give us any identified licenses, look in the url field (this can be slower and less accurate, which is why it has lower priority)
       (let [uri-expressions (lcm/uri->expressions-info url)]
         (if (or (empty? uri-expressions)
                 (and (= 1 (count uri-expressions)) (lcm/unidentified? (first (keys uri-expressions)))))
@@ -258,7 +258,7 @@
            version      (if (release-version? version) (ga-release-version group-id artifact-id) version)
            file-version (resolve-snapshot-version group-id artifact-id version)
            gav-path     (str (s/replace group-id "." "/") "/" artifact-id "/" version "/" artifact-id "-" file-version ".pom")
-           local-pom (io/file (str @local-maven-repo-a separator (s/replace gav-path "/" separator)))]
+           local-pom    (io/file (str @local-maven-repo-a separator (s/replace gav-path "/" separator)))]
        (if (and (.exists local-pom)
                 (.isFile local-pom))
          (.toURI local-pom)

test/lice_comb/data/CC-BY-4.0/LICENSE

@@ -393,3 +393,4 @@ the avoidance of doubt, this paragraph does not form part of the
 public licenses.
 
 Creative Commons may be contacted at creativecommons.org.
+

test/lice_comb/files_test.clj

@@ -92,14 +92,14 @@
   (testing "Handed a directory"
     (is (nil? (file->expressions "."))))
   (testing "Files on disk"
-;    (is (= #{"CC-BY-4.0"} (file->expressions  (str test-data-path "/CC-BY-4.0/LICENSE"))))  ; Failing due to https://github.com/spdx/license-list-XML/issues/1960
-    (is (valid= #{"MPL-2.0"}   (file->expressions  (str test-data-path "/MPL-2.0/LICENSE")))))
+;    (is (= #{"CC-BY-4.0"}    (file->expressions  (str test-data-path "/CC-BY-4.0/LICENSE"))))  ; Failing due to https://github.com/spdx/Spdx-Java-Library/issues/233
+    (is (valid= #{"MPL-2.0"} (file->expressions  (str test-data-path "/MPL-2.0/LICENSE")))))
   (testing "URLs"
     (is (valid= #{"Apache-2.0"} (file->expressions  "https://www.apache.org/licenses/LICENSE-2.0.txt")))
     (is (valid= #{"Apache-2.0"} (file->expressions  (io/as-url "https://www.apache.org/licenses/LICENSE-2.0.txt")))))
   (testing "InputStreams"
     (is (thrown? clojure.lang.ExceptionInfo (with-open [is (io/input-stream "https://www.apache.org/licenses/LICENSE-2.0.txt")] (file->expressions  is))))
-    (is (valid= #{"Apache-2.0"} (with-open [is (io/input-stream "https://www.apache.org/licenses/LICENSE-2.0.txt")]                  (file->expressions  is "LICENSE_2.0.txt")))))
+    (is (valid= #{"Apache-2.0"} (with-open [is (io/input-stream "https://www.apache.org/licenses/LICENSE-2.0.txt")]             (file->expressions  is "LICENSE_2.0.txt")))))
   (testing "POM files"
     (is (valid= #{"Apache-2.0"}   (file->expressions  (str test-data-path "/simple.pom"))))
     (is (valid= #{"BSD-3-Clause"} (file->expressions  (str test-data-path "/no-xml-ns.pom"))))
@@ -136,10 +136,10 @@
     (is (nil? (dir->expressions  "this_directory_does_not_exist")))
     (is (nil? (dir->expressions  "deps.edn"))))
   (testing "Valid directory"
-    (is (valid= ;#{"GPL-2.0-only WITH Classpath-exception-2.0" "BSD-3-Clause" "Apache-2.0" "Unlicense AND CC0-1.0" "MIT" "MPL-2.0" "CC-BY-4.0"}  ; CC-BY-4.0 failing due to https://github.com/spdx/license-list-XML/issues/1960
+    (is (valid= ;#{"GPL-2.0-only WITH Classpath-exception-2.0" "BSD-3-Clause" "Apache-2.0" "Unlicense AND CC0-1.0" "MIT" "MPL-2.0" "CC-BY-4.0"}  ; Failing due to https://github.com/spdx/Spdx-Java-Library/issues/233
                 #{"GPL-2.0-only WITH Classpath-exception-2.0" "BSD-3-Clause" "Apache-2.0" "Unlicense AND CC0-1.0" "MIT" "MPL-2.0"}
                 (dir->expressions "."))))
   (testing "Valid directory - include ZIP compressed files"
-    (is (valid= ;#{"GPL-2.0-only WITH Classpath-exception-2.0" "BSD-3-Clause" "Apache-2.0" "Unlicense AND CC0-1.0" "MIT" "MPL-2.0" "CC-BY-4.0" "AGPL-3.0-or-later"}  ; CC-BY-4.0 failing due to https://github.com/spdx/license-list-XML/issues/1960
+    (is (valid= ;#{"GPL-2.0-only WITH Classpath-exception-2.0" "BSD-3-Clause" "Apache-2.0" "Unlicense AND CC0-1.0" "MIT" "MPL-2.0" "CC-BY-4.0" "AGPL-3.0-or-later"}  ; Failing due to https://github.com/spdx/Spdx-Java-Library/issues/233
                 #{"GPL-2.0-only WITH Classpath-exception-2.0" "BSD-3-Clause" "Apache-2.0" "Unlicense AND CC0-1.0" "MIT" "MPL-2.0" "AGPL-3.0-or-later"}
                 (dir->expressions  "." {:include-zips? true})))))