Release 7.026

Author: cnuernber

CHANGELOG.md

@@ -1,4 +1,7 @@
 # Changelog
+# 7.036
+ * Major optimization (>9x!) loading of arrow files when large string tables/dictionaries are used.
+ 
 # 7.035
  * Latest dtype-next (10.124) - contains upgrades to ham-fisted which allow pmap et al. to accept arbitrary executor services.
  * Fix for [issue 438](https://github.com/techascent/tech.ml.dataset/issues/438) - keyword dataset names in tribuo.

deps.edn

@@ -1,6 +1,6 @@
 {:paths ["src" "resources" "target/classes"]
  :deps {;;org.clojure/clojure {:mvn/version "1.11.1"}
-        cnuernber/dtype-next {:mvn/version "10.124"}
+        cnuernber/dtype-next {:mvn/version "10.125"}
         techascent/tech.io {:mvn/version "4.31"
                             :exclusions [org.apache.commons/commons-compress]}
         org.apache.datasketches/datasketches-java {:mvn/version "4.2.0"}
@@ -14,7 +14,7 @@
    :exec-fn codox.main/-main
    :exec-args {:group-id "techascent"
                :artifact-id "tech.ml.dataset"
-               :version "7.035"
+               :version "7.036"
                :name "TMD"
                :description "A Clojure high performance data processing system"
                :metadata {:doc/format :markdown}

docs/000-getting-started.html

@@ -580,6 +580,12 @@ null [6 3]:
   (tech.v3.dataset.base/drop-rows dataset-or-col row-indexes)))
 
 
+(defn empty-column-names
+  "Return a sequence of column names whose empty set length matches the row count of the dataset."
+  ([ds]
+  (tech.v3.dataset-api/empty-column-names ds)))
+
+
 (defn empty-dataset
   ([]
   (tech.v3.dataset.impl.dataset/empty-dataset )))
@@ -974,6 +980,12 @@ test/data/stocks.csv [10 3]:
   (tech.v3.dataset.base/remove-columns dataset colname-seq-or-fn)))
 
 
+(defn remove-empty-columns
+  "Remove all columns that have no data - missing set length equals row count."
+  ([ds]
+  (tech.v3.dataset-api/remove-empty-columns ds)))
+
+
 (defn remove-rows
   "Same as drop-rows."
   ([dataset-or-col row-indexes]

docs/100-walkthrough.html

@@ -420,6 +420,12 @@ null [6 3]:
   (tech.v3.dataset.metamorph-api/drop-rows row-indexes)))
 
 
+(defn empty-column-names
+  "Return a sequence of column names whose empty set length matches the row count of the dataset."
+  ([]
+  (tech.v3.dataset.metamorph-api/empty-column-names )))
+
+
 (defn empty-dataset
   ([]
   (tech.v3.dataset.metamorph-api/empty-dataset )))
@@ -839,6 +845,12 @@ test/data/stocks.csv [10 3]:
   (tech.v3.dataset.metamorph-api/remove-columns colname-seq-or-fn)))
 
 
+(defn remove-empty-columns
+  "Remove all columns that have no data - missing set length equals row count."
+  ([]
+  (tech.v3.dataset.metamorph-api/remove-empty-columns )))
+
+
 (defn remove-rows
   "Same as drop-rows."
   ([row-indexes]

docs/200-quick-reference.html

@@ -1244,16 +1244,24 @@ Dependent block frames are not supported!!")
   ^List [offsets data n-elems]
   (let [n-elems (long n-elems)
         offsets (dtype/->reader offsets)]
-    (reify ObjectReader
-      (elemwiseDatatype [rdr] :string)
-      (lsize [rdr] n-elems)
-      (readObject [rdr idx]
-        (let [start-off (long (offsets idx))
-              end-off (long (offsets (inc idx)))]
-          (-> (dtype/sub-buffer data start-off
-                                (- end-off start-off))
-              (dtype/->byte-array)
-              (String.)))))))
+    (if (instance? NativeBuffer data)
+      (reify ObjectReader
+        (elemwiseDatatype [rdr] :string)
+        (lsize [rdr] n-elems)
+        (readObject [rdr idx]
+          (let [start-off (long (offsets idx))
+                end-off (long (offsets (inc idx)))]
+            (native-buffer/native-buffer->string data start-off (- end-off start-off)))))
+      (reify ObjectReader
+        (elemwiseDatatype [rdr] :string)
+        (lsize [rdr] n-elems)
+        (readObject [rdr idx]
+          (let [start-off (long (offsets idx))
+                end-off (long (offsets (inc idx)))]
+            (-> (dtype/sub-buffer data start-off
+                                  (- end-off start-off))
+                (dtype/->byte-array)
+                (String.))))))))
 
 (defn- offsets-data->bytedata-reader
   ^List [offsets data n-elems]
@@ -1306,15 +1314,13 @@ Dependent block frames are not supported!!")
 (defn- string-data->column-data
   [dict-map encoding offset-buf-dtype buffers n-elems]
   (if encoding
-    (let [str-list (get-in dict-map [(:id encoding) :strings])
-          index-data (-> (first buffers)
-                         (native-buffer/set-native-datatype
-                          (get-in encoding [:index-type :datatype]))
-                         (dtype/sub-buffer 0 n-elems)
-                         (dtype/clone))
-          retval (StringTable. str-list nil (dyn-int-list/make-from-container
-                                             index-data))]
-      retval)
+    (StringTable. (get-in dict-map [(:id encoding) :strings])
+                  nil
+                  (-> (first buffers)
+                      (native-buffer/set-native-datatype
+                       (get-in encoding [:index-type :datatype]))
+                      (native-buffer/->jvm-array 0 n-elems)
+                      (dyn-int-list/make-from-container)))
     (let [[offsets varchar-data] buffers]
       (-> (offsets-data->string-reader (native-buffer/set-native-datatype
                                         offsets offset-buf-dtype)