Release 7.051

Author: cnuernber

CHANGELOG.md

@@ -1,4 +1,7 @@
 # Changelog
+# 7.051
+ * Much faster string table clone and much faster arrow write of string tables.
+ 
 # 7.050
  * fix bug in stringtable clone.
 

deps.edn

@@ -14,7 +14,7 @@
    :exec-fn codox.main/-main
    :exec-args {:group-id "techascent"
                :artifact-id "tech.ml.dataset"
-               :version "7.050"
+               :version "7.051"
                :name "TMD"
                :description "A Clojure high performance data processing system"
                :metadata {:doc/format :markdown}

docs/000-getting-started.html

@@ -1032,7 +1032,7 @@
    :columns
    (->>
     (columns ds)
-    (pmap column->data)
+    (hamf/pmap column->data)
     (vec))})
 
 

docs/100-walkthrough.html

@@ -599,7 +599,7 @@ _unnamed [7 3]:
                            rs-rfn (hamf-proto/->rfn rs)
                            rs-init (hamf-proto/->init-val-fn rs)
                            init-fn (hamf-fn/function _k (rs-init))]
-                       (doall (hamf/pgroups
+                       (dorun (hamf/pgroups
                                (ds-base/row-count ds)
                                (fn [^long sidx ^long eidx]
                                  (let [tid (.getId (Thread/currentThread))]

docs/200-quick-reference.html

@@ -37,12 +37,13 @@
           ^objects rv (make-array String sz)
           local-int->str int->str
           local-data data]
-      (dorun (hamf/pgroups sz (fn string-table-clone [^long sidx ^long eidx]
+      (StringTable. (ArrayLists/toList (hamf/object-array int->str)) nil (dtype-proto/clone data))
+      #_(dorun (hamf/pgroups sz (fn string-table-clone [^long sidx ^long eidx]
                                 (loop [sidx sidx]
                                   (when (< sidx eidx)
                                     (ArrayHelpers/aset rv sidx (.get int->str (.getLong local-data sidx)))
                                     (recur (inc sidx)))))))
-      (ArrayLists/toList rv)))
+      #_(ArrayLists/toList rv)))
   PStrTable
   (get-str-table [_this] {:int->str int->str
                          :str->int str->int})

docs/columns-readers-and-datatypes.html

@@ -585,7 +585,7 @@ Dependent block frames are not supported!!")
         offsets (dtype/make-list :int32)]
     (if (nil? prev-str-t)
       (dotimes [str-idx (count int->str)]
-        (let [strdata (int->str str-idx)
+        (let [strdata (.get int->str str-idx)
               _ (when (= strdata :failure)
                   (throw (Exception. "Invalid string table - missing entries.")))
               str-bytes (.getBytes (str strdata))
@@ -1990,7 +1990,7 @@ Please use stream->dataset-seq.")))
                  metadata (meta col)]
              (if (nil? prev-ds)
                (assoc ds (metadata :name)
-                      #:tech.v3.dataset{:data (str-table/string-table-from-strings col)
+                      #:tech.v3.dataset{:data (tech.v3.dataset.base/ensure-column-string-table col)
                                         :missing missing
                                         :metadata metadata
                                         :name (metadata :name)})