7.045

Author: cnuernber

CHANGELOG.md

@@ -1,4 +1,7 @@
 # Changelog
+# 7.045
+ * Bulk add-constant! method used for adding missing values.
+ 
 # 7.044
  * initial support for clearing dataset parsers - resets their row count but does not reset the schema.  Use tech.v3.dataset.protocols/ds-clear.
 

deps.edn

@@ -1,6 +1,6 @@
 {:paths ["src" "resources" "target/classes"]
  :deps {;;org.clojure/clojure {:mvn/version "1.11.1"}
-        cnuernber/dtype-next {:mvn/version "10.131"}
+        cnuernber/dtype-next {:mvn/version "10.133"}
         techascent/tech.io {:mvn/version "4.31"
                             :exclusions [org.apache.commons/commons-compress]}
         org.apache.datasketches/datasketches-java {:mvn/version "4.2.0"}
@@ -14,7 +14,7 @@
    :exec-fn codox.main/-main
    :exec-args {:group-id "techascent"
                :artifact-id "tech.ml.dataset"
-               :version "7.043"
+               :version "7.045"
                :name "TMD"
                :description "A Clojure high performance data processing system"
                :metadata {:doc/format :markdown}

docs/000-getting-started.html

@@ -11,7 +11,7 @@
             [tech.v3.parallel.for :as parallel-for]
             [clj-commons.primitive-math :as pmath])
   (:import [tech.v3.datatype LongBuffer]
-           [ham_fisted IMutList]))
+           [ham_fisted IMutList Casts]))
 
 (set! *warn-on-reflection* true)
 (set! *unchecked-math* :warn-on-boxed)
@@ -68,6 +68,12 @@
         (set! backing-store (dtype/make-list :int32 backing-store))
         (set! int-width 32)))
     (.addLong backing-store value))
+  (add [this idx ct value]
+    (when-not (== idx (.size this))
+      (throw (RuntimeException. "Insertion other than at end is not supported.")))
+    (let [lv (Casts/longCast value)]
+      (dotimes [c ct]
+        (.addLong this lv))))
   (getLong [_this idx]
     (.getLong backing-store idx))
   (readLong [_this idx]

docs/100-walkthrough.html

@@ -148,11 +148,11 @@
 
 (defn add-missing-values!
   ^long [^IMutList container ^RoaringBitmap missing
-   missing-value ^long idx]
+         missing-value ^long idx]
   (let [n-elems (.size container)]
     (when (< n-elems idx)
       (.add missing (long n-elems) idx)
-      (.addAllReducible container (hamf/repeat (- idx n-elems) missing-value)))
+      (.add container n-elems (- idx n-elems) missing-value))
     (- idx n-elems)))
 
 

docs/200-quick-reference.html

@@ -90,9 +90,15 @@
         colparser-compute-fn (reify Function
                                (apply [this col-idx]
                                  (let [colname (col-idx->colname col-idx)
-                                       colname (if (empty? colname)
-                                                 (make-colname col-idx)
-                                                 (utils/remove-zero-width-spaces colname))
+                                       colname (cond
+                                                 (nil? colname) (make-colname col-idx)
+                                                 (string? colname) (if (empty? colname)
+                                                                     (make-colname col-idx)
+                                                                     (utils/remove-zero-width-spaces colname))
+                                                 (sequential? colname) (if (empty? colname)
+                                                                         (make-colname colname)
+                                                                         colname)
+                                                 :else colname)
                                        colname (if (and ensure-unique-column-names?
                                                         (get colname->idx colname))
                                                  (unique-column-name-fn col-idx colname)

docs/columns-readers-and-datatypes.html

@@ -78,6 +78,17 @@
                                retval)))))]
       (.addLong data item-idx))
     true)
+  (add [this idx ct value]
+    (let [value (or value "")
+          item-idx (int (.computeIfAbsent
+                         str->int
+                         value
+                         (reify Function
+                           (apply [this keyval]
+                             (let [retval (.size int->str)]
+                               (.add int->str keyval)
+                               retval)))))]
+      (.add data (.size data) ct item-idx)))
   (readObject [_this idx]
     (.get int->str (.getLong data idx)))
   (writeObject [this idx value]