Adding ability to do use your own containers during parse time.

Author: cnuernber

CHANGELOG.md

@@ -1,6 +1,6 @@
 {:paths ["src" "resources" "target/classes"]
  :deps {;;org.clojure/clojure {:mvn/version "1.11.1"}
-        cnuernber/dtype-next {:mvn/version "10.130"}
+        cnuernber/dtype-next {:mvn/version "10.131"}
         techascent/tech.io {:mvn/version "4.31"
                             :exclusions [org.apache.commons/commons-compress]}
         org.apache.datasketches/datasketches-java {:mvn/version "4.2.0"}
@@ -14,7 +14,7 @@
    :exec-fn codox.main/-main
    :exec-args {:group-id "techascent"
                :artifact-id "tech.ml.dataset"
-               :version "7.042"
+               :version "7.043"
                :name "TMD"
                :description "A Clojure high performance data processing system"
                :metadata {:doc/format :markdown}

deps.edn

@@ -45,6 +45,9 @@
     (dtype-proto/->native-buffer backing-store))
   LongBuffer
   (elemwiseDatatype [_this] (dtype-proto/elemwise-datatype backing-store))
+  (clear [this]
+    (set! backing-store (abuf/as-growable-list (dtype/make-list :int8 1500) 0))
+    (set! int-width 8))
   (lsize [_this] (.size backing-store))
   (size [_this] (.size backing-store))
   (subBuffer [this sidx eidx]

docs/000-getting-started.html

@@ -73,26 +73,28 @@
 
 (defn make-container
   (^IMutList [dtype options]
-   (case dtype
-     :string (str-table/make-string-table 0 "")
-     :text
-     (let [^IMutList list-data
-           (try
-             (if (and (not= false (get options :text-temp-dir false))
-                      @file-backed-text-enabled*)
-               (let [tmp-dir (:text-temp-dir options)]
-                 (file-backed-text/file-backed-text (merge
-                                                     {:suffix ".txt"}
-                                                     (when tmp-dir
-                                                       {:temp-dir tmp-dir}))))
-               (dtype/make-list :text))
-             (catch Throwable e
-               (when-not @warn-atom*
-                 (reset! warn-atom* true)
-                 (log/warn e "File backed text failed.  Falling back to in-memory"))
-               (dtype/make-list :text)))]
-             list-data)
-     (dtype/make-list dtype)))
+   (if-let [rv (get-in options [:datatype-parsers dtype])]
+     (rv dtype options)
+     (case dtype
+       :string (str-table/make-string-table 0 "")
+       :text
+       (let [^IMutList list-data
+             (try
+               (if (and (not= false (get options :text-temp-dir false))
+                        @file-backed-text-enabled*)
+                 (let [tmp-dir (:text-temp-dir options)]
+                   (file-backed-text/file-backed-text (merge
+                                                       {:suffix ".txt"}
+                                                       (when tmp-dir
+                                                         {:temp-dir tmp-dir}))))
+                 (dtype/make-list :text))
+               (catch Throwable e
+                 (when-not @warn-atom*
+                   (reset! warn-atom* true)
+                   (log/warn e "File backed text failed.  Falling back to in-memory"))
+                 (dtype/make-list :text)))]
+         list-data)
+       (dtype/make-list dtype))))
   (^IMutList [dtype]
    (make-container dtype nil)))
 

docs/100-walkthrough.html

@@ -2,6 +2,7 @@
   "Per-column parsers."
   (:require [tech.v3.dataset.io.datetime :as parse-dt]
             [tech.v3.dataset.impl.column-base :as column-base]
+            [tech.v3.dataset.protocols :as ds-proto]
             [tech.v3.datatype.packing :as packing]
             [tech.v3.datatype :as dtype]
             [tech.v3.datatype.casting :as casting]
@@ -162,7 +163,9 @@
   (merge
    #:tech.v3.dataset{:data (or (dtype/as-array-buffer container)
                                (dtype/as-native-buffer container)
-                               container)
+                               (if (instance? clojure.lang.IDeref container)
+                                 @container
+                                 container))
                      :missing missing
                      :force-datatype? true}
    (when (and failed-values
@@ -496,6 +499,13 @@
               (.contains missing idx))
         nil
         (.get container idx))))
+  ds-proto/PClearable
+  (ds-clear [this]
+    (.clear container)
+    (.clear missing)
+    (set! last-idx -1)
+    (set! max-idx -1)
+    (set! mc 0))
   PParser
   (addValue [_p idx value]
     (set! max-idx idx)

docs/200-quick-reference.html

@@ -90,12 +90,8 @@
         colparser-compute-fn (reify Function
                                (apply [this col-idx]
                                  (let [colname (col-idx->colname col-idx)
-                                       colname (cond
-                                                 (number? colname)
-                                                 colname
-                                                 (empty? colname)
+                                       colname (if (empty? colname)
                                                  (make-colname col-idx)
-                                                 :else
                                                  (utils/remove-zero-width-spaces colname))
                                        colname (if (and ensure-unique-column-names?
                                                         (get colname->idx colname))

docs/columns-readers-and-datatypes.html

@@ -5,6 +5,7 @@
   (:require [tech.v3.datatype :as dtype]
             [tech.v3.dataset.io.column-parsers :as column-parsers]
             [tech.v3.dataset.io.context :as parse-context]
+            [tech.v3.dataset.protocols :as ds-proto]
             [tech.v3.parallel.for :as pfor]
             [tech.v3.datatype.argtypes :as argtypes]
             [tech.v3.dataset.impl.dataset :as ds-impl]
@@ -15,13 +16,15 @@
             [ham-fisted.protocols :as hamf-proto])
   (:import [java.util HashMap Map$Entry Map Map$Entry LinkedHashMap Iterator]
            [java.util.function Function Consumer]
-           [tech.v3.dataset.protocols PDatasetParser]
+           [tech.v3.dataset.protocols PDatasetParser PClearable]
            [clojure.lang IDeref Counted Indexed]
            [ham_fisted Reductions$IndexedAccum Reducible Consumers$IncConsumer
             ITypedReduce]))
 
 
-(defrecord ParseRecord [^long col-idx column-name column-parser])
+(defrecord ParseRecord [^long col-idx column-name column-parser]
+  ds-proto/PClearable
+  (ds-clear [this] (ds-proto/ds-clear column-parser)))
 
 
 (deftype ^:private MapseqReducer [options parsers consumer ^Consumers$IncConsumer row-idx]
@@ -59,7 +62,12 @@
               init (hamf/range (.value row-idx)))))
   IDeref
   (deref [this]
-    (parse-context/parsers->dataset (assoc options :key-fn nil) parsers (.value row-idx))))
+    (parse-context/parsers->dataset (assoc options :key-fn nil) parsers (.value row-idx)))
+  PClearable
+  (ds-clear [this]
+    (reduce (fn [_ p] (ds-proto/ds-clear p)) nil (.values ^Map parsers))
+    (.setValue row-idx 0)
+    this))
 
 
 (defn mapseq-reducer

docs/index.html

@@ -43,6 +43,10 @@
   (add-rows [p rows]
     "rows need only be reducible"))
 
+(defprotocol PClearable
+  (ds-clear [p]
+    "Reset to initial state.  Avoid conflict with collection/clear"))
+
 
 (defprotocol PDatasetTransform
   (transform [t dataset]))

docs/nippy-serialization-rocks.html

@@ -39,6 +39,10 @@
   ObjectBuffer
   (elemwiseDatatype [_this] :string)
   (lsize [_this] (.size data))
+  (clear [this]
+    (.clear int->str)
+    (.clear str->int)
+    (.clear data))
   (size [_this] (.size data))
   (subBuffer [this sidx eidx]
     (ChunkedList/sublistCheck sidx eidx (.lsize this))

docs/supported-datatypes.html

@@ -1193,7 +1193,9 @@ Dependent block frames are not supported!!")
 
         buf-entries (buffers->buf-entries buffers)
         last-entry (last buf-entries)
-        body-len (pad (+ (long (last-entry :offset)) (long (last-entry :length))))
+        body-len (if last-entry
+                   (pad (+ (long (last-entry :offset)) (long (last-entry :length))))
+                   0)
         builder (FlatBufferBuilder.)
         msg-start (.getCurrentPosition writer)
         _ (write-message-header writer