Merge branch 'deephaven:main' into coverage-workflow-2

.github/workflows/nightly-check-ci.yml

@@ -117,7 +117,7 @@ jobs:
           webhook-type: webhook-trigger
           payload: |
             {
-              "slack_message": "Nightly build failure in ${{ matrix.gradle-task }} on Java ${{ matrix.test-jvm-version }} @ ${{ github.head_ref }} ${{ github.sha }}" ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              "slack_message": "Nightly build failure in ${{ matrix.gradle-task }} on Java ${{ matrix.test-jvm-version }} @ ${{ github.head_ref }} ${{ github.sha }} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
             }
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_NIGHTLY_FAILURE }}

.github/workflows/publish-ci.yml

@@ -81,36 +81,86 @@ jobs:
             web/client-api/types/build/*.tgz
 
       - name: Publish deephaven-core to PyPi
+        id: publish-deephaven-core
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: py/server/build/wheel/
         continue-on-error: true
 
       - name: Publish deephaven-server to PyPi
+        id: publish-deephaven-server
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: py/embedded-server/build/wheel/
         continue-on-error: true
 
       - name: Publish pydeephaven to PyPi
+        id: publish-pydeephaven
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: py/client/build/wheel/
         continue-on-error: true
 
       - name: Publish pydeephaven-ticking to PyPi
+        id: publish-pydeephaven-ticking
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: py/client-ticking/build/wheel/
         continue-on-error: true
 
       - name: Publish @deephaven/jsapi-types to npmjs
+        id: publish-deephaven-jsapi-types
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         env:
           NODE_AUTH_TOKEN: ${{ secrets.DEEPHAVENBOT_NPM_TOKEN }}
         run: npm publish --provenance --tag latest web/client-api/types/build/deephaven-jsapi-types-*.tgz
         continue-on-error: true
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-deephaven-core.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-deephaven-core.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-deephaven-server.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-deephaven-server.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-pydeephaven.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-pydeephaven.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-pydeephaven-ticking.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-pydeephaven-ticking.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - uses: slackapi/slack-github-action@v2.0.0
+        if: ${{ steps.publish-deephaven-jsapi-types.outcome == 'failure' }}
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL_PUBLISH_FAILURE }}
+          webhook-type: webhook-trigger
+          payload: |
+            step_id: "${{ steps.publish-deephaven-jsapi-types.id }}"
+            action_url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

cpp-client/deephaven/dhclient/CMakeLists.txt

@@ -74,6 +74,7 @@ set(ALL_FILES
     include/private/deephaven/client/impl/util.h
 
     src/arrowutil/arrow_client_table.cc
+    src/arrowutil/arrow_column_source.cc
     include/private/deephaven/client/arrowutil/arrow_client_table.h
     include/private/deephaven/client/arrowutil/arrow_column_source.h
     include/private/deephaven/client/arrowutil/arrow_value_converter.h
@@ -107,6 +108,7 @@ set(ALL_FILES
     src/utility/table_maker.cc
 
     include/public/deephaven/client/utility/arrow_util.h
+    include/public/deephaven/client/utility/internal_types.h
     include/public/deephaven/client/utility/misc_types.h
     include/public/deephaven/client/utility/table_maker.h
 )

cpp-client/deephaven/dhclient/include/private/deephaven/client/arrowutil/arrow_column_source.h

@@ -22,9 +22,22 @@ namespace internal {
 // null-ness by determining whether the optional has a value.
 // kTimestamp is its own special case, where nullness is determined by the underlying nanos
 // being equal to Deephaven's NULL_LONG.
-// kLocalDate and kLocalTime are like kTimestamp except they resolve to different data types.
+// kLocalDate and kLocalTime are similar to kTimestamp in nullness except they resolve to different
+// data types.
 enum class ArrowProcessingStyle { kNormal, kBooleanOrString, kTimestamp, kLocalDate, kLocalTime };
 
+/**
+ * When 'array' has dynamic type arrow::TimestampArray or arrow::Time64Array, look at the
+ * underlying time resolution of the arrow type and calculate a conversion factor from that unit
+ * to nanoseconds. For example if the underlying time unit is arrow::TimeUnit::MILLI, then the
+ * conversion factor would be 1_000_000, meaning that one needs to multiply incoming millisecond
+ * values by one million to convert them to nanoseconds. If 'array' is not one of those types,
+ * return 1.
+ * @param array The Arrow array
+ * @return For supported time types, the conversion factor to nanoseconds. Otherwise, 1.
+ */
+size_t CalcTimeNanoScaleFactor(const arrow::Array &array);
+
 template<ArrowProcessingStyle Style, typename TColumnSourceBase, typename TArrowArray, typename TChunk>
 class GenericArrowColumnSource final : public TColumnSourceBase {
   using BooleanChunk = deephaven::dhcore::chunk::BooleanChunk;
@@ -37,7 +50,8 @@ class GenericArrowColumnSource final : public TColumnSourceBase {
   using UInt64Chunk = deephaven::dhcore::chunk::UInt64Chunk;
 
 public:
-  static std::shared_ptr<GenericArrowColumnSource> OfArrowArray(std::shared_ptr<TArrowArray> array) {
+  static std::shared_ptr<GenericArrowColumnSource>
+  OfArrowArray(std::shared_ptr<TArrowArray> array) {
     std::vector<std::shared_ptr<TArrowArray>> arrays{std::move(array)};
     return OfArrowArrayVec(std::move(arrays));
   }
@@ -48,7 +62,9 @@ class GenericArrowColumnSource final : public TColumnSourceBase {
   }
 
   explicit GenericArrowColumnSource(std::vector<std::shared_ptr<TArrowArray>> arrays) :
-      arrays_(std::move(arrays)) {}
+      arrays_(std::move(arrays)) {
+    time_nano_scale_factor_ = arrays_.empty() ? 1 : CalcTimeNanoScaleFactor(*arrays_.front());
+  }
 
   ~GenericArrowColumnSource() final = default;
 
@@ -67,13 +83,14 @@ class GenericArrowColumnSource final : public TColumnSourceBase {
 
     // This algorithm is a little tricky because the source data and RowSequence are both
     // segmented, perhaps in different ways.
-    auto *typed_dest = VerboseCast<TChunk*>(DEEPHAVEN_LOCATION_EXPR(dest_data));
+    auto *typed_dest = VerboseCast<TChunk *>(DEEPHAVEN_LOCATION_EXPR(dest_data));
     auto *destp = typed_dest->data();
     auto outerp = arrays_.begin();
     size_t src_segment_begin = 0;
     size_t src_segment_end = (*outerp)->length();
 
-    auto *null_destp = optional_dest_null_flags != nullptr ? optional_dest_null_flags->data() : nullptr;
+    auto *null_destp =
+        optional_dest_null_flags != nullptr ? optional_dest_null_flags->data() : nullptr;
 
     rows.ForEachInterval([&](uint64_t requested_segment_begin, uint64_t requested_segment_end) {
       while (true) {
@@ -147,11 +164,12 @@ class GenericArrowColumnSource final : public TColumnSourceBase {
           const auto *src_endp = innerp->raw_values() + relative_end;
 
           for (const auto *ip = src_beginp; ip != src_endp; ++ip) {
-            *destp = DateTime::FromNanos(*ip);
+            auto is_null = *ip == DeephavenTraits<int64_t>::kNullValue;
+            *destp = DateTime::FromNanos(is_null ? *ip : (*ip * time_nano_scale_factor_));
             ++destp;
 
             if (null_destp != nullptr) {
-              *null_destp = *ip == DeephavenTraits<int64_t>::kNullValue;
+              *null_destp = is_null;
               ++null_destp;
             }
           }
@@ -175,11 +193,12 @@ class GenericArrowColumnSource final : public TColumnSourceBase {
           const auto *src_endp = innerp->raw_values() + relative_end;
 
           for (const auto *ip = src_beginp; ip != src_endp; ++ip) {
-            *destp = LocalTime::FromNanos(*ip);
+            auto is_null = *ip == DeephavenTraits<int64_t>::kNullValue;
+            *destp = LocalTime::FromNanos(is_null ? *ip : (*ip * time_nano_scale_factor_));
             ++destp;
 
             if (null_destp != nullptr) {
-              *null_destp = *ip == DeephavenTraits<int64_t>::kNullValue;
+              *null_destp = is_null;
               ++null_destp;
             }
           }
@@ -200,6 +219,18 @@ class GenericArrowColumnSource final : public TColumnSourceBase {
 
 private:
   std::vector<std::shared_ptr<TArrowArray>> arrays_;
+  /**
+   * This value is valid for Style == ArrowProcessingStyle::kTimestamp and
+   * ArrowProcessingStyle::kLocalTime, and ignored for other ArrowProcessingStyle enumeration
+   * values.
+   *
+   * These ArrowProcessingStyles come into play when processing the arrow types
+   * arrow::TimestampType and arrow::Time64Type respectively.
+   *
+   * The value stores a conversion factor from whatever the input scale is to nanoseconds.
+   * For example, if the input timescale is milliseconds, this value will be 1_000_000.
+   */
+  size_t time_nano_scale_factor_ = 1;
 };
 }  // namespace internal
 

cpp-client/deephaven/dhclient/include/public/deephaven/client/utility/internal_types.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+ */
+#pragma once
+
+#include <arrow/flight/types.h>
+
+namespace deephaven::client::utility {
+/**
+ * For Deephaven use only
+ */
+namespace internal {
+/**
+ * This class exists only for the benefit of the unit tests. Our normal DateTime class has a
+ * native time resolution of nanoseconds. This class allows our unit tests to upload a
+ * DateTime having a different time unit so we can confirm that the client and server both handle
+ * it correctly.
+ */
+template<arrow::TimeUnit::type UNIT>
+struct InternalDateTime {
+  explicit InternalDateTime(int64_t value) : value_(value) {}
+
+  int64_t value_ = 0;
+};
+
+/**
+ * This class exists only for the benefit of the unit tests. Our normal LocalTime class has a
+ * native time resolution of nanoseconds. This class allows our unit tests to upload a
+ * LocalTime having a different time unit so we can confirm that the client and server both handle
+ * it correctly.
+ */
+template<arrow::TimeUnit::type UNIT>
+struct InternalLocalTime {
+  // Arrow Time64 only supports micro and nano units
+  static_assert(UNIT == arrow::TimeUnit::MICRO || UNIT == arrow::TimeUnit::NANO);
+
+  explicit InternalLocalTime(int64_t value) : value_(value) {}
+
+  int64_t value_ = 0;
+};
+}  // namespace internal
+} // namespace deephaven::client::utility

cpp-client/deephaven/dhclient/include/public/deephaven/client/utility/table_maker.h

@@ -19,6 +19,9 @@
 
 #include "deephaven/client/client.h"
 #include "deephaven/client/utility/arrow_util.h"
+#include "deephaven/client/utility/internal_types.h"
+#include "deephaven/client/utility/misc_types.h"
+#include "deephaven/dhcore/types.h"
 #include "deephaven/dhcore/utility/utility.h"
 #include "deephaven/third_party/fmt/format.h"
 
@@ -356,6 +359,38 @@ struct TypeConverterTraits<std::optional<T>> {
   }
 };
 
+template<arrow::TimeUnit::type UNIT>
+struct TypeConverterTraits<deephaven::client::utility::internal::InternalDateTime<UNIT>> {
+  static std::shared_ptr<arrow::DataType> GetDataType() {
+    return arrow::timestamp(UNIT, "UTC");
+  }
+  static arrow::TimestampBuilder GetBuilder() {
+    return arrow::TimestampBuilder(GetDataType(), arrow::default_memory_pool());
+  }
+  static int64_t Reinterpret(const deephaven::client::utility::internal::InternalDateTime<UNIT> &o) {
+    return o.value_;
+  }
+  static std::string_view GetDeephavenTypeName() {
+    return "java.time.ZonedDateTime";
+  }
+};
+
+template<arrow::TimeUnit::type UNIT>
+struct TypeConverterTraits<deephaven::client::utility::internal::InternalLocalTime<UNIT>> {
+  static std::shared_ptr<arrow::DataType> GetDataType() {
+    return arrow::time64(UNIT);
+  }
+  static arrow::Time64Builder GetBuilder() {
+    return arrow::Time64Builder(GetDataType(), arrow::default_memory_pool());
+  }
+  static int64_t Reinterpret(const deephaven::client::utility::internal::InternalLocalTime<UNIT> &o) {
+    return o.value_;
+  }
+  static std::string_view GetDeephavenTypeName() {
+    return "java.time.LocalTime";
+  }
+};
+
 template<typename T>
 TypeConverter TypeConverter::CreateNew(const std::vector<T> &values) {
   using deephaven::client::utility::OkOrThrow;