From 7d4a458bc74e8ed47fc4369207a62a02b0cb90dc Mon Sep 17 00:00:00 2001 From: StreamNative Bot <44651360+streamnativebot@users.noreply.github.com> Date: Sun, 19 May 2024 19:16:02 -0700 Subject: [PATCH] Docs sync 20240519 (#414) Co-authored-by: urfreespace --- .../io-activemq-sink.md | 2 +- .../io-activemq-sink.md | 2 +- .../io-activemq-source.md | 2 +- .../io-activemq-source.md | 2 +- .../{v3.0.4.5 => v3.0.4.6}/amqp-1-0-sink.md | 4 +- .../{v3.2.2.6 => v3.2.2.7}/amqp-1-0-sink.md | 4 +- .../{v3.0.4.5 => v3.0.4.6}/amqp-1-0-source.md | 4 +- .../{v3.2.2.6 => v3.2.2.7}/amqp-1-0-source.md | 4 +- .../aws-eventbridge-sink.md | 2 +- .../aws-eventbridge-sink.md | 2 +- .../{v3.0.4.5 => v3.0.4.6}/aws-lambda-sink.md | 28 +- .../{v3.2.2.6 => v3.2.2.7}/aws-lambda-sink.md | 2 +- .../{v3.0.4.5 => v3.0.4.6}/aws-s3-sink.md | 4 +- .../{v3.2.2.6 => v3.2.2.7}/aws-s3-sink.md | 4 +- .../azure-blob-storage-sink.md | 4 +- .../azure-blob-storage-sink.md | 4 +- .../google-bigquery-sink.md | 2 +- .../google-bigquery-sink.md | 2 +- .../google-bigquery-source.md | 2 +- .../google-bigquery-source.md | 2 +- .../google-cloud-storage-sink.md | 4 +- .../google-cloud-storage-sink.md | 4 +- .../google-pubsub-sink.md | 2 +- .../google-pubsub-sink.md | 2 +- .../google-pubsub-source.md | 2 +- .../google-pubsub-source.md | 2 +- .../{v3.2.2.6 => v3.0.4.6}/lakehouse-sink.md | 22 +- .../{v3.0.4.5 => v3.2.2.7}/lakehouse-sink.md | 22 +- .../lakehouse-source.md | 14 +- .../lakehouse-source.md | 14 +- .../{v3.2.2.6 => v3.0.4.6}/pinecone-sink.md | 2 +- .../pinecone-sink/v3.2.2.7/pinecone-sink.md | 262 ++++++++++++++++++ .../{v3.0.4.5 => v3.0.4.6}/snowflake-sink.md | 25 +- .../{v3.2.2.6 => v3.2.2.7}/snowflake-sink.md | 25 +- .../{v3.0.4.5 => v3.0.4.6}/sqs-sink.md | 2 +- .../{v3.2.2.6 => v3.2.2.7}/sqs-sink.md | 2 +- .../{v3.0.4.5 => v3.0.4.6}/sqs-source.md | 2 +- .../{v3.2.2.6 => v3.2.2.7}/sqs-source.md | 2 +- 38 files changed, 394 insertions(+), 98 deletions(-) rename connectors/activemq-sink/{v3.0.4.5 => v3.0.4.6}/io-activemq-sink.md (99%) rename connectors/activemq-sink/{v3.2.2.6 => v3.2.2.7}/io-activemq-sink.md (99%) rename connectors/activemq-source/{v3.0.4.5 => v3.0.4.6}/io-activemq-source.md (99%) rename connectors/activemq-source/{v3.2.2.6 => v3.2.2.7}/io-activemq-source.md (99%) rename connectors/amqp-1-0-sink/{v3.0.4.5 => v3.0.4.6}/amqp-1-0-sink.md (99%) rename connectors/amqp-1-0-sink/{v3.2.2.6 => v3.2.2.7}/amqp-1-0-sink.md (99%) rename connectors/amqp-1-0-source/{v3.0.4.5 => v3.0.4.6}/amqp-1-0-source.md (99%) rename connectors/amqp-1-0-source/{v3.2.2.6 => v3.2.2.7}/amqp-1-0-source.md (99%) rename connectors/aws-eventbridge-sink/{v3.0.4.5 => v3.0.4.6}/aws-eventbridge-sink.md (99%) rename connectors/aws-eventbridge-sink/{v3.2.2.6 => v3.2.2.7}/aws-eventbridge-sink.md (99%) rename connectors/aws-lambda-sink/{v3.0.4.5 => v3.0.4.6}/aws-lambda-sink.md (94%) rename connectors/aws-lambda-sink/{v3.2.2.6 => v3.2.2.7}/aws-lambda-sink.md (99%) rename connectors/aws-s3-sink/{v3.0.4.5 => v3.0.4.6}/aws-s3-sink.md (99%) rename connectors/aws-s3-sink/{v3.2.2.6 => v3.2.2.7}/aws-s3-sink.md (99%) rename connectors/azure-blob-storage-sink/{v3.0.4.5 => v3.0.4.6}/azure-blob-storage-sink.md (99%) rename connectors/azure-blob-storage-sink/{v3.2.2.6 => v3.2.2.7}/azure-blob-storage-sink.md (99%) rename connectors/google-bigquery-sink/{v3.0.4.5 => v3.0.4.6}/google-bigquery-sink.md (99%) rename connectors/google-bigquery-sink/{v3.2.2.6 => v3.2.2.7}/google-bigquery-sink.md (99%) rename connectors/google-bigquery-source/{v3.0.4.5 => v3.0.4.6}/google-bigquery-source.md (99%) rename connectors/google-bigquery-source/{v3.2.2.6 => v3.2.2.7}/google-bigquery-source.md (99%) rename connectors/google-cloud-storage-sink/{v3.0.4.5 => v3.0.4.6}/google-cloud-storage-sink.md (99%) rename connectors/google-cloud-storage-sink/{v3.2.2.6 => v3.2.2.7}/google-cloud-storage-sink.md (99%) rename connectors/google-pubsub-sink/{v3.2.2.6 => v3.0.4.6}/google-pubsub-sink.md (99%) rename connectors/google-pubsub-sink/{v3.0.4.5 => v3.2.2.7}/google-pubsub-sink.md (99%) rename connectors/google-pubsub-source/{v3.0.4.5 => v3.0.4.6}/google-pubsub-source.md (99%) rename connectors/google-pubsub-source/{v3.2.2.6 => v3.2.2.7}/google-pubsub-source.md (99%) rename connectors/lakehouse-sink/{v3.2.2.6 => v3.0.4.6}/lakehouse-sink.md (96%) rename connectors/lakehouse-sink/{v3.0.4.5 => v3.2.2.7}/lakehouse-sink.md (96%) rename connectors/lakehouse-source/{v3.2.2.6 => v3.0.4.6}/lakehouse-source.md (95%) rename connectors/lakehouse-source/{v3.0.4.5 => v3.2.2.7}/lakehouse-source.md (95%) rename connectors/pinecone-sink/{v3.2.2.6 => v3.0.4.6}/pinecone-sink.md (99%) create mode 100644 connectors/pinecone-sink/v3.2.2.7/pinecone-sink.md rename connectors/snowflake-sink/{v3.0.4.5 => v3.0.4.6}/snowflake-sink.md (85%) rename connectors/snowflake-sink/{v3.2.2.6 => v3.2.2.7}/snowflake-sink.md (86%) rename connectors/sqs-sink/{v3.0.4.5 => v3.0.4.6}/sqs-sink.md (99%) rename connectors/sqs-sink/{v3.2.2.6 => v3.2.2.7}/sqs-sink.md (99%) rename connectors/sqs-source/{v3.0.4.5 => v3.0.4.6}/sqs-source.md (99%) rename connectors/sqs-source/{v3.2.2.6 => v3.2.2.7}/sqs-source.md (99%) diff --git a/connectors/activemq-sink/v3.0.4.5/io-activemq-sink.md b/connectors/activemq-sink/v3.0.4.6/io-activemq-sink.md similarity index 99% rename from connectors/activemq-sink/v3.0.4.5/io-activemq-sink.md rename to connectors/activemq-sink/v3.0.4.6/io-activemq-sink.md index b0a340f0..f04a1a33 100644 --- a/connectors/activemq-sink/v3.0.4.5/io-activemq-sink.md +++ b/connectors/activemq-sink/v3.0.4.6/io-activemq-sink.md @@ -11,7 +11,7 @@ tags: apache-pulsar,pulsar-io,source-connector,sink-connector,apache-activemq alias: ActiveMQ sink connector features: ["ActiveMQ Connector integrates Apache Pulsar with Apache ActiveMQ. "] icon: "/images/connectors/activemq_logo_white_vertical.jpg" -download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-activemq support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" diff --git a/connectors/activemq-sink/v3.2.2.6/io-activemq-sink.md b/connectors/activemq-sink/v3.2.2.7/io-activemq-sink.md similarity index 99% rename from connectors/activemq-sink/v3.2.2.6/io-activemq-sink.md rename to connectors/activemq-sink/v3.2.2.7/io-activemq-sink.md index 789567ef..17aebe5f 100644 --- a/connectors/activemq-sink/v3.2.2.6/io-activemq-sink.md +++ b/connectors/activemq-sink/v3.2.2.7/io-activemq-sink.md @@ -11,7 +11,7 @@ tags: apache-pulsar,pulsar-io,source-connector,sink-connector,apache-activemq alias: ActiveMQ sink connector features: ["ActiveMQ Connector integrates Apache Pulsar with Apache ActiveMQ. "] icon: "/images/connectors/activemq_logo_white_vertical.jpg" -download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-activemq support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" diff --git a/connectors/activemq-source/v3.0.4.5/io-activemq-source.md b/connectors/activemq-source/v3.0.4.6/io-activemq-source.md similarity index 99% rename from connectors/activemq-source/v3.0.4.5/io-activemq-source.md rename to connectors/activemq-source/v3.0.4.6/io-activemq-source.md index d2634195..ad742f97 100644 --- a/connectors/activemq-source/v3.0.4.5/io-activemq-source.md +++ b/connectors/activemq-source/v3.0.4.6/io-activemq-source.md @@ -11,7 +11,7 @@ tags: apache-pulsar,pulsar-io,source-connector,sink-connector,apache-activemq alias: ActiveMQ source connector features: ["ActiveMQ Connector integrates Apache Pulsar with Apache ActiveMQ. "] icon: "/images/connectors/activemq_logo_white_vertical.jpg" -download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-activemq support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" diff --git a/connectors/activemq-source/v3.2.2.6/io-activemq-source.md b/connectors/activemq-source/v3.2.2.7/io-activemq-source.md similarity index 99% rename from connectors/activemq-source/v3.2.2.6/io-activemq-source.md rename to connectors/activemq-source/v3.2.2.7/io-activemq-source.md index 69a82dca..eddfaed2 100644 --- a/connectors/activemq-source/v3.2.2.6/io-activemq-source.md +++ b/connectors/activemq-source/v3.2.2.7/io-activemq-source.md @@ -11,7 +11,7 @@ tags: apache-pulsar,pulsar-io,source-connector,sink-connector,apache-activemq alias: ActiveMQ source connector features: ["ActiveMQ Connector integrates Apache Pulsar with Apache ActiveMQ. "] icon: "/images/connectors/activemq_logo_white_vertical.jpg" -download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-activemq support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" diff --git a/connectors/amqp-1-0-sink/v3.0.4.5/amqp-1-0-sink.md b/connectors/amqp-1-0-sink/v3.0.4.6/amqp-1-0-sink.md similarity index 99% rename from connectors/amqp-1-0-sink/v3.0.4.5/amqp-1-0-sink.md rename to connectors/amqp-1-0-sink/v3.0.4.6/amqp-1-0-sink.md index f33ef613..9984a4af 100644 --- a/connectors/amqp-1-0-sink/v3.0.4.5/amqp-1-0-sink.md +++ b/connectors/amqp-1-0-sink/v3.0.4.6/amqp-1-0-sink.md @@ -11,7 +11,7 @@ tags: alias: AMQP 1.0 Sink Connector features: ["support sink/source for AMQP version 1.0.0"] icon: "/images/connectors/amqp-logo.png" -download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-amqp-1-0 support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -27,7 +27,7 @@ id: "amqp-1-0-sink" The AMQP 1.0 sink connector pulls messages from Pulsar topics and persists messages to [AMQP 1.0](https://www.amqp.org/). -![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.0.4.5/docs/amqp-1-0-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.0.4.6/docs/amqp-1-0-sink.png) ## Quick start diff --git a/connectors/amqp-1-0-sink/v3.2.2.6/amqp-1-0-sink.md b/connectors/amqp-1-0-sink/v3.2.2.7/amqp-1-0-sink.md similarity index 99% rename from connectors/amqp-1-0-sink/v3.2.2.6/amqp-1-0-sink.md rename to connectors/amqp-1-0-sink/v3.2.2.7/amqp-1-0-sink.md index c161f06d..04c24e04 100644 --- a/connectors/amqp-1-0-sink/v3.2.2.6/amqp-1-0-sink.md +++ b/connectors/amqp-1-0-sink/v3.2.2.7/amqp-1-0-sink.md @@ -11,7 +11,7 @@ tags: alias: AMQP 1.0 Sink Connector features: ["support sink/source for AMQP version 1.0.0"] icon: "/images/connectors/amqp-logo.png" -download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-amqp-1-0 support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -27,7 +27,7 @@ id: "amqp-1-0-sink" The AMQP 1.0 sink connector pulls messages from Pulsar topics and persists messages to [AMQP 1.0](https://www.amqp.org/). -![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.2.2.6/docs/amqp-1-0-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.2.2.7/docs/amqp-1-0-sink.png) ## Quick start diff --git a/connectors/amqp-1-0-source/v3.0.4.5/amqp-1-0-source.md b/connectors/amqp-1-0-source/v3.0.4.6/amqp-1-0-source.md similarity index 99% rename from connectors/amqp-1-0-source/v3.0.4.5/amqp-1-0-source.md rename to connectors/amqp-1-0-source/v3.0.4.6/amqp-1-0-source.md index e9b20d64..fa134880 100644 --- a/connectors/amqp-1-0-source/v3.0.4.5/amqp-1-0-source.md +++ b/connectors/amqp-1-0-source/v3.0.4.6/amqp-1-0-source.md @@ -11,7 +11,7 @@ tags: alias: AMQP 1.0 Source Connector features: ["support sink/source for AMQP version 1.0.0"] icon: "/images/connectors/amqp-logo.png" -download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-amqp-1-0 support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -27,7 +27,7 @@ id: "amqp-1-0-source" The AMQP 1.0 source connector receives messages from [AMQP 1.0](https://www.amqp.org/) and writes messages to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.0.4.5/docs/amqp-1-0-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.0.4.6/docs/amqp-1-0-source.png) ## Quick start diff --git a/connectors/amqp-1-0-source/v3.2.2.6/amqp-1-0-source.md b/connectors/amqp-1-0-source/v3.2.2.7/amqp-1-0-source.md similarity index 99% rename from connectors/amqp-1-0-source/v3.2.2.6/amqp-1-0-source.md rename to connectors/amqp-1-0-source/v3.2.2.7/amqp-1-0-source.md index f67bb686..ede73963 100644 --- a/connectors/amqp-1-0-source/v3.2.2.6/amqp-1-0-source.md +++ b/connectors/amqp-1-0-source/v3.2.2.7/amqp-1-0-source.md @@ -11,7 +11,7 @@ tags: alias: AMQP 1.0 Source Connector features: ["support sink/source for AMQP version 1.0.0"] icon: "/images/connectors/amqp-logo.png" -download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-amqp-1-0 support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -27,7 +27,7 @@ id: "amqp-1-0-source" The AMQP 1.0 source connector receives messages from [AMQP 1.0](https://www.amqp.org/) and writes messages to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.2.2.6/docs/amqp-1-0-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.2.2.7/docs/amqp-1-0-source.png) ## Quick start diff --git a/connectors/aws-eventbridge-sink/v3.0.4.5/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v3.0.4.6/aws-eventbridge-sink.md similarity index 99% rename from connectors/aws-eventbridge-sink/v3.0.4.5/aws-eventbridge-sink.md rename to connectors/aws-eventbridge-sink/v3.0.4.6/aws-eventbridge-sink.md index 44d68730..c94663fc 100644 --- a/connectors/aws-eventbridge-sink/v3.0.4.5/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v3.0.4.6/aws-eventbridge-sink.md @@ -26,7 +26,7 @@ id: "aws-eventbridge-sink" The [Amazon EventBridge](https://aws.amazon.com/eventbridge/) sink connector pulls data from Pulsar topics and persists data to Amazon EventBridge. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-eventbridge/v3.0.4.5/docs/aws-eventbridge-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-eventbridge/v3.0.4.6/docs/aws-eventbridge-sink.png) ## Quick start diff --git a/connectors/aws-eventbridge-sink/v3.2.2.6/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v3.2.2.7/aws-eventbridge-sink.md similarity index 99% rename from connectors/aws-eventbridge-sink/v3.2.2.6/aws-eventbridge-sink.md rename to connectors/aws-eventbridge-sink/v3.2.2.7/aws-eventbridge-sink.md index c513183f..fffcebe7 100644 --- a/connectors/aws-eventbridge-sink/v3.2.2.6/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v3.2.2.7/aws-eventbridge-sink.md @@ -26,7 +26,7 @@ id: "aws-eventbridge-sink" The [Amazon EventBridge](https://aws.amazon.com/eventbridge/) sink connector pulls data from Pulsar topics and persists data to Amazon EventBridge. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-eventbridge/v3.2.2.6/docs/aws-eventbridge-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-eventbridge/v3.2.2.7/docs/aws-eventbridge-sink.png) ## Quick start diff --git a/connectors/aws-lambda-sink/v3.0.4.5/aws-lambda-sink.md b/connectors/aws-lambda-sink/v3.0.4.6/aws-lambda-sink.md similarity index 94% rename from connectors/aws-lambda-sink/v3.0.4.5/aws-lambda-sink.md rename to connectors/aws-lambda-sink/v3.0.4.6/aws-lambda-sink.md index 5cfaab5e..2b9075c6 100644 --- a/connectors/aws-lambda-sink/v3.0.4.5/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v3.0.4.6/aws-lambda-sink.md @@ -25,7 +25,7 @@ id: "aws-lambda-sink" The [AWS Lambda](https://aws.amazon.com/lambda/) sink connector is a [Pulsar IO connector](http://pulsar.apache.org/docs/en/next/io-overview/) for pulling messages from Pulsar topics to AWS Lambda to invoke Lambda functions. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-lambda/v3.0.4.5/docs/lambda-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-lambda/v3.0.4.6/docs/lambda-sink.png) # How to get @@ -35,7 +35,7 @@ This section describes how to build the AWS Lambda sink connector. You can get the AWS Lambda sink connector using one of the following methods if you use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run connectors in a cluster. -- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-aws-lambda/releases/download/v3.0.4.5/pulsar-io-google-pubsub-3.0.4.5.nar). +- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-aws-lambda/releases/download/v3.0.4.6/pulsar-io-google-pubsub-3.0.4.6.nar). - Build it from the source code. @@ -57,7 +57,7 @@ To build the AWS Lambda sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-aws-lambda-3.0.4.5.nar + pulsar-io-aws-lambda-3.0.4.6.nar ``` ## Work with Function Mesh @@ -93,7 +93,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-aws-lambda-topic" ], - "archive": "connectors/pulsar-io-aws-lambda-3.0.4.5.nar", + "archive": "connectors/pulsar-io-aws-lambda-3.0.4.6.nar", "parallelism": 1, "configs": { @@ -115,7 +115,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you name: "aws-lambda-sink" inputs: - "test-aws-lambda-topic" - archive: "connectors/pulsar-io-aws-lambda-3.0.4.5.nar" + archive: "connectors/pulsar-io-aws-lambda-3.0.4.6.nar" parallelism: 1 configs: @@ -139,7 +139,7 @@ kind: Sink metadata: name: aws-lambda-sink-sample spec: - image: streamnative/pulsar-io-aws-lambda:3.0.4.5 + image: streamnative/pulsar-io-aws-lambda:3.0.4.6 className: org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink replicas: 1 input: @@ -163,7 +163,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-aws-lambda-3.0.4.5.nar + jar: connectors/pulsar-io-aws-lambda-3.0.4.6.nar clusterName: test-pulsar autoAck: true ``` @@ -186,7 +186,7 @@ This example shows how to create an AWS Lambda sink connector on a Pulsar cluste ```bash PULSAR_HOME/bin/pulsar-admin sinks create \ ---archive pulsar-io-aws-lambda-3.0.4.5.nar \ +--archive pulsar-io-aws-lambda-3.0.4.6.nar \ --sink-config-file aws-lambda-sink-config.yaml \ --classname org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink \ --name aws-lambda-sink @@ -209,8 +209,8 @@ This example describes how to use the AWS Lambda sink connector to pull messages 2. Copy the NAR package of the AWS Lambda connector to the Pulsar connectors directory. ``` - cp pulsar-io-aws-lambda-3.0.4.5.nar - PULSAR_HOME/connectors/pulsar-io-aws-lambda-3.0.4.5.nar + cp pulsar-io-aws-lambda-3.0.4.6.nar + PULSAR_HOME/connectors/pulsar-io-aws-lambda-3.0.4.6.nar ``` 3. Start Pulsar in standalone mode. @@ -244,8 +244,8 @@ This example explains how to create an AWS Lambda sink connector in an on-premis 1. Copy the NAR package of the AWS Lambda connector to the Pulsar connectors directory. ``` - cp pulsar-io-aws-lambda-3.0.4.5.nar - PULSAR_HOME/connectors/pulsar-io-aws-lambda-3.0.4.5.nar + cp pulsar-io-aws-lambda-3.0.4.6.nar + PULSAR_HOME/connectors/pulsar-io-aws-lambda-3.0.4.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -297,7 +297,7 @@ This example describes how to create an AWS Lambda sink connector for a Kuberbet metadata: name: aws-lambda-sink-sample spec: - image: streamnative/pulsar-io-aws-lambda:3.0.4.5 + image: streamnative/pulsar-io-aws-lambda:3.0.4.6 className: org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink replicas: 1 input: @@ -321,7 +321,7 @@ This example describes how to create an AWS Lambda sink connector for a Kuberbet cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-aws-lambda-3.0.4.5.nar + jar: connectors/pulsar-io-aws-lambda-3.0.4.6.nar clusterName: test-pulsar autoAck: true ``` diff --git a/connectors/aws-lambda-sink/v3.2.2.6/aws-lambda-sink.md b/connectors/aws-lambda-sink/v3.2.2.7/aws-lambda-sink.md similarity index 99% rename from connectors/aws-lambda-sink/v3.2.2.6/aws-lambda-sink.md rename to connectors/aws-lambda-sink/v3.2.2.7/aws-lambda-sink.md index 6f0dbd96..ffe6b64d 100644 --- a/connectors/aws-lambda-sink/v3.2.2.6/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v3.2.2.7/aws-lambda-sink.md @@ -27,7 +27,7 @@ The [AWS Lambda](https://aws.amazon.com/lambda/) sink connector is a [Pulsar IO connector](http://pulsar.apache.org/docs/en/next/io-overview/) for pulling messages from Pulsar topics to AWS Lambda to invoke Lambda functions. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-lambda/v3.2.2.6/docs/lambda-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-lambda/v3.2.2.7/docs/lambda-sink.png) ## Quick start diff --git a/connectors/aws-s3-sink/v3.0.4.5/aws-s3-sink.md b/connectors/aws-s3-sink/v3.0.4.6/aws-s3-sink.md similarity index 99% rename from connectors/aws-s3-sink/v3.0.4.5/aws-s3-sink.md rename to connectors/aws-s3-sink/v3.0.4.6/aws-s3-sink.md index d16ced75..7d611bac 100644 --- a/connectors/aws-s3-sink/v3.0.4.5/aws-s3-sink.md +++ b/connectors/aws-s3-sink/v3.0.4.6/aws-s3-sink.md @@ -11,7 +11,7 @@ tags: alias: AWS S3 Sink Connector features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-cloud-storage support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "aws-s3-sink" The [AWS S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html) sink connector pulls data from Pulsar topics and persists data to AWS S3 buckets. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.0.4.5/docs/aws-s3-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.0.4.6/docs/aws-s3-sink.png) ## Quick start diff --git a/connectors/aws-s3-sink/v3.2.2.6/aws-s3-sink.md b/connectors/aws-s3-sink/v3.2.2.7/aws-s3-sink.md similarity index 99% rename from connectors/aws-s3-sink/v3.2.2.6/aws-s3-sink.md rename to connectors/aws-s3-sink/v3.2.2.7/aws-s3-sink.md index 768e7652..b369ec3d 100644 --- a/connectors/aws-s3-sink/v3.2.2.6/aws-s3-sink.md +++ b/connectors/aws-s3-sink/v3.2.2.7/aws-s3-sink.md @@ -11,7 +11,7 @@ tags: alias: AWS S3 Sink Connector features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-cloud-storage support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "aws-s3-sink" The [AWS S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html) sink connector pulls data from Pulsar topics and persists data to AWS S3 buckets. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.2.2.6/docs/aws-s3-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.2.2.7/docs/aws-s3-sink.png) ## Quick start diff --git a/connectors/azure-blob-storage-sink/v3.0.4.5/azure-blob-storage-sink.md b/connectors/azure-blob-storage-sink/v3.0.4.6/azure-blob-storage-sink.md similarity index 99% rename from connectors/azure-blob-storage-sink/v3.0.4.5/azure-blob-storage-sink.md rename to connectors/azure-blob-storage-sink/v3.0.4.6/azure-blob-storage-sink.md index bf346421..7b867f80 100644 --- a/connectors/azure-blob-storage-sink/v3.0.4.5/azure-blob-storage-sink.md +++ b/connectors/azure-blob-storage-sink/v3.0.4.6/azure-blob-storage-sink.md @@ -11,7 +11,7 @@ tags: alias: Azure Blob Storage Sink Connector features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-cloud-storage support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "azure-blob-storage-sink" The [Azure Blob Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-overview) sink connector pulls data from Pulsar topics and persists data to Azure Blob Storage containers. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.0.4.5/docs/azure-blob-storage-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.0.4.6/docs/azure-blob-storage-sink.png) ## Quick start diff --git a/connectors/azure-blob-storage-sink/v3.2.2.6/azure-blob-storage-sink.md b/connectors/azure-blob-storage-sink/v3.2.2.7/azure-blob-storage-sink.md similarity index 99% rename from connectors/azure-blob-storage-sink/v3.2.2.6/azure-blob-storage-sink.md rename to connectors/azure-blob-storage-sink/v3.2.2.7/azure-blob-storage-sink.md index c432afe3..8494844e 100644 --- a/connectors/azure-blob-storage-sink/v3.2.2.6/azure-blob-storage-sink.md +++ b/connectors/azure-blob-storage-sink/v3.2.2.7/azure-blob-storage-sink.md @@ -11,7 +11,7 @@ tags: alias: Azure Blob Storage Sink Connector features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-cloud-storage support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "azure-blob-storage-sink" The [Azure Blob Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-overview) sink connector pulls data from Pulsar topics and persists data to Azure Blob Storage containers. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.2.2.6/docs/azure-blob-storage-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.2.2.7/docs/azure-blob-storage-sink.png) ## Quick start diff --git a/connectors/google-bigquery-sink/v3.0.4.5/google-bigquery-sink.md b/connectors/google-bigquery-sink/v3.0.4.6/google-bigquery-sink.md similarity index 99% rename from connectors/google-bigquery-sink/v3.0.4.5/google-bigquery-sink.md rename to connectors/google-bigquery-sink/v3.0.4.6/google-bigquery-sink.md index cb35dcea..dd040683 100644 --- a/connectors/google-bigquery-sink/v3.0.4.5/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v3.0.4.6/google-bigquery-sink.md @@ -25,7 +25,7 @@ id: "google-bigquery-sink" The [Google Cloud BigQuery](https://cloud.google.com/bigquery) sink connector pulls data from Pulsar topics and persists data to Google Cloud BigQuery tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.0.4.5/docs/google-bigquery-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.0.4.6/docs/google-bigquery-sink.png) ## Quick start diff --git a/connectors/google-bigquery-sink/v3.2.2.6/google-bigquery-sink.md b/connectors/google-bigquery-sink/v3.2.2.7/google-bigquery-sink.md similarity index 99% rename from connectors/google-bigquery-sink/v3.2.2.6/google-bigquery-sink.md rename to connectors/google-bigquery-sink/v3.2.2.7/google-bigquery-sink.md index cb348c9a..89c31685 100644 --- a/connectors/google-bigquery-sink/v3.2.2.6/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v3.2.2.7/google-bigquery-sink.md @@ -25,7 +25,7 @@ id: "google-bigquery-sink" The [Google Cloud BigQuery](https://cloud.google.com/bigquery) sink connector pulls data from Pulsar topics and persists data to Google Cloud BigQuery tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.2.2.6/docs/google-bigquery-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.2.2.7/docs/google-bigquery-sink.png) ## Quick start diff --git a/connectors/google-bigquery-source/v3.0.4.5/google-bigquery-source.md b/connectors/google-bigquery-source/v3.0.4.6/google-bigquery-source.md similarity index 99% rename from connectors/google-bigquery-source/v3.0.4.5/google-bigquery-source.md rename to connectors/google-bigquery-source/v3.0.4.6/google-bigquery-source.md index ce4c9846..83761f72 100644 --- a/connectors/google-bigquery-source/v3.0.4.5/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v3.0.4.6/google-bigquery-source.md @@ -25,7 +25,7 @@ id: "google-bigquery-source" The [Google Cloud BigQuery](https://cloud.google.com/bigquery) Source Connector feeds data from Google Cloud BigQuery tables and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.0.4.5/docs/google-bigquery-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.0.4.6/docs/google-bigquery-source.png) ## Quick start diff --git a/connectors/google-bigquery-source/v3.2.2.6/google-bigquery-source.md b/connectors/google-bigquery-source/v3.2.2.7/google-bigquery-source.md similarity index 99% rename from connectors/google-bigquery-source/v3.2.2.6/google-bigquery-source.md rename to connectors/google-bigquery-source/v3.2.2.7/google-bigquery-source.md index 9e9e0938..7ad76332 100644 --- a/connectors/google-bigquery-source/v3.2.2.6/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v3.2.2.7/google-bigquery-source.md @@ -25,7 +25,7 @@ id: "google-bigquery-source" The [Google Cloud BigQuery](https://cloud.google.com/bigquery) Source Connector feeds data from Google Cloud BigQuery tables and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.2.2.6/docs/google-bigquery-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.2.2.7/docs/google-bigquery-source.png) ## Quick start diff --git a/connectors/google-cloud-storage-sink/v3.0.4.5/google-cloud-storage-sink.md b/connectors/google-cloud-storage-sink/v3.0.4.6/google-cloud-storage-sink.md similarity index 99% rename from connectors/google-cloud-storage-sink/v3.0.4.5/google-cloud-storage-sink.md rename to connectors/google-cloud-storage-sink/v3.0.4.6/google-cloud-storage-sink.md index 0f0ed4c9..071fd619 100644 --- a/connectors/google-cloud-storage-sink/v3.0.4.5/google-cloud-storage-sink.md +++ b/connectors/google-cloud-storage-sink/v3.0.4.6/google-cloud-storage-sink.md @@ -11,7 +11,7 @@ tags: alias: Google Cloud Storage Sink Connector features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-cloud-storage support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "google-cloud-storage-sink" The [Google Cloud Storage](https://cloud.google.com/storage/docs) sink connector pulls data from Pulsar topics and persists data to Google Cloud Storage buckets. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.0.4.5/docs/google-cloud-storage-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.0.4.6/docs/google-cloud-storage-sink.png) ## Quick start diff --git a/connectors/google-cloud-storage-sink/v3.2.2.6/google-cloud-storage-sink.md b/connectors/google-cloud-storage-sink/v3.2.2.7/google-cloud-storage-sink.md similarity index 99% rename from connectors/google-cloud-storage-sink/v3.2.2.6/google-cloud-storage-sink.md rename to connectors/google-cloud-storage-sink/v3.2.2.7/google-cloud-storage-sink.md index 6754cfbc..3ddf02f5 100644 --- a/connectors/google-cloud-storage-sink/v3.2.2.6/google-cloud-storage-sink.md +++ b/connectors/google-cloud-storage-sink/v3.2.2.7/google-cloud-storage-sink.md @@ -11,7 +11,7 @@ tags: alias: Google Cloud Storage Sink Connector features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-cloud-storage support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "google-cloud-storage-sink" The [Google Cloud Storage](https://cloud.google.com/storage/docs) sink connector pulls data from Pulsar topics and persists data to Google Cloud Storage buckets. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.2.2.6/docs/google-cloud-storage-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.2.2.7/docs/google-cloud-storage-sink.png) ## Quick start diff --git a/connectors/google-pubsub-sink/v3.2.2.6/google-pubsub-sink.md b/connectors/google-pubsub-sink/v3.0.4.6/google-pubsub-sink.md similarity index 99% rename from connectors/google-pubsub-sink/v3.2.2.6/google-pubsub-sink.md rename to connectors/google-pubsub-sink/v3.0.4.6/google-pubsub-sink.md index 6b9e0ce0..895654da 100644 --- a/connectors/google-pubsub-sink/v3.2.2.6/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v3.0.4.6/google-pubsub-sink.md @@ -25,7 +25,7 @@ id: "google-pubsub-sink" The [Google Cloud PubSub](https://cloud.google.com/pubsub) sink connector pulls data from Pulsar topics and persists data to Google Cloud PubSub tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.2.2.6/docs/google-pubsub-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.0.4.6/docs/google-pubsub-sink.png) ## Quick start diff --git a/connectors/google-pubsub-sink/v3.0.4.5/google-pubsub-sink.md b/connectors/google-pubsub-sink/v3.2.2.7/google-pubsub-sink.md similarity index 99% rename from connectors/google-pubsub-sink/v3.0.4.5/google-pubsub-sink.md rename to connectors/google-pubsub-sink/v3.2.2.7/google-pubsub-sink.md index 9ef64cec..cf0e1202 100644 --- a/connectors/google-pubsub-sink/v3.0.4.5/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v3.2.2.7/google-pubsub-sink.md @@ -25,7 +25,7 @@ id: "google-pubsub-sink" The [Google Cloud PubSub](https://cloud.google.com/pubsub) sink connector pulls data from Pulsar topics and persists data to Google Cloud PubSub tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.0.4.5/docs/google-pubsub-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.2.2.7/docs/google-pubsub-sink.png) ## Quick start diff --git a/connectors/google-pubsub-source/v3.0.4.5/google-pubsub-source.md b/connectors/google-pubsub-source/v3.0.4.6/google-pubsub-source.md similarity index 99% rename from connectors/google-pubsub-source/v3.0.4.5/google-pubsub-source.md rename to connectors/google-pubsub-source/v3.0.4.6/google-pubsub-source.md index 3cb9f60e..f1cbd40b 100644 --- a/connectors/google-pubsub-source/v3.0.4.5/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v3.0.4.6/google-pubsub-source.md @@ -25,7 +25,7 @@ id: "google-pubsub-source" The [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) source connector feeds data from Google Cloud Pub/Sub topics and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.0.4.5/docs/google-pubsub-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.0.4.6/docs/google-pubsub-source.png) ## Quick start diff --git a/connectors/google-pubsub-source/v3.2.2.6/google-pubsub-source.md b/connectors/google-pubsub-source/v3.2.2.7/google-pubsub-source.md similarity index 99% rename from connectors/google-pubsub-source/v3.2.2.6/google-pubsub-source.md rename to connectors/google-pubsub-source/v3.2.2.7/google-pubsub-source.md index 7b84fefd..37ab747c 100644 --- a/connectors/google-pubsub-source/v3.2.2.6/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v3.2.2.7/google-pubsub-source.md @@ -25,7 +25,7 @@ id: "google-pubsub-source" The [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) source connector feeds data from Google Cloud Pub/Sub topics and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.2.2.6/docs/google-pubsub-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.2.2.7/docs/google-pubsub-source.png) ## Quick start diff --git a/connectors/lakehouse-sink/v3.2.2.6/lakehouse-sink.md b/connectors/lakehouse-sink/v3.0.4.6/lakehouse-sink.md similarity index 96% rename from connectors/lakehouse-sink/v3.2.2.6/lakehouse-sink.md rename to connectors/lakehouse-sink/v3.0.4.6/lakehouse-sink.md index b68a442b..e5133b87 100644 --- a/connectors/lakehouse-sink/v3.2.2.6/lakehouse-sink.md +++ b/connectors/lakehouse-sink/v3.0.4.6/lakehouse-sink.md @@ -11,7 +11,7 @@ tags: alias: Lakehouse Sink Connector features: ["pulsar lakehouse connector"] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-lakehouse support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "lakehouse-sink" The Lakehouse sink connector (including the [Hudi](https://hudi.apache.org), [Iceberg](https://iceberg.apache.org/), and [Delta Lake](https://delta.io/) sink connectors) fetches data from a Pulsar topic and saves data to the Lakehouse tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.2.2.6/docs/lakehouse-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.0.4.6/docs/lakehouse-sink.png) # How to get @@ -62,7 +62,7 @@ To build the Lakehouse sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-lakehouse-3.2.2.6.nar + pulsar-io-lakehouse-3.0.4.6.nar ``` # How to configure @@ -147,7 +147,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-3.2.2.6.nar", + "archive": "connectors/pulsar-io-hudi-3.0.4.6.nar", "processingGuarantees": "EFFECTIVELY_ONCE", "parallelism": 1, "configs": { @@ -171,7 +171,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-3.2.2.6-cloud.nar", + "archive": "connectors/pulsar-io-hudi-3.0.4.6-cloud.nar", "parallelism": 1, "processingGuarantees": "EFFECTIVELY_ONCE", "configs": { @@ -199,7 +199,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.2.2.6.nar", + "archive": "connectors/pulsar-io-lakehouse-3.0.4.6.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -227,7 +227,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.2.2.6-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.0.4.6-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -258,7 +258,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.2.2.6.nar", + "archive": "connectors/pulsar-io-lakehouse-3.0.4.6.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -280,7 +280,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.2.2.6-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.0.4.6-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -350,7 +350,7 @@ This example describes how to use the Lakehouse sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.2.2.6.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.2.6.nar + cp pulsar-io-lakehouse-3.0.4.6.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.0.4.6.nar ``` 2. Start Pulsar in standalone mode. @@ -383,7 +383,7 @@ This example explains how to create a Lakehouse sink connector in an on-premises 1. Copy the NAR package of the Lakehouse sink connector to the Pulsar connectors directory. ```bash - cp pulsar-io-lakehouse-3.2.2.6.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.2.6.nar + cp pulsar-io-lakehouse-3.0.4.6.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.0.4.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/connectors/lakehouse-sink/v3.0.4.5/lakehouse-sink.md b/connectors/lakehouse-sink/v3.2.2.7/lakehouse-sink.md similarity index 96% rename from connectors/lakehouse-sink/v3.0.4.5/lakehouse-sink.md rename to connectors/lakehouse-sink/v3.2.2.7/lakehouse-sink.md index caca43c6..d8ad277b 100644 --- a/connectors/lakehouse-sink/v3.0.4.5/lakehouse-sink.md +++ b/connectors/lakehouse-sink/v3.2.2.7/lakehouse-sink.md @@ -11,7 +11,7 @@ tags: alias: Lakehouse Sink Connector features: ["pulsar lakehouse connector"] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-lakehouse support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "lakehouse-sink" The Lakehouse sink connector (including the [Hudi](https://hudi.apache.org), [Iceberg](https://iceberg.apache.org/), and [Delta Lake](https://delta.io/) sink connectors) fetches data from a Pulsar topic and saves data to the Lakehouse tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.0.4.5/docs/lakehouse-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.2.2.7/docs/lakehouse-sink.png) # How to get @@ -62,7 +62,7 @@ To build the Lakehouse sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-lakehouse-3.0.4.5.nar + pulsar-io-lakehouse-3.2.2.7.nar ``` # How to configure @@ -147,7 +147,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-3.0.4.5.nar", + "archive": "connectors/pulsar-io-hudi-3.2.2.7.nar", "processingGuarantees": "EFFECTIVELY_ONCE", "parallelism": 1, "configs": { @@ -171,7 +171,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-3.0.4.5-cloud.nar", + "archive": "connectors/pulsar-io-hudi-3.2.2.7-cloud.nar", "parallelism": 1, "processingGuarantees": "EFFECTIVELY_ONCE", "configs": { @@ -199,7 +199,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.0.4.5.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.2.7.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -227,7 +227,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.0.4.5-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.2.7-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -258,7 +258,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.0.4.5.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.2.7.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -280,7 +280,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.0.4.5-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.2.7-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -350,7 +350,7 @@ This example describes how to use the Lakehouse sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.0.4.5.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.0.4.5.nar + cp pulsar-io-lakehouse-3.2.2.7.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.2.7.nar ``` 2. Start Pulsar in standalone mode. @@ -383,7 +383,7 @@ This example explains how to create a Lakehouse sink connector in an on-premises 1. Copy the NAR package of the Lakehouse sink connector to the Pulsar connectors directory. ```bash - cp pulsar-io-lakehouse-3.0.4.5.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.0.4.5.nar + cp pulsar-io-lakehouse-3.2.2.7.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.2.7.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/connectors/lakehouse-source/v3.2.2.6/lakehouse-source.md b/connectors/lakehouse-source/v3.0.4.6/lakehouse-source.md similarity index 95% rename from connectors/lakehouse-source/v3.2.2.6/lakehouse-source.md rename to connectors/lakehouse-source/v3.0.4.6/lakehouse-source.md index 783d9fd7..3528eb5e 100644 --- a/connectors/lakehouse-source/v3.2.2.6/lakehouse-source.md +++ b/connectors/lakehouse-source/v3.0.4.6/lakehouse-source.md @@ -11,7 +11,7 @@ tags: alias: Lakehouse Source Connector features: ["pulsar lakehouse connector"] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.2.2.6 +download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.0.4.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-lakehouse support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "lakehouse-source" The Lakehouse source connector (currently only including the [Delta Lake](https://delta.io/) source connector) fetches the Lakehouse table's changelog and saves changelogs into a Pulsar topic. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.2.2.6/docs/lakehouse-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.0.4.6/docs/lakehouse-source.png) # How to get @@ -62,7 +62,7 @@ To build the Lakehouse source connector from the source code, follow these steps ```bash ls target - pulsar-io-lakehouse-3.2.2.6.nar + pulsar-io-lakehouse-3.0.4.6.nar ``` # How to configure @@ -110,7 +110,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-3.2.2.6.nar", + "archive": "connectors/pulsar-io-lakehouse-3.0.4.6.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -135,7 +135,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-3.2.2.6-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.0.4.6-cloud.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -195,7 +195,7 @@ This example describes how to use the Lakehouse source connector to fetch data f 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.2.2.6.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.2.6.nar + cp pulsar-io-lakehouse-3.0.4.6.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.0.4.6.nar ``` 2. Start Pulsar in standalone mode. @@ -226,7 +226,7 @@ This example explains how to create a Lakehouse source connector in an on-premis 1. Copy the NAR package of the Lakehouse source connector to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.2.2.6.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.2.6.nar + cp pulsar-io-lakehouse-3.0.4.6.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.0.4.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/connectors/lakehouse-source/v3.0.4.5/lakehouse-source.md b/connectors/lakehouse-source/v3.2.2.7/lakehouse-source.md similarity index 95% rename from connectors/lakehouse-source/v3.0.4.5/lakehouse-source.md rename to connectors/lakehouse-source/v3.2.2.7/lakehouse-source.md index 323bcc73..c84efe0f 100644 --- a/connectors/lakehouse-source/v3.0.4.5/lakehouse-source.md +++ b/connectors/lakehouse-source/v3.2.2.7/lakehouse-source.md @@ -11,7 +11,7 @@ tags: alias: Lakehouse Source Connector features: ["pulsar lakehouse connector"] icon: "/images/pulsar-hub.svg" -download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.0.4.5 +download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.2.2.7 support: streamnative support_link: https://github.com/streamnative/pulsar-io-lakehouse support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -25,7 +25,7 @@ id: "lakehouse-source" The Lakehouse source connector (currently only including the [Delta Lake](https://delta.io/) source connector) fetches the Lakehouse table's changelog and saves changelogs into a Pulsar topic. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.0.4.5/docs/lakehouse-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.2.2.7/docs/lakehouse-source.png) # How to get @@ -62,7 +62,7 @@ To build the Lakehouse source connector from the source code, follow these steps ```bash ls target - pulsar-io-lakehouse-3.0.4.5.nar + pulsar-io-lakehouse-3.2.2.7.nar ``` # How to configure @@ -110,7 +110,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-3.0.4.5.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.2.7.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -135,7 +135,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-3.0.4.5-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.2.2.7-cloud.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -195,7 +195,7 @@ This example describes how to use the Lakehouse source connector to fetch data f 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.0.4.5.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.0.4.5.nar + cp pulsar-io-lakehouse-3.2.2.7.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.2.7.nar ``` 2. Start Pulsar in standalone mode. @@ -226,7 +226,7 @@ This example explains how to create a Lakehouse source connector in an on-premis 1. Copy the NAR package of the Lakehouse source connector to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.0.4.5.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.0.4.5.nar + cp pulsar-io-lakehouse-3.2.2.7.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.2.2.7.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/connectors/pinecone-sink/v3.2.2.6/pinecone-sink.md b/connectors/pinecone-sink/v3.0.4.6/pinecone-sink.md similarity index 99% rename from connectors/pinecone-sink/v3.2.2.6/pinecone-sink.md rename to connectors/pinecone-sink/v3.0.4.6/pinecone-sink.md index c8d6ca82..4bcc8e22 100644 --- a/connectors/pinecone-sink/v3.2.2.6/pinecone-sink.md +++ b/connectors/pinecone-sink/v3.0.4.6/pinecone-sink.md @@ -31,7 +31,7 @@ takes in messages and writes them if they are in a proper format to a Pinecone index. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-pinecone/v3.2.2.6/docs/pinecone.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-pinecone/v3.0.4.6/docs/pinecone.png) ## Quick start diff --git a/connectors/pinecone-sink/v3.2.2.7/pinecone-sink.md b/connectors/pinecone-sink/v3.2.2.7/pinecone-sink.md new file mode 100644 index 00000000..620d3a1f --- /dev/null +++ b/connectors/pinecone-sink/v3.2.2.7/pinecone-sink.md @@ -0,0 +1,262 @@ +--- +description: A connector to pinecone.io +author: illegalnumbers,dependabot[bot],shibd,streamnativebot +contributors: illegalnumbers,dependabot[bot],shibd,streamnativebot +language: Java,Shell,Dockerfile +document: +source: Private source +license: StreamNative, Inc.. All Rights Reserved +license_link: +tags: +alias: Pinecone Connector +features: ["A connector to pinecone.io"] +icon: "/images/connectors/pinecone-logo.png" +download: +support: streamnative +support_link: https://streamnative.io +support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +owner_name: "streamnative" +owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +dockerfile: +sn_available: "true" +id: "pinecone-sink" +--- + + + +# Pinecone Sink Connector + +This connector allows access to pinecone.io with a pulsar topic. The sink connector +takes in messages and writes them if they are in a proper format to a Pinecone +index. + + +![](https://raw.githubusercontent.com/streamnative/pulsar-io-pinecone/v3.2.2.7/docs/pinecone.png) + +## Quick start + +1. Pay for a license. +2. Create an index on pinecone.io +3. Download the image (from streamnative.io/pulsar-io-pinecone). +4. Provide the configuration below and start the connector. + +### Prerequisites + +The prerequisites for connecting a Pinecone sink connector to external systems include: + +1. A pinecone.io api key +2. A index name +3. A environment name +4. A project name +5. A namespace name +6. A connection url + +See conf/pulsar-io-template.yaml for more information. + +### 1. Create a connector + +The following command shows how to use [pulsarctl](https://github.com/streamnative/pulsarctl) to create a `builtin` connector. If you want to create a `non-builtin` connector, +you need to replace `--sink-type pinecone` with `--archive /path/to/pulsar-io-pinecone.nar`. You can find the button to download the `nar` package at the beginning of the document. + +{% callout title="For StreamNative Cloud User" type="note" %} +If you are a StreamNative Cloud user, you need [set up your environment](https://docs.streamnative.io/docs/connector-setup) first. +{% /callout %} + +```bash +pulsarctl sink create \ + --sink-type pinecone \ + --name pinecone \ + --tenant public \ + --namespace default \ + --inputs "Your topic name" \ + --parallelism 1 \ + --sink-config \ + '{ "apiKey": "abcd-123","indexName": "test","environment": "gcp-starter","project": "default", "namespace": "test", "connectionUrl": "test-project.svc.gcp-starter.pinecone.io", "dimensions": 1 }' +``` + +The `--sink-config` is the minimum necessary configuration for starting this connector, and it is a JSON string. You need to substitute the relevant parameters with your own. +If you want to configure more parameters, see [Configuration Properties](#configuration-properties) for reference. + +{% callout title="Note" type="note" %} +You can also choose to use a variety of other tools to create a connector: +- [pulsar-admin](https://pulsar.apache.org/docs/3.1.x/io-use/): The command arguments for `pulsar-admin` are similar to those of `pulsarctl`. You can find an example for [StreamNative Cloud Doc](https://docs.streamnative.io/docs/connector-create#create-a-built-in-connector ). +- [RestAPI](https://pulsar.apache.org/sink-rest-api/?version=3.1.1): You can find an example for [StreamNative Cloud Doc](https://docs.streamnative.io/docs/connector-create#create-a-built-in-connector). +- [Terraform](https://github.com/hashicorp/terraform): You can find an example for [StreamNative Cloud Doc](https://docs.streamnative.io/docs/connector-create#create-a-built-in-connector). +- [Function Mesh](https://functionmesh.io/docs/connectors/run-connector): The docker image can be found at the beginning of the document. +{% /callout %} + +### 2. Send messages to the topic + +{% callout title="Note" type="note" %} +If your connector is created on StreamNative Cloud, you need to authenticate your clients. See [Build applications using Pulsar clients](https://docs.streamnative.io/docs/qs-connect#jumpstart-for-beginners) for more information. +{% /callout %} + +``` java +@Data +@ToString +public class TestMessage { + public static void main(String[] args) { + PulsarClient client = PulsarClient.builder() + .serviceUrl("{{Your Pulsar URL}}") + .build(); + + Producer producer = client.newProducer(Schema.STRING) + .topic("my-topic") + .create(); + + String testMessage = '{ "id": "v1", "values": [1.0]}'; + + MessageId msgID = producer.send(testMessage); + System.out.println("Publish " + testMessage + " and message ID " + msgID); + + producer.flush(); + producer.close(); + client.close(); + } +} + +``` + +### 3. Querying Data From Index + +You can look in the query UI from Pinecone or you can run a raw Pinecone +query yourself using a client. There are several on the Pinecone website +which are listed including Python, Node, and cURL. + +```python +# Taken from https://www.pinecone.io/ +# Mock vectorized search query (vectorize with LLM of choice) +query = [0.1] # len(query) = 1, same as the indexed vectors + +# Send query with (optional) filter to index and get back 1 result (top_k=1) +index.query( + vector=query, + top_k=1 +) +``` + +## Configuration Properties + +Before using the Pinecone sink connector, you need to configure it. This table outlines the properties and the descriptions. + +| Name | Type | Required | Sensitive | Default | Description | +|-------------------------------|---------|----------|-----------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| apiKey | string | True | True | None | The API key for the Pinecone service. Find this in the Pinecone dashboard. +| indexName | string | True | False | None | The name of the Pinecone index to which you want to write data. Find this in the Pinecone dashboard. +| environment | string | True | False | None | The name of the Pinecone environment to which you want to write data. Find this in the Pinecone dashboard. +| project | string | True | False | None | The name of the Pinecone project to which you want to write data. Find this in the Pinecone dashboard. +| namespace | string | True | False | None | The name of the Pinecone namespace to which you want to write data. Find this in the Pinecone dashboard. +| connectionUrl | string | True | False | None | The connection URL configured in the Pinecone dashboard. Find this in the Pinecone dashboard. +| dimensions | integer | False | False | None | The number of dimensions required by the index. If a request is made to upsert data into an index with a different number of dimensions, the request will fail. If not provided the connector will make it's best attempt to upsert the data and if the connection fails due to a mismatch the message will eventually be DLQ'd. +| queryMetadata | JSON | False | False | None | The metadata to be associated with the request to the index.This should be a JSON object in the form {"key": "value", "key2": "value2" }. + +## Advanced features + +### Monitoring + +Currently we provide several metrics for monitoring. + +- `pinecone-upsert-successful` +- `pinecone-upsert-failed` +- `pinecone-connector-active` +- `pinecone-upsert-failed-no-config` +- `pinecone-upsert-failed-no-client` +- `pinecone-upsert-failed-parsing-error` +- `pinecone-upsert-failed-dimension-error` + +These can all be used to manage the connectors status. + +### Troubleshooting + +If you get a failed upsert problem the most likely candidate is the formatting +of your messages. These are required to be in a format like the following. + +``` +{ "id": "string", "values": [float, float, ...]} +``` + +or the form +``` +{ "metadata": { "key": "value", "key2": "value2", ... }, id: "string", "values": [float, float, ...]} +``` + +Other likely candidates are problems with your connection to Pinecone. Check your +configuration values and any exceptions that are ocurring from the connector. + +Some example commands for debugging locally are as follows. + +Produce a sample message. + +``` +pulsar-client produce persistent://public/default/pinecone-source -m '{"id":"v1", "values": [3.0]}' -s '\n' +``` + +Clear a backlog of messages. +``` +pulsar-admin --admin-url http://localhost:8080 topics clear-backlog --subscription public/default/pinecone persistent://public/default/pinecone-source +``` + +Delete a topic subscription. +``` +pulsar-admin --admin-url http://localhost:8080 topics unsubscribe \ + --subscription public/default/pinecone \ + persistent://public/default/pinecone-source +``` + +Consume a group of messages. +``` +pulsar-client consume persistent://public/default/pinecone-source -s public/default/pinecone +``` + +If you need to add a maven shell using jenv you can do this with a +helpful script. +``` +mvn dependency:build-classpath -DincludeTypes=jar -Dmdep.outputFile=.cp.txt +jshell --class-path `cat .cp.txt`:target/classes +``` + +And remember if you have maven problems on install that you need to +use JDK 8 with this project. + +``` +mvn --version # should be java 8 +jenv exec mvn # if using jenv you can exec the local version using + # this +``` + +### Delivery guarantees + +The Pulsar IO connector framework provides three [delivery guarantees](https://pulsar.apache.org/docs/next/functions-concepts#processing-guarantees-and-subscription-types): `at-most-once`, `at-least-once`, and `effectively-once`. + +Currently, the Pinecone sink connector provides the at-least-once delivery guarantee. + +### Examples + +With the source connector you can connect to Pinecone with a valid configuration +and then write messages to it. An example using localrun is shown below. + +``` +pulsar-admin --admin-url http://localhost:8080/ sinks localrun --broker-service-url pulsar://localhost:6650/ --archive "file:///Users/your-user/src/pulsar-io-pinecone/target/pulsar-io-pinecone-0.0.1.jar" --classname "org.apache.pulsar.ecosystem.io.pinecone.PineconeConnectorSink" --name "pinecone" --sink-config '{ "apiKey": "abcd-123","indexName": "test","environment": "gcp-starter","project": "default", "namespace": "test", "connectionUrl": "test-project.svc.gcp-starter.pinecone.io", "dimensions": 1 }' --inputs persistent://public/default/pinecone-source +``` + +This can be used when building the JAR of the project from scratch using +`mvn clean install`. + +Similar configuration can be setup when using an image mounted with a config file +defining environment variables or when using in Kubernetes. + +This table lists the schema types that currently are supported to be converted. + + + +| Schema | Supported | +|-----------------|-----------| +| AVRO | No | +| PRIMITIVE | Yes | (only bytes and string) +| PROTOBUF_NATIVE | No | +| PROTOBUF | Yes | +| JSON | Yes | +| KEY_VALUE | No | + + + diff --git a/connectors/snowflake-sink/v3.0.4.5/snowflake-sink.md b/connectors/snowflake-sink/v3.0.4.6/snowflake-sink.md similarity index 85% rename from connectors/snowflake-sink/v3.0.4.5/snowflake-sink.md rename to connectors/snowflake-sink/v3.0.4.6/snowflake-sink.md index 33ee8229..4762341d 100644 --- a/connectors/snowflake-sink/v3.0.4.5/snowflake-sink.md +++ b/connectors/snowflake-sink/v3.0.4.6/snowflake-sink.md @@ -118,8 +118,7 @@ pulsarctl sinks create \ "schema": "demo", "database": "st_tuts", "privateKey": "...", - "warehouse": "compute_wh", - "topic2table": "input-snowflake:emp_basic" + "warehouse": "compute_wh" }' ``` @@ -200,7 +199,7 @@ This table outlines the properties and the descriptions. | `bufferFlushTimeInSeconds` | int | No | 60 | The number of seconds between buffer flushes, where the flush is from the Pulsar’s memory cache to the internal stage. By default, it is set to `60` seconds. | | `autoCreateTable` | boolean | No | false | Automatically create a table when the table does not exist. | | `processingGuarantees` | String | No | "ATLEAST_ONCE" | Specify the processing guarantee semantics. Currently, the Snowflake connector only supports `ATLEAST_ONCE` processing guarantee semantics. | -| `topic2table` | String | No | ""(empty string) | Specify the mapping relationship between topics and tables. Each topic and the mapped table name should be separated by a colon, such as `topic1:table1,topic2:table2`. | +| `topic2table` | String | No | false | ""(empty string) | Specify the mapping relationship between topics and tables. The topic name should be its complete name. Each topic and the mapped table name should be separated by a colon, such as `persistent://public/default/topic1:table1,persistent://public/default/topic2:table2`. | | `metadataField` | String | No | "__message_id__,__partition__,__topic__,__event_time__" | The metadata fields for each snowflake record. You can separate multiple fields with commas. The supported metadata fields are: __schema_version__ , __partition__ , __event_time__ , __publish_time__ , __message_id__ , __sequence_id__ , __producer_name__ , __topic__. Currently, the Snowflake sink connector currently does not support custom metadata. | ## Advanced features @@ -218,9 +217,10 @@ autoCreateTable=true ``` The Snowflake sink connector allows you to specify the mapping relationship between topics and tables. Each topic and its mapped table name should be separated by a colon. +And please note that the topic name should be its complete name. The parameter `topic2table` see example below: ``` -topic2table=topic1:table1,topic2:table2 +topic2table=persistent://public/default/topic1:table1,persistent://public/default/topic2:table2 ``` ## Metadata Fields @@ -246,6 +246,23 @@ The snowflake connector supports converting some Pulsar schemas, as listed in th | PROTOBUF | No | | PROTOBUF_NATIVE | No | +All data will be converted and written in JSON format under the "content" column. Below is a table showing the +conversion for each Schema Type: + +| Schema Type | Converted Content | Example | +|------------------------------------------|------------------------------------------------------------|--------------------------------------| +| BYTES | Base64-encoded String | "SGVsbG8=" (Hello in base64) | +| Boolean | Boolean | true | +| INT8, INT16, INT32, INT64, FLOAT, DOUBLE | Number | 1234 | +| STRING | String | "Hello" | +| JSON | JSON Object | {"name": "John", "age": 30} | +| AVRO | JSON Object | {"name": "John", "age": 30} | +| DATE, TIME, TIMESTAMP | Number (milliseconds since Jan 1, 1970, GMT) | 1654849667447 | +| INSTANT | Number (seconds since 1970-01-01T00:00:00Z) | 1654826254.091 | +| LOCAL_DATE | Array [Year, Month, Day] | [2022, 12, 1] | +| LOCAL_TIME | Array [Hour, Minute, Second, Nanosecond] | [16, 30, 28, 150000000] | +| LOCAL_DATE_TIME | Array [Year, Month, Day, Hour, Minute, Second, Nanosecond] | [2022, 12, 1, 16, 30, 28, 150000000] | + ## Batch progress To increase write throughput, you can configure the buffer size and latency for the Snowflake sink connector. diff --git a/connectors/snowflake-sink/v3.2.2.6/snowflake-sink.md b/connectors/snowflake-sink/v3.2.2.7/snowflake-sink.md similarity index 86% rename from connectors/snowflake-sink/v3.2.2.6/snowflake-sink.md rename to connectors/snowflake-sink/v3.2.2.7/snowflake-sink.md index 3fbe3cb3..9af75b4d 100644 --- a/connectors/snowflake-sink/v3.2.2.6/snowflake-sink.md +++ b/connectors/snowflake-sink/v3.2.2.7/snowflake-sink.md @@ -118,8 +118,7 @@ pulsarctl sinks create \ "schema": "demo", "database": "st_tuts", "privateKey": "...", - "warehouse": "compute_wh", - "topic2table": "input-snowflake:emp_basic" + "warehouse": "compute_wh" }' ``` @@ -200,7 +199,7 @@ This table outlines the properties and the descriptions. | `bufferFlushTimeInSeconds` | int | No | false | 60 | The number of seconds between buffer flushes, where the flush is from the Pulsar’s memory cache to the internal stage. By default, it is set to `60` seconds. | | `autoCreateTable` | boolean | No | false | false | Automatically create a table when the table does not exist. | | `processingGuarantees` | String | No | false | "ATLEAST_ONCE" | Specify the processing guarantee semantics. Currently, the Snowflake connector only supports `ATLEAST_ONCE` processing guarantee semantics. | -| `topic2table` | String | No | false | ""(empty string) | Specify the mapping relationship between topics and tables. Each topic and the mapped table name should be separated by a colon, such as `topic1:table1,topic2:table2`. | +| `topic2table` | String | No | false | ""(empty string) | Specify the mapping relationship between topics and tables. The topic name should be its complete name. Each topic and the mapped table name should be separated by a colon, such as `persistent://public/default/topic1:table1,persistent://public/default/topic2:table2`. | | `metadataField` | String | No | false | "__message_id__,__partition__,__topic__,__event_time__" | The metadata fields for each snowflake record. You can separate multiple fields with commas. The supported metadata fields are: __schema_version__ , __partition__ , __event_time__ , __publish_time__ , __message_id__ , __sequence_id__ , __producer_name__ , __topic__. Currently, the Snowflake sink connector currently does not support custom metadata. | ## Advanced features @@ -218,9 +217,10 @@ autoCreateTable=true ``` The Snowflake sink connector allows you to specify the mapping relationship between topics and tables. Each topic and its mapped table name should be separated by a colon. +And please note that the topic name should be its complete name. The parameter `topic2table` see example below: ``` -topic2table=topic1:table1,topic2:table2 +topic2table=persistent://public/default/topic1:table1,persistent://public/default/topic2:table2 ``` ## Metadata Fields @@ -246,6 +246,23 @@ The snowflake connector supports converting some Pulsar schemas, as listed in th | PROTOBUF | No | | PROTOBUF_NATIVE | No | +All data will be converted and written in JSON format under the "content" column. Below is a table showing the +conversion for each Schema Type: + +| Schema Type | Converted Content | Example | +|------------------------------------------|------------------------------------------------------------|--------------------------------------| +| BYTES | Base64-encoded String | "SGVsbG8=" (Hello in base64) | +| Boolean | Boolean | true | +| INT8, INT16, INT32, INT64, FLOAT, DOUBLE | Number | 1234 | +| STRING | String | "Hello" | +| JSON | JSON Object | {"name": "John", "age": 30} | +| AVRO | JSON Object | {"name": "John", "age": 30} | +| DATE, TIME, TIMESTAMP | Number (milliseconds since Jan 1, 1970, GMT) | 1654849667447 | +| INSTANT | Number (seconds since 1970-01-01T00:00:00Z) | 1654826254.091 | +| LOCAL_DATE | Array [Year, Month, Day] | [2022, 12, 1] | +| LOCAL_TIME | Array [Hour, Minute, Second, Nanosecond] | [16, 30, 28, 150000000] | +| LOCAL_DATE_TIME | Array [Year, Month, Day, Hour, Minute, Second, Nanosecond] | [2022, 12, 1, 16, 30, 28, 150000000] | + ## Batch progress To increase write throughput, you can configure the buffer size and latency for the Snowflake sink connector. diff --git a/connectors/sqs-sink/v3.0.4.5/sqs-sink.md b/connectors/sqs-sink/v3.0.4.6/sqs-sink.md similarity index 99% rename from connectors/sqs-sink/v3.0.4.5/sqs-sink.md rename to connectors/sqs-sink/v3.0.4.6/sqs-sink.md index 8cb585ae..77fb0bef 100644 --- a/connectors/sqs-sink/v3.0.4.5/sqs-sink.md +++ b/connectors/sqs-sink/v3.0.4.6/sqs-sink.md @@ -25,7 +25,7 @@ id: "sqs-sink" The [AWS Simple Queue Service (SQS)](https://aws.amazon.com/sqs/?nc1=h_ls) sink connector pulls data from Pulsar topics and persists data to AWS SQS. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.0.4.5/docs/sqs-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.0.4.6/docs/sqs-sink.png) ## Quick start diff --git a/connectors/sqs-sink/v3.2.2.6/sqs-sink.md b/connectors/sqs-sink/v3.2.2.7/sqs-sink.md similarity index 99% rename from connectors/sqs-sink/v3.2.2.6/sqs-sink.md rename to connectors/sqs-sink/v3.2.2.7/sqs-sink.md index 5d7ef2ef..bc9ba426 100644 --- a/connectors/sqs-sink/v3.2.2.6/sqs-sink.md +++ b/connectors/sqs-sink/v3.2.2.7/sqs-sink.md @@ -25,7 +25,7 @@ id: "sqs-sink" The [AWS Simple Queue Service (SQS)](https://aws.amazon.com/sqs/?nc1=h_ls) sink connector pulls data from Pulsar topics and persists data to AWS SQS. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.2.2.6/docs/sqs-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.2.2.7/docs/sqs-sink.png) ## Quick start diff --git a/connectors/sqs-source/v3.0.4.5/sqs-source.md b/connectors/sqs-source/v3.0.4.6/sqs-source.md similarity index 99% rename from connectors/sqs-source/v3.0.4.5/sqs-source.md rename to connectors/sqs-source/v3.0.4.6/sqs-source.md index 481dfa9e..4355e2f6 100644 --- a/connectors/sqs-source/v3.0.4.5/sqs-source.md +++ b/connectors/sqs-source/v3.0.4.6/sqs-source.md @@ -25,7 +25,7 @@ id: "sqs-source" The [AWS Simple Queue Service (SQS)](https://aws.amazon.com/sqs/?nc1=h_ls) source connector feeds data from Amazon AWS SQS and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.0.4.5/docs/sqs-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.0.4.6/docs/sqs-source.png) ## Quick start diff --git a/connectors/sqs-source/v3.2.2.6/sqs-source.md b/connectors/sqs-source/v3.2.2.7/sqs-source.md similarity index 99% rename from connectors/sqs-source/v3.2.2.6/sqs-source.md rename to connectors/sqs-source/v3.2.2.7/sqs-source.md index d2895643..0a7d70df 100644 --- a/connectors/sqs-source/v3.2.2.6/sqs-source.md +++ b/connectors/sqs-source/v3.2.2.7/sqs-source.md @@ -25,7 +25,7 @@ id: "sqs-source" The [AWS Simple Queue Service (SQS)](https://aws.amazon.com/sqs/?nc1=h_ls) source connector feeds data from Amazon AWS SQS and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.2.2.6/docs/sqs-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.2.2.7/docs/sqs-source.png) ## Quick start