Skip to content

Commit b06c109

Browse files
yongtangyongtang
authored
Revert "Remove IgniteDataset (#1589)" (#1594)
This reverts commit 3663c78. Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
1 parent e7a8ab5 commit b06c109

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+6371
-0
lines changed

‎tensorflow_io/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ cc_binary(
3535
"//tensorflow_io/core:serialization_ops",
3636
"//tensorflow_io/core:sql_ops",
3737
"//tensorflow_io/core:text_ops",
38+
"//tensorflow_io/core:ignite_ops",
3839
"//tensorflow_io/core:mongodb_ops",
3940
"@local_config_tf//:libtensorflow_framework",
4041
"@local_config_tf//:tf_header_lib",

‎tensorflow_io/core/BUILD

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -713,6 +713,63 @@ cc_library(
713713
alwayslink = 1,
714714
)
715715

716+
cc_library(
717+
name = "ignite_ops",
718+
srcs = [
719+
"kernels/ignite/client/ignite_byte_swapper.h",
720+
"kernels/ignite/client/ignite_client.h",
721+
"kernels/ignite/client/ignite_plain_client.h",
722+
"kernels/ignite/client/ignite_ssl_wrapper.cc",
723+
"kernels/ignite/client/ignite_ssl_wrapper.h",
724+
"kernels/ignite/dataset/ignite_binary_object_parser.cc",
725+
"kernels/ignite/dataset/ignite_binary_object_parser.h",
726+
"kernels/ignite/dataset/ignite_dataset.cc",
727+
"kernels/ignite/dataset/ignite_dataset.h",
728+
"kernels/ignite/dataset/ignite_dataset_iterator.cc",
729+
"kernels/ignite/dataset/ignite_dataset_iterator.h",
730+
"kernels/ignite/dataset/ignite_dataset_ops.cc",
731+
"kernels/ignite/ggfs/ggfs.cc",
732+
"kernels/ignite/ggfs/ggfs.h",
733+
"kernels/ignite/ggfs/ggfs_client.cc",
734+
"kernels/ignite/ggfs/ggfs_client.h",
735+
"kernels/ignite/ggfs/ggfs_random_access_file.cc",
736+
"kernels/ignite/ggfs/ggfs_random_access_file.h",
737+
"kernels/ignite/ggfs/ggfs_writable_file.cc",
738+
"kernels/ignite/ggfs/ggfs_writable_file.h",
739+
"kernels/ignite/igfs/igfs.cc",
740+
"kernels/ignite/igfs/igfs.h",
741+
"kernels/ignite/igfs/igfs_client.cc",
742+
"kernels/ignite/igfs/igfs_client.h",
743+
"kernels/ignite/igfs/igfs_extended_tcp_client.cc",
744+
"kernels/ignite/igfs/igfs_extended_tcp_client.h",
745+
"kernels/ignite/igfs/igfs_messages.cc",
746+
"kernels/ignite/igfs/igfs_messages.h",
747+
"kernels/ignite/igfs/igfs_random_access_file.cc",
748+
"kernels/ignite/igfs/igfs_random_access_file.h",
749+
"kernels/ignite/igfs/igfs_writable_file.cc",
750+
"kernels/ignite/igfs/igfs_writable_file.h",
751+
"ops/ignite_ops.cc",
752+
] + select({
753+
"@bazel_tools//src/conditions:windows": [
754+
"kernels/ignite/client/ignite_plain_client_windows.cc",
755+
],
756+
"//conditions:default": [
757+
"kernels/ignite/client/ignite_plain_client_unix.cc",
758+
],
759+
}),
760+
copts = tf_io_copts(),
761+
defines = [
762+
"WIN32_LEAN_AND_MEAN",
763+
],
764+
linkstatic = True,
765+
deps = [
766+
"@boringssl//:ssl",
767+
"@local_config_tf//:libtensorflow_framework",
768+
"@local_config_tf//:tf_header_lib",
769+
],
770+
alwayslink = 1,
771+
)
772+
716773
cc_library(
717774
name = "pulsar_ops",
718775
srcs = [

‎tensorflow_io/core/kernels/ignite/client/ignite_byte_swapper.h

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_BYTE_SWAPPER_H_
17+
#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_BYTE_SWAPPER_H_
18+
19+
#include <stdint.h>
20+
21+
#include "tensorflow/core/platform/byte_order.h"
22+
23+
namespace tensorflow {
24+
25+
class ByteSwapper {
26+
public:
27+
ByteSwapper(bool big_endian) { swap_ = big_endian == port::kLittleEndian; }
28+
29+
void SwapIfRequiredInt16(int16_t *x) const {
30+
if (swap_) {
31+
Swap16(x);
32+
}
33+
}
34+
35+
void SwapIfRequiredUnsignedInt16(uint16_t *x) const {
36+
if (swap_) {
37+
Swap16(reinterpret_cast<int16_t *>(x));
38+
}
39+
}
40+
41+
void SwapIfRequiredInt32(int32_t *x) const {
42+
if (swap_) {
43+
Swap32(x);
44+
}
45+
}
46+
47+
void SwapIfRequiredFloat(float *x) const {
48+
if (swap_) {
49+
Swap32(reinterpret_cast<int32_t *>(x));
50+
}
51+
}
52+
53+
void SwapIfRequiredInt64(int64_t *x) const {
54+
if (swap_) {
55+
Swap64(x);
56+
}
57+
}
58+
59+
void SwapIfRequiredDouble(double *x) const {
60+
if (swap_) {
61+
Swap64(reinterpret_cast<int64_t *>(x));
62+
}
63+
}
64+
65+
void SwapIfRequiredInt16Arr(int16_t *x, int32_t length) const {
66+
if (swap_) {
67+
for (int32_t i = 0; i < length; i++) Swap16(&x[i]);
68+
}
69+
}
70+
71+
void SwapIfRequiredUnsignedInt16Arr(uint16_t *x, int32_t length) const {
72+
if (swap_) {
73+
for (int32_t i = 0; i < length; i++)
74+
Swap16(reinterpret_cast<int16_t *>(&x[i]));
75+
}
76+
}
77+
78+
void SwapIfRequiredInt32Arr(int32_t *x, int32_t length) const {
79+
if (swap_) {
80+
for (int32_t i = 0; i < length; i++) Swap32(&x[i]);
81+
}
82+
}
83+
84+
void SwapIfRequiredFloatArr(float *x, int32_t length) const {
85+
if (swap_) {
86+
for (int32_t i = 0; i < length; i++)
87+
Swap32(reinterpret_cast<int32_t *>(&x[i]));
88+
}
89+
}
90+
91+
void SwapIfRequiredInt64Arr(int64_t *x, int32_t length) const {
92+
if (swap_) {
93+
for (int32_t i = 0; i < length; i++) Swap64(&x[i]);
94+
}
95+
}
96+
97+
void SwapIfRequiredDoubleArr(double *x, int32_t length) const {
98+
if (swap_) {
99+
for (int32_t i = 0; i < length; i++)
100+
Swap64(reinterpret_cast<int64_t *>(&x[i]));
101+
}
102+
}
103+
104+
private:
105+
void Swap16(int16_t *x) const {
106+
*x = ((*x & 0xFF) << 8) | ((*x >> 8) & 0xFF);
107+
}
108+
109+
void Swap32(int32_t *x) const {
110+
*x = ((*x & 0xFF) << 24) | (((*x >> 8) & 0xFF) << 16) |
111+
(((*x >> 16) & 0xFF) << 8) | ((*x >> 24) & 0xFF);
112+
}
113+
114+
void Swap64(int64_t *x) const {
115+
*x = ((*x & 0xFF) << 56) | (((*x >> 8) & 0xFF) << 48) |
116+
(((*x >> 16) & 0xFF) << 40) | (((*x >> 24) & 0xFF) << 32) |
117+
(((*x >> 32) & 0xFF) << 24) | (((*x >> 40) & 0xFF) << 16) |
118+
(((*x >> 48) & 0xFF) << 8) | ((*x >> 56) & 0xFF);
119+
}
120+
121+
bool swap_;
122+
};
123+
124+
} // namespace tensorflow
125+
126+
#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_BYTE_SWAPPER_H_

‎tensorflow_io/core/kernels/ignite/client/ignite_client.h

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_CLIENT_H_
17+
#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_CLIENT_H_
18+
19+
#include "tensorflow/core/lib/core/errors.h"
20+
#include "tensorflow/core/lib/core/status.h"
21+
#include "tensorflow_io/core/kernels/ignite/client/ignite_byte_swapper.h"
22+
23+
namespace tensorflow {
24+
25+
class Client {
26+
public:
27+
Client(bool big_endian) : byte_swapper_(ByteSwapper(big_endian)) {}
28+
virtual Status Connect() = 0;
29+
virtual Status Disconnect() = 0;
30+
virtual bool IsConnected() = 0;
31+
virtual int GetSocketDescriptor() = 0;
32+
virtual Status ReadData(uint8_t *buf, const int32_t length) = 0;
33+
virtual Status WriteData(const uint8_t *buf, const int32_t length) = 0;
34+
35+
Status ReadByte(uint8_t *data) { return ReadData(data, 1); }
36+
37+
Status ReadShort(int16_t *data) {
38+
TF_RETURN_IF_ERROR(ReadData((uint8_t *)data, 2));
39+
byte_swapper_.SwapIfRequiredInt16(data);
40+
41+
return Status::OK();
42+
}
43+
44+
Status ReadInt(int32_t *data) {
45+
TF_RETURN_IF_ERROR(ReadData((uint8_t *)data, 4));
46+
byte_swapper_.SwapIfRequiredInt32(data);
47+
48+
return Status::OK();
49+
}
50+
51+
Status ReadLong(int64_t *data) {
52+
TF_RETURN_IF_ERROR(ReadData((uint8_t *)data, 8));
53+
byte_swapper_.SwapIfRequiredInt64(data);
54+
55+
return Status::OK();
56+
}
57+
58+
Status WriteByte(const uint8_t data) { return WriteData(&data, 1); }
59+
60+
Status WriteShort(const int16_t data) {
61+
int16_t tmp = data;
62+
byte_swapper_.SwapIfRequiredInt16(&tmp);
63+
return WriteData((uint8_t *)&tmp, 2);
64+
}
65+
66+
Status WriteInt(const int32_t data) {
67+
int32_t tmp = data;
68+
byte_swapper_.SwapIfRequiredInt32(&tmp);
69+
return WriteData((uint8_t *)&tmp, 4);
70+
}
71+
72+
Status WriteLong(const int64_t data) {
73+
int64_t tmp = data;
74+
byte_swapper_.SwapIfRequiredInt64(&tmp);
75+
return WriteData((uint8_t *)&tmp, 8);
76+
}
77+
78+
private:
79+
const ByteSwapper byte_swapper_;
80+
};
81+
82+
} // namespace tensorflow
83+
84+
#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_CLIENT_H_

‎tensorflow_io/core/kernels/ignite/client/ignite_plain_client.h

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
#ifndef TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_PLAIN_CLIENT_H_
17+
#define TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_PLAIN_CLIENT_H_
18+
19+
#include "tensorflow_io/core/kernels/ignite/client/ignite_client.h"
20+
21+
namespace tensorflow {
22+
23+
class PlainClient : public Client {
24+
public:
25+
PlainClient(string host, int port, bool big_endian);
26+
~PlainClient();
27+
28+
Status Connect() override;
29+
Status Disconnect() override;
30+
bool IsConnected() override;
31+
int GetSocketDescriptor() override;
32+
Status ReadData(uint8_t* buf, const int32_t length) override;
33+
Status WriteData(const uint8_t* buf, const int32_t length) override;
34+
35+
private:
36+
const string host_;
37+
const int port_;
38+
int sock_;
39+
};
40+
41+
} // namespace tensorflow
42+
43+
#endif // TENSORFLOW_CONTRIB_IGNITE_KERNELS_CLIENT_IGNITE_PLAIN_CLIENT_H_

0 commit comments

Comments
 (0)