Optional configs to pass additional_headers/additional_metadata to indexes (#297)

## Problem

There are some debugging scenarios where, in order to debug or track a
specific request, we would like the ability to pass additional headers
with each request.

## Solution

- Add `additional_headers` kwarg to `Index()` for REST calls
- Add `additional_metadata` field within existing `grpc_config` object
for configuring equivalent for GRPC
- Add unit tests for both ways of doing it. Metadata is a [similar
concept](https://grpc.io/docs/guides/metadata/) to http request headers
in GRPC, not to be confused with vector metadata.

## Type of Change

- [x] None of the above: New feature, but really for Pinecone
developer/support use only.


### Usage (REST)

```python
from pinecone import Pinecone

pc = Pinecone(api_key='xxx')
index = pc.Index(
    host='hosturl', 
    additional_headers={ 'header-1': 'header-1-value' }
)

# Now do things
index.upsert(...)
```

### Usage (GRPC)

```python
from pinecone.grpc import PineconeGRPC, GRPCClientConfig

pc = PineconeGRPC(api_key='YOUR_API_KEY')

grpc_config = GRPCClientConfig(additional_metadata={'extra-header': 'value123'})
index = pc.Index(
    name='my-index', 
    host='host', 
    grpc_config=grpc_config
)

# do stuff
index.upsert(...)
```

## Test Plan

Besides unit tests, I will try running some test commands with
`PINECONE_DEBUG_CURL='true'` enabled to see what request is being sent.

Author: jhamon

pinecone/control/pinecone.py

@@ -446,6 +446,7 @@ def _get_status(self, name: str):
         response = api_instance.describe_index(name)
         return response["status"]
 
+
     def Index(self, name: str = '', host: str = '', **kwargs):
         """
         Target an index for data operations.

pinecone/data/index.py

@@ -69,15 +69,25 @@ class Index():
     For improved performance, use the Pinecone GRPC index client.
     """
 
-    def __init__(self, api_key: str, host: str, pool_threads=1, **kwargs):
-        api_key = api_key or kwargs.get("api_key", None)
-        host = host or kwargs.get('host', None)
-        pool_threads = pool_threads or kwargs.get("pool_threads")
-
+    def __init__(
+            self,
+            api_key: str,
+            host: str,
+            pool_threads: Optional[int] = 1,
+            additional_headers: Optional[Dict[str, str]] = {},
+            **kwargs
+        ):
         self._config = ConfigBuilder.build(api_key=api_key, host=host, **kwargs)
 
-        api_client = ApiClient(configuration=self._config.openapi_config, pool_threads=pool_threads)
+        api_client = ApiClient(configuration=self._config.openapi_config, 
+                               pool_threads=pool_threads)
+
+        # Configure request headers
         api_client.user_agent = get_user_agent()
+        extra_headers = additional_headers or {}
+        for key, value in extra_headers.items():
+            api_client.set_default_header(key, value)
+
         self._api_client = api_client
         self._vector_api = VectorOperationsApi(api_client=api_client)
 

pinecone/grpc/base.py

@@ -39,7 +39,15 @@ def __init__(
         self.config = config
         self.grpc_client_config = grpc_config or GRPCClientConfig()
         self.retry_config = self.grpc_client_config.retry_config or RetryConfig()
-        self.fixed_metadata = {"api-key": config.api_key, "service-name": index_name, "client-version": CLIENT_VERSION}
+
+        self.fixed_metadata = {
+            "api-key": config.api_key, 
+            "service-name": index_name, 
+            "client-version": CLIENT_VERSION
+        }
+        if self.grpc_client_config.additional_metadata:
+            self.fixed_metadata.update(self.grpc_client_config.additional_metadata)
+
         self._endpoint_override = _endpoint_override
 
         self.method_config = json.dumps(

pinecone/grpc/config.py

@@ -18,6 +18,10 @@ class GRPCClientConfig(NamedTuple):
     :type retry_config: RetryConfig, optional
     :param grpc_channel_options: A dict of gRPC channel arguments
     :type grpc_channel_options: Dict[str, str]
+    :param additional_metadata: Additional metadata to be sent to the server with each request. Note that this
+        metadata refers to [gRPC metadata](https://grpc.io/docs/guides/metadata/) which is a concept similar
+        to HTTP headers. This is unrelated to the metadata can be stored with a vector in the index.
+    :type additional_metadata: Dict[str, str]
     """
 
     secure: bool = True
@@ -26,6 +30,7 @@ class GRPCClientConfig(NamedTuple):
     reuse_channel: bool = True
     retry_config: Optional[RetryConfig] = None
     grpc_channel_options: Optional[Dict[str, str]] = None
+    additional_metadata: Optional[Dict[str, str]] = None
 
     @classmethod
     def _from_dict(cls, kwargs: dict):

tests/unit/test_control.py

@@ -28,6 +28,16 @@ def test_passing_additional_headers(self):
 
         for key, value in extras.items():
             assert p.index_api.api_client.default_headers[key] == value
+        assert 'User-Agent' in p.index_api.api_client.default_headers
+        assert len(p.index_api.api_client.default_headers) == 3
+
+    def test_overwrite_useragent(self):
+        # This doesn't seem like a common use case, but we may want to allow this
+        # when embedding the client in other pinecone tools such as canopy.
+        extras = {"User-Agent": "test-user-agent"}
+        p = Pinecone(api_key="123-456-789", additional_headers=extras)
+        assert p.index_api.api_client.default_headers['User-Agent'] == 'test-user-agent'
+        assert len(p.index_api.api_client.default_headers) == 1
 
     @pytest.mark.parametrize("timeout_value, describe_index_responses, expected_describe_index_calls, expected_sleep_calls", [
         # When timeout=None, describe_index is called until ready

tests/unit/test_index_initialization.py

@@ -0,0 +1,53 @@
+import pytest
+from pinecone import Pinecone
+
+class TestIndexClientInitialization():
+    @pytest.mark.parametrize(
+        'additional_headers', 
+        [
+            None, 
+            {}
+        ]
+    )
+    def test_no_additional_headers_leaves_useragent_only(self, additional_headers):
+        pc = Pinecone(api_key='YOUR_API_KEY')
+        index = pc.Index(host='myhost', additional_headers=additional_headers)
+        assert len(index._api_client.default_headers) == 1
+        assert 'User-Agent' in index._api_client.default_headers
+        assert 'python-client-' in index._api_client.default_headers['User-Agent']
+
+    def test_additional_headers_one_additional(self):
+        pc = Pinecone(api_key='YOUR_API_KEY')
+        index = pc.Index(
+            host='myhost', 
+            additional_headers={'test-header': 'test-header-value'}
+        )
+        assert 'test-header' in index._api_client.default_headers
+        assert len(index._api_client.default_headers) == 2
+
+    def test_multiple_additional_headers(self):
+        pc = Pinecone(api_key='YOUR_API_KEY')
+        index = pc.Index(
+            host='myhost', 
+            additional_headers={
+                'test-header': 'test-header-value', 
+                'test-header2': 'test-header-value2'
+            }
+        )
+        assert 'test-header' in index._api_client.default_headers
+        assert 'test-header2' in index._api_client.default_headers
+        assert len(index._api_client.default_headers) == 3
+
+    def test_overwrite_useragent(self):
+        # This doesn't seem like a common use case, but we may want to allow this
+        # when embedding the client in other pinecone tools such as canopy.
+        pc = Pinecone(api_key='YOUR_API_KEY')
+        index = pc.Index(
+            host='myhost', 
+            additional_headers={
+                'User-Agent': 'test-user-agent'
+            }
+        )
+        assert len(index._api_client.default_headers) == 1
+        assert 'User-Agent' in index._api_client.default_headers
+        assert index._api_client.default_headers['User-Agent'] == 'test-user-agent'

tests/unit_grpc/test_grpc_index_initialization.py

@@ -11,6 +11,27 @@ def test_init_with_default_config(self):
         assert index.grpc_client_config.reuse_channel == True
         assert index.grpc_client_config.retry_config == None
         assert index.grpc_client_config.grpc_channel_options == None
+        assert index.grpc_client_config.additional_metadata == None
+
+        # Default metadata, grpc equivalent to http request headers
+        assert len(index.fixed_metadata) == 3
+        assert index.fixed_metadata['api-key'] == 'YOUR_API_KEY'
+        assert index.fixed_metadata['service-name'] == 'my-index'
+        assert index.fixed_metadata['client-version'] != None
+
+    def test_init_with_additional_metadata(self):
+        pc = PineconeGRPC(api_key='YOUR_API_KEY')
+        config = GRPCClientConfig(additional_metadata={
+            'debug-header': 'value123',
+            'debug-header2': 'value456'
+        })
+        index = pc.Index(name='my-index', host='host', grpc_config=config)
+        assert len(index.fixed_metadata) == 5
+        assert index.fixed_metadata['api-key'] == 'YOUR_API_KEY'
+        assert index.fixed_metadata['service-name'] == 'my-index'
+        assert index.fixed_metadata['client-version'] != None
+        assert index.fixed_metadata['debug-header'] == 'value123'
+        assert index.fixed_metadata['debug-header2'] == 'value456'
 
     def test_init_with_grpc_config_from_dict(self):
         pc = PineconeGRPC(api_key='YOUR_API_KEY')