pinecone.data.index

  1from tqdm.autonotebook import tqdm
  2
  3import logging
  4import json
  5from typing import Union, List, Optional, Dict, Any
  6
  7from pinecone.config import ConfigBuilder
  8
  9from pinecone.core.openapi.shared import API_VERSION
 10from pinecone.core.openapi.data import ApiClient
 11from pinecone.core.openapi.data.models import (
 12    FetchResponse,
 13    QueryRequest,
 14    QueryResponse,
 15    RpcStatus,
 16    ScoredVector,
 17    SingleQueryResults,
 18    DescribeIndexStatsResponse,
 19    UpsertRequest,
 20    UpsertResponse,
 21    Vector,
 22    DeleteRequest,
 23    UpdateRequest,
 24    DescribeIndexStatsRequest,
 25    ListResponse,
 26    SparseValues,
 27)
 28from pinecone.core.openapi.data.api.data_plane_api import DataPlaneApi
 29from ..utils import (
 30    setup_openapi_client,
 31    parse_non_empty_args,
 32    build_plugin_setup_client,
 33    validate_and_convert_errors,
 34)
 35from .features.bulk_import import ImportFeatureMixin
 36from .vector_factory import VectorFactory
 37from .query_results_aggregator import QueryResultsAggregator, QueryNamespacesResults
 38
 39from multiprocessing.pool import ApplyResult
 40from concurrent.futures import as_completed
 41
 42from pinecone_plugin_interface import load_and_install as install_plugins
 43
 44logger = logging.getLogger(__name__)
 45
 46__all__ = [
 47    "Index",
 48    "FetchResponse",
 49    "QueryRequest",
 50    "QueryResponse",
 51    "RpcStatus",
 52    "ScoredVector",
 53    "SingleQueryResults",
 54    "DescribeIndexStatsResponse",
 55    "UpsertRequest",
 56    "UpsertResponse",
 57    "UpdateRequest",
 58    "Vector",
 59    "DeleteRequest",
 60    "UpdateRequest",
 61    "DescribeIndexStatsRequest",
 62    "SparseValues",
 63]
 64
 65_OPENAPI_ENDPOINT_PARAMS = (
 66    "_return_http_data_only",
 67    "_preload_content",
 68    "_request_timeout",
 69    "_check_input_type",
 70    "_check_return_type",
 71    "_host_index",
 72    "async_req",
 73    "async_threadpool_executor",
 74)
 75
 76
 77def parse_query_response(response: QueryResponse):
 78    response._data_store.pop("results", None)
 79    return response
 80
 81
 82class Index(ImportFeatureMixin):
 83    """
 84    A client for interacting with a Pinecone index via REST API.
 85    For improved performance, use the Pinecone GRPC index client.
 86    """
 87
 88    def __init__(
 89        self,
 90        api_key: str,
 91        host: str,
 92        pool_threads: Optional[int] = 1,
 93        additional_headers: Optional[Dict[str, str]] = {},
 94        openapi_config=None,
 95        **kwargs,
 96    ):
 97        super().__init__(
 98            api_key=api_key,
 99            host=host,
100            pool_threads=pool_threads,
101            additional_headers=additional_headers,
102            openapi_config=openapi_config,
103            **kwargs,
104        )
105
106        self.config = ConfigBuilder.build(
107            api_key=api_key, host=host, additional_headers=additional_headers, **kwargs
108        )
109        self._openapi_config = ConfigBuilder.build_openapi_config(self.config, openapi_config)
110        self._pool_threads = pool_threads
111
112        if kwargs.get("connection_pool_maxsize", None):
113            self._openapi_config.connection_pool_maxsize = kwargs.get("connection_pool_maxsize")
114
115        self._vector_api = setup_openapi_client(
116            api_client_klass=ApiClient,
117            api_klass=DataPlaneApi,
118            config=self.config,
119            openapi_config=self._openapi_config,
120            pool_threads=self._pool_threads,
121            api_version=API_VERSION,
122        )
123
124        self._load_plugins()
125
126    def _load_plugins(self):
127        """@private"""
128        try:
129            # I don't expect this to ever throw, but wrapping this in a
130            # try block just in case to make sure a bad plugin doesn't
131            # halt client initialization.
132            openapi_client_builder = build_plugin_setup_client(
133                config=self.config,
134                openapi_config=self._openapi_config,
135                pool_threads=self._pool_threads,
136            )
137            install_plugins(self, openapi_client_builder)
138        except Exception as e:
139            logger.error(f"Error loading plugins in Index: {e}")
140
141    def __enter__(self):
142        return self
143
144    def __exit__(self, exc_type, exc_value, traceback):
145        self._vector_api.api_client.close()
146
147    @validate_and_convert_errors
148    def upsert(
149        self,
150        vectors: Union[List[Vector], List[tuple], List[dict]],
151        namespace: Optional[str] = None,
152        batch_size: Optional[int] = None,
153        show_progress: bool = True,
154        **kwargs,
155    ) -> UpsertResponse:
156        """
157        The upsert operation writes vectors into a namespace.
158        If a new value is upserted for an existing vector id, it will overwrite the previous value.
159
160        To upsert in parallel follow: https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel
161
162        A vector can be represented by a 1) Vector object, a 2) tuple or 3) a dictionary
163
164        If a tuple is used, it must be of the form `(id, values, metadata)` or `(id, values)`.
165        where id is a string, vector is a list of floats, metadata is a dict,
166        and sparse_values is a dict of the form `{'indices': List[int], 'values': List[float]}`.
167
168        Examples:
169            >>> ('id1', [1.0, 2.0, 3.0], {'key': 'value'}, {'indices': [1, 2], 'values': [0.2, 0.4]})
170            >>> ('id1', [1.0, 2.0, 3.0], None, {'indices': [1, 2], 'values': [0.2, 0.4]})
171            >>> ('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])
172
173        If a Vector object is used, a Vector object must be of the form
174        `Vector(id, values, metadata, sparse_values)`, where metadata and sparse_values are optional
175        arguments.
176
177        Examples:
178            >>> Vector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'})
179            >>> Vector(id='id2', values=[1.0, 2.0, 3.0])
180            >>> Vector(id='id3', values=[1.0, 2.0, 3.0], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]))
181
182        **Note:** the dimension of each vector must match the dimension of the index.
183
184        If a dictionary is used, it must be in the form `{'id': str, 'values': List[float], 'sparse_values': {'indices': List[int], 'values': List[float]}, 'metadata': dict}`
185
186        Examples:
187            >>> index.upsert([('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])])
188            >>>
189            >>> index.upsert([{'id': 'id1', 'values': [1.0, 2.0, 3.0], 'metadata': {'key': 'value'}},
190            >>>               {'id': 'id2', 'values': [1.0, 2.0, 3.0], 'sparse_values': {'indices': [1, 8], 'values': [0.2, 0.4]}])
191            >>> index.upsert([Vector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'}),
192            >>>               Vector(id='id2', values=[1.0, 2.0, 3.0], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]))])
193
194        API reference: https://docs.pinecone.io/reference/upsert
195
196        Args:
197            vectors (Union[List[Vector], List[Tuple]]): A list of vectors to upsert.
198            namespace (str): The namespace to write to. If not specified, the default namespace is used. [optional]
199            batch_size (int): The number of vectors to upsert in each batch.
200                               If not specified, all vectors will be upserted in a single batch. [optional]
201            show_progress (bool): Whether to show a progress bar using tqdm.
202                                  Applied only if batch_size is provided. Default is True.
203        Keyword Args:
204            Supports OpenAPI client keyword arguments. See pinecone.core.client.models.UpsertRequest for more details.
205
206        Returns: UpsertResponse, includes the number of vectors upserted.
207        """
208        _check_type = kwargs.pop("_check_type", True)
209
210        if kwargs.get("async_req", False) and batch_size is not None:
211            raise ValueError(
212                "async_req is not supported when batch_size is provided."
213                "To upsert in parallel, please follow: "
214                "https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel"
215            )
216
217        if batch_size is None:
218            return self._upsert_batch(vectors, namespace, _check_type, **kwargs)
219
220        if not isinstance(batch_size, int) or batch_size <= 0:
221            raise ValueError("batch_size must be a positive integer")
222
223        pbar = tqdm(total=len(vectors), disable=not show_progress, desc="Upserted vectors")
224        total_upserted = 0
225        for i in range(0, len(vectors), batch_size):
226            batch_result = self._upsert_batch(
227                vectors[i : i + batch_size], namespace, _check_type, **kwargs
228            )
229            pbar.update(batch_result.upserted_count)
230            # we can't use here pbar.n for the case show_progress=False
231            total_upserted += batch_result.upserted_count
232
233        return UpsertResponse(upserted_count=total_upserted)
234
235    def _upsert_batch(
236        self,
237        vectors: Union[List[Vector], List[tuple], List[dict]],
238        namespace: Optional[str],
239        _check_type: bool,
240        **kwargs,
241    ) -> UpsertResponse:
242        args_dict = parse_non_empty_args([("namespace", namespace)])
243
244        def vec_builder(v):
245            return VectorFactory.build(v, check_type=_check_type)
246
247        return self._vector_api.upsert(
248            UpsertRequest(
249                vectors=list(map(vec_builder, vectors)),
250                **args_dict,
251                _check_type=_check_type,
252                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS},
253            ),
254            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
255        )
256
257    @staticmethod
258    def _iter_dataframe(df, batch_size):
259        for i in range(0, len(df), batch_size):
260            batch = df.iloc[i : i + batch_size].to_dict(orient="records")
261            yield batch
262
263    def upsert_from_dataframe(
264        self, df, namespace: Optional[str] = None, batch_size: int = 500, show_progress: bool = True
265    ) -> UpsertResponse:
266        """Upserts a dataframe into the index.
267
268        Args:
269            df: A pandas dataframe with the following columns: id, values, sparse_values, and metadata.
270            namespace: The namespace to upsert into.
271            batch_size: The number of rows to upsert in a single batch.
272            show_progress: Whether to show a progress bar.
273        """
274        try:
275            import pandas as pd
276        except ImportError:
277            raise RuntimeError(
278                "The `pandas` package is not installed. Please install pandas to use `upsert_from_dataframe()`"
279            )
280
281        if not isinstance(df, pd.DataFrame):
282            raise ValueError(f"Only pandas dataframes are supported. Found: {type(df)}")
283
284        pbar = tqdm(total=len(df), disable=not show_progress, desc="sending upsert requests")
285        results = []
286        for chunk in self._iter_dataframe(df, batch_size=batch_size):
287            res = self.upsert(vectors=chunk, namespace=namespace)
288            pbar.update(len(chunk))
289            results.append(res)
290
291        upserted_count = 0
292        for res in results:
293            upserted_count += res.upserted_count
294
295        return UpsertResponse(upserted_count=upserted_count)
296
297    @validate_and_convert_errors
298    def delete(
299        self,
300        ids: Optional[List[str]] = None,
301        delete_all: Optional[bool] = None,
302        namespace: Optional[str] = None,
303        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
304        **kwargs,
305    ) -> Dict[str, Any]:
306        """
307        The Delete operation deletes vectors from the index, from a single namespace.
308        No error raised if the vector id does not exist.
309        Note: for any delete call, if namespace is not specified, the default namespace is used.
310
311        Delete can occur in the following mutual exclusive ways:
312        1. Delete by ids from a single namespace
313        2. Delete all vectors from a single namespace by setting delete_all to True
314        3. Delete all vectors from a single namespace by specifying a metadata filter
315            (note that for this option delete all must be set to False)
316
317        API reference: https://docs.pinecone.io/reference/delete_post
318
319        Examples:
320            >>> index.delete(ids=['id1', 'id2'], namespace='my_namespace')
321            >>> index.delete(delete_all=True, namespace='my_namespace')
322            >>> index.delete(filter={'key': 'value'}, namespace='my_namespace')
323
324        Args:
325            ids (List[str]): Vector ids to delete [optional]
326            delete_all (bool): This indicates that all vectors in the index namespace should be deleted.. [optional]
327                                Default is False.
328            namespace (str): The namespace to delete vectors from [optional]
329                            If not specified, the default namespace is used.
330            filter (Dict[str, Union[str, float, int, bool, List, dict]]):
331                    If specified, the metadata filter here will be used to select the vectors to delete.
332                    This is mutually exclusive with specifying ids to delete in the ids param or using delete_all=True.
333                    See https://www.pinecone.io/docs/metadata-filtering/.. [optional]
334
335        Keyword Args:
336          Supports OpenAPI client keyword arguments. See pinecone.core.client.models.DeleteRequest for more details.
337
338
339          Returns: An empty dictionary if the delete operation was successful.
340        """
341        _check_type = kwargs.pop("_check_type", False)
342        args_dict = parse_non_empty_args(
343            [("ids", ids), ("delete_all", delete_all), ("namespace", namespace), ("filter", filter)]
344        )
345
346        return self._vector_api.delete(
347            DeleteRequest(
348                **args_dict,
349                **{
350                    k: v
351                    for k, v in kwargs.items()
352                    if k not in _OPENAPI_ENDPOINT_PARAMS and v is not None
353                },
354                _check_type=_check_type,
355            ),
356            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
357        )
358
359    @validate_and_convert_errors
360    def fetch(self, ids: List[str], namespace: Optional[str] = None, **kwargs) -> FetchResponse:
361        """
362        The fetch operation looks up and returns vectors, by ID, from a single namespace.
363        The returned vectors include the vector data and/or metadata.
364
365        API reference: https://docs.pinecone.io/reference/fetch
366
367        Examples:
368            >>> index.fetch(ids=['id1', 'id2'], namespace='my_namespace')
369            >>> index.fetch(ids=['id1', 'id2'])
370
371        Args:
372            ids (List[str]): The vector IDs to fetch.
373            namespace (str): The namespace to fetch vectors from.
374                             If not specified, the default namespace is used. [optional]
375        Keyword Args:
376            Supports OpenAPI client keyword arguments. See pinecone.core.client.models.FetchResponse for more details.
377
378
379        Returns: FetchResponse object which contains the list of Vector objects, and namespace name.
380        """
381        args_dict = parse_non_empty_args([("namespace", namespace)])
382        return self._vector_api.fetch(ids=ids, **args_dict, **kwargs)
383
384    @validate_and_convert_errors
385    def query(
386        self,
387        *args,
388        top_k: int,
389        vector: Optional[List[float]] = None,
390        id: Optional[str] = None,
391        namespace: Optional[str] = None,
392        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
393        include_values: Optional[bool] = None,
394        include_metadata: Optional[bool] = None,
395        sparse_vector: Optional[
396            Union[SparseValues, Dict[str, Union[List[float], List[int]]]]
397        ] = None,
398        **kwargs,
399    ) -> Union[QueryResponse, ApplyResult]:
400        """
401        The Query operation searches a namespace, using a query vector.
402        It retrieves the ids of the most similar items in a namespace, along with their similarity scores.
403
404        API reference: https://docs.pinecone.io/reference/query
405
406        Examples:
407            >>> index.query(vector=[1, 2, 3], top_k=10, namespace='my_namespace')
408            >>> index.query(id='id1', top_k=10, namespace='my_namespace')
409            >>> index.query(vector=[1, 2, 3], top_k=10, namespace='my_namespace', filter={'key': 'value'})
410            >>> index.query(id='id1', top_k=10, namespace='my_namespace', include_metadata=True, include_values=True)
411            >>> index.query(vector=[1, 2, 3], sparse_vector={'indices': [1, 2], 'values': [0.2, 0.4]},
412            >>>             top_k=10, namespace='my_namespace')
413            >>> index.query(vector=[1, 2, 3], sparse_vector=SparseValues([1, 2], [0.2, 0.4]),
414            >>>             top_k=10, namespace='my_namespace')
415
416        Args:
417            vector (List[float]): The query vector. This should be the same length as the dimension of the index
418                                  being queried. Each `query()` request can contain only one of the parameters
419                                  `id` or `vector`.. [optional]
420            id (str): The unique ID of the vector to be used as a query vector.
421                      Each `query()` request can contain only one of the parameters
422                      `vector` or  `id`. [optional]
423            top_k (int): The number of results to return for each query. Must be an integer greater than 1.
424            namespace (str): The namespace to fetch vectors from.
425                             If not specified, the default namespace is used. [optional]
426            filter (Dict[str, Union[str, float, int, bool, List, dict]):
427                    The filter to apply. You can use vector metadata to limit your search.
428                    See https://www.pinecone.io/docs/metadata-filtering/.. [optional]
429            include_values (bool): Indicates whether vector values are included in the response.
430                                   If omitted the server will use the default value of False [optional]
431            include_metadata (bool): Indicates whether metadata is included in the response as well as the ids.
432                                     If omitted the server will use the default value of False  [optional]
433            sparse_vector: (Union[SparseValues, Dict[str, Union[List[float], List[int]]]]): sparse values of the query vector.
434                            Expected to be either a SparseValues object or a dict of the form:
435                             {'indices': List[int], 'values': List[float]}, where the lists each have the same length.
436
437        Returns: QueryResponse object which contains the list of the closest vectors as ScoredVector objects,
438                 and namespace name.
439        """
440
441        response = self._query(
442            *args,
443            top_k=top_k,
444            vector=vector,
445            id=id,
446            namespace=namespace,
447            filter=filter,
448            include_values=include_values,
449            include_metadata=include_metadata,
450            sparse_vector=sparse_vector,
451            **kwargs,
452        )
453
454        if kwargs.get("async_req", False) or kwargs.get("async_threadpool_executor", False):
455            return response
456        else:
457            return parse_query_response(response)
458
459    def _query(
460        self,
461        *args,
462        top_k: int,
463        vector: Optional[List[float]] = None,
464        id: Optional[str] = None,
465        namespace: Optional[str] = None,
466        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
467        include_values: Optional[bool] = None,
468        include_metadata: Optional[bool] = None,
469        sparse_vector: Optional[
470            Union[SparseValues, Dict[str, Union[List[float], List[int]]]]
471        ] = None,
472        **kwargs,
473    ) -> QueryResponse:
474        if len(args) > 0:
475            raise ValueError(
476                "The argument order for `query()` has changed; please use keyword arguments instead of positional arguments. Example: index.query(vector=[0.1, 0.2, 0.3], top_k=10, namespace='my_namespace')"
477            )
478
479        if vector is not None and id is not None:
480            raise ValueError("Cannot specify both `id` and `vector`")
481
482        _check_type = kwargs.pop("_check_type", False)
483
484        sparse_vector = self._parse_sparse_values_arg(sparse_vector)
485        args_dict = parse_non_empty_args(
486            [
487                ("vector", vector),
488                ("id", id),
489                ("queries", None),
490                ("top_k", top_k),
491                ("namespace", namespace),
492                ("filter", filter),
493                ("include_values", include_values),
494                ("include_metadata", include_metadata),
495                ("sparse_vector", sparse_vector),
496            ]
497        )
498
499        response = self._vector_api.query(
500            QueryRequest(
501                **args_dict,
502                _check_type=_check_type,
503                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS},
504            ),
505            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
506        )
507        return response
508
509    @validate_and_convert_errors
510    def query_namespaces(
511        self,
512        vector: List[float],
513        namespaces: List[str],
514        top_k: Optional[int] = None,
515        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
516        include_values: Optional[bool] = None,
517        include_metadata: Optional[bool] = None,
518        sparse_vector: Optional[
519            Union[SparseValues, Dict[str, Union[List[float], List[int]]]]
520        ] = None,
521        **kwargs,
522    ) -> QueryNamespacesResults:
523        """The query_namespaces() method is used to make a query to multiple namespaces in parallel and combine the results into one result set.
524
525        Since several asynchronous calls are made on your behalf when calling this method, you will need to tune the pool_threads and connection_pool_maxsize parameter of the Index constructor to suite your workload.
526
527        Examples:
528
529        ```python
530        from pinecone import Pinecone
531
532        pc = Pinecone(api_key="your-api-key")
533        index = pc.Index(
534            host="index-name",
535            pool_threads=32,
536            connection_pool_maxsize=32
537        )
538
539        query_vec = [0.1, 0.2, 0.3] # An embedding that matches the index dimension
540        combined_results = index.query_namespaces(
541            vector=query_vec,
542            namespaces=['ns1', 'ns2', 'ns3', 'ns4'],
543            top_k=10,
544            filter={'genre': {"$eq": "drama"}},
545            include_values=True,
546            include_metadata=True
547        )
548        for vec in combined_results.matches:
549            print(vec.id, vec.score)
550        print(combined_results.usage)
551        ```
552
553        Args:
554            vector (List[float]): The query vector, must be the same length as the dimension of the index being queried.
555            namespaces (List[str]): The list of namespaces to query.
556            top_k (Optional[int], optional): The number of results you would like to request from each namespace. Defaults to 10.
557            filter (Optional[Dict[str, Union[str, float, int, bool, List, dict]]], optional): Pass an optional filter to filter results based on metadata. Defaults to None.
558            include_values (Optional[bool], optional): Boolean field indicating whether vector values should be included with results. Defaults to None.
559            include_metadata (Optional[bool], optional): Boolean field indicating whether vector metadata should be included with results. Defaults to None.
560            sparse_vector (Optional[ Union[SparseValues, Dict[str, Union[List[float], List[int]]]] ], optional): If you are working with a dotproduct index, you can pass a sparse vector as part of your hybrid search. Defaults to None.
561
562        Returns:
563            QueryNamespacesResults: A QueryNamespacesResults object containing the combined results from all namespaces, as well as the combined usage cost in read units.
564        """
565        if namespaces is None or len(namespaces) == 0:
566            raise ValueError("At least one namespace must be specified")
567        if len(vector) == 0:
568            raise ValueError("Query vector must not be empty")
569
570        overall_topk = top_k if top_k is not None else 10
571        aggregator = QueryResultsAggregator(top_k=overall_topk)
572
573        target_namespaces = set(namespaces)  # dedup namespaces
574        async_futures = [
575            self.query(
576                vector=vector,
577                namespace=ns,
578                top_k=overall_topk,
579                filter=filter,
580                include_values=include_values,
581                include_metadata=include_metadata,
582                sparse_vector=sparse_vector,
583                async_threadpool_executor=True,
584                _preload_content=False,
585                **kwargs,
586            )
587            for ns in target_namespaces
588        ]
589
590        for result in as_completed(async_futures):
591            raw_result = result.result()
592            response = json.loads(raw_result.data.decode("utf-8"))
593            aggregator.add_results(response)
594
595        final_results = aggregator.get_results()
596        return final_results
597
598    @validate_and_convert_errors
599    def update(
600        self,
601        id: str,
602        values: Optional[List[float]] = None,
603        set_metadata: Optional[
604            Dict[str, Union[str, float, int, bool, List[int], List[float], List[str]]]
605        ] = None,
606        namespace: Optional[str] = None,
607        sparse_values: Optional[
608            Union[SparseValues, Dict[str, Union[List[float], List[int]]]]
609        ] = None,
610        **kwargs,
611    ) -> Dict[str, Any]:
612        """
613        The Update operation updates vector in a namespace.
614        If a value is included, it will overwrite the previous value.
615        If a set_metadata is included,
616        the values of the fields specified in it will be added or overwrite the previous value.
617
618        API reference: https://docs.pinecone.io/reference/update
619
620        Examples:
621            >>> index.update(id='id1', values=[1, 2, 3], namespace='my_namespace')
622            >>> index.update(id='id1', set_metadata={'key': 'value'}, namespace='my_namespace')
623            >>> index.update(id='id1', values=[1, 2, 3], sparse_values={'indices': [1, 2], 'values': [0.2, 0.4]},
624            >>>              namespace='my_namespace')
625            >>> index.update(id='id1', values=[1, 2, 3], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]),
626            >>>              namespace='my_namespace')
627
628        Args:
629            id (str): Vector's unique id.
630            values (List[float]): vector values to set. [optional]
631            set_metadata (Dict[str, Union[str, float, int, bool, List[int], List[float], List[str]]]]):
632                metadata to set for vector. [optional]
633            namespace (str): Namespace name where to update the vector.. [optional]
634            sparse_values: (Dict[str, Union[List[float], List[int]]]): sparse values to update for the vector.
635                           Expected to be either a SparseValues object or a dict of the form:
636                           {'indices': List[int], 'values': List[float]} where the lists each have the same length.
637
638        Keyword Args:
639            Supports OpenAPI client keyword arguments. See pinecone.core.client.models.UpdateRequest for more details.
640
641        Returns: An empty dictionary if the update was successful.
642        """
643        _check_type = kwargs.pop("_check_type", False)
644        sparse_values = self._parse_sparse_values_arg(sparse_values)
645        args_dict = parse_non_empty_args(
646            [
647                ("values", values),
648                ("set_metadata", set_metadata),
649                ("namespace", namespace),
650                ("sparse_values", sparse_values),
651            ]
652        )
653        return self._vector_api.update(
654            UpdateRequest(
655                id=id,
656                **args_dict,
657                _check_type=_check_type,
658                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS},
659            ),
660            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
661        )
662
663    @validate_and_convert_errors
664    def describe_index_stats(
665        self, filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None, **kwargs
666    ) -> DescribeIndexStatsResponse:
667        """
668        The DescribeIndexStats operation returns statistics about the index's contents.
669        For example: The vector count per namespace and the number of dimensions.
670
671        API reference: https://docs.pinecone.io/reference/describe_index_stats_post
672
673        Examples:
674            >>> index.describe_index_stats()
675            >>> index.describe_index_stats(filter={'key': 'value'})
676
677        Args:
678            filter (Dict[str, Union[str, float, int, bool, List, dict]]):
679            If this parameter is present, the operation only returns statistics for vectors that satisfy the filter.
680            See https://www.pinecone.io/docs/metadata-filtering/.. [optional]
681
682        Returns: DescribeIndexStatsResponse object which contains stats about the index.
683        """
684        _check_type = kwargs.pop("_check_type", False)
685        args_dict = parse_non_empty_args([("filter", filter)])
686
687        return self._vector_api.describe_index_stats(
688            DescribeIndexStatsRequest(
689                **args_dict,
690                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS},
691                _check_type=_check_type,
692            ),
693            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
694        )
695
696    @validate_and_convert_errors
697    def list_paginated(
698        self,
699        prefix: Optional[str] = None,
700        limit: Optional[int] = None,
701        pagination_token: Optional[str] = None,
702        namespace: Optional[str] = None,
703        **kwargs,
704    ) -> ListResponse:
705        """
706        The list_paginated operation finds vectors based on an id prefix within a single namespace.
707        It returns matching ids in a paginated form, with a pagination token to fetch the next page of results.
708        This id list can then be passed to fetch or delete operations, depending on your use case.
709
710        Consider using the `list` method to avoid having to handle pagination tokens manually.
711
712        Examples:
713            >>> results = index.list_paginated(prefix='99', limit=5, namespace='my_namespace')
714            >>> [v.id for v in results.vectors]
715            ['99', '990', '991', '992', '993']
716            >>> results.pagination.next
717            eyJza2lwX3Bhc3QiOiI5OTMiLCJwcmVmaXgiOiI5OSJ9
718            >>> next_results = index.list_paginated(prefix='99', limit=5, namespace='my_namespace', pagination_token=results.pagination.next)
719
720        Args:
721            prefix (Optional[str]): The id prefix to match. If unspecified, an empty string prefix will
722                                    be used with the effect of listing all ids in a namespace [optional]
723            limit (Optional[int]): The maximum number of ids to return. If unspecified, the server will use a default value. [optional]
724            pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned
725                in the response if additional results are available. [optional]
726            namespace (Optional[str]): The namespace to fetch vectors from. If not specified, the default namespace is used. [optional]
727
728        Returns: ListResponse object which contains the list of ids, the namespace name, pagination information, and usage showing the number of read_units consumed.
729        """
730        args_dict = parse_non_empty_args(
731            [
732                ("prefix", prefix),
733                ("limit", limit),
734                ("namespace", namespace),
735                ("pagination_token", pagination_token),
736            ]
737        )
738        return self._vector_api.list(**args_dict, **kwargs)
739
740    @validate_and_convert_errors
741    def list(self, **kwargs):
742        """
743        The list operation accepts all of the same arguments as list_paginated, and returns a generator that yields
744        a list of the matching vector ids in each page of results. It automatically handles pagination tokens on your
745        behalf.
746
747        Examples:
748            >>> for ids in index.list(prefix='99', limit=5, namespace='my_namespace'):
749            >>>     print(ids)
750            ['99', '990', '991', '992', '993']
751            ['994', '995', '996', '997', '998']
752            ['999']
753
754        Args:
755            prefix (Optional[str]): The id prefix to match. If unspecified, an empty string prefix will
756                                    be used with the effect of listing all ids in a namespace [optional]
757            limit (Optional[int]): The maximum number of ids to return. If unspecified, the server will use a default value. [optional]
758            pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned
759                in the response if additional results are available. [optional]
760            namespace (Optional[str]): The namespace to fetch vectors from. If not specified, the default namespace is used. [optional]
761        """
762        done = False
763        while not done:
764            results = self.list_paginated(**kwargs)
765            if len(results.vectors) > 0:
766                yield [v.id for v in results.vectors]
767
768            if results.pagination:
769                kwargs.update({"pagination_token": results.pagination.next})
770            else:
771                done = True
772
773    @staticmethod
774    def _parse_sparse_values_arg(
775        sparse_values: Optional[Union[SparseValues, Dict[str, Union[List[float], List[int]]]]],
776    ) -> Optional[SparseValues]:
777        if sparse_values is None:
778            return None
779
780        if isinstance(sparse_values, SparseValues):
781            return sparse_values
782
783        if (
784            not isinstance(sparse_values, dict)
785            or "indices" not in sparse_values
786            or "values" not in sparse_values
787        ):
788            raise ValueError(
789                "Invalid sparse values argument. Expected a dict of: {'indices': List[int], 'values': List[float]}."
790                f"Received: {sparse_values}"
791            )
792
793        return SparseValues(indices=sparse_values["indices"], values=sparse_values["values"])