pinecone.data.index

  1from tqdm.autonotebook import tqdm
  2
  3from typing import Union, List, Tuple, Optional, Dict, Any
  4
  5from pinecone.config import ConfigBuilder
  6
  7from pinecone.core.openapi.shared import API_VERSION
  8from pinecone.core.openapi.data.models import SparseValues
  9from pinecone.core.openapi.data import ApiClient
 10from pinecone.core.openapi.data.models import (
 11    FetchResponse,
 12    QueryRequest,
 13    QueryResponse,
 14    RpcStatus,
 15    ScoredVector,
 16    SingleQueryResults,
 17    DescribeIndexStatsResponse,
 18    UpsertRequest,
 19    UpsertResponse,
 20    UpdateRequest,
 21    Vector,
 22    DeleteRequest,
 23    UpdateRequest,
 24    DescribeIndexStatsRequest,
 25    ListResponse,
 26)
 27from .features.bulk_import import ImportFeatureMixin
 28from pinecone.core.openapi.data.api.data_plane_api import DataPlaneApi
 29from ..utils import setup_openapi_client, parse_non_empty_args
 30from .vector_factory import VectorFactory
 31
 32__all__ = [
 33    "Index",
 34    "FetchResponse",
 35    "QueryRequest",
 36    "QueryResponse",
 37    "RpcStatus",
 38    "ScoredVector",
 39    "SingleQueryResults",
 40    "DescribeIndexStatsResponse",
 41    "UpsertRequest",
 42    "UpsertResponse",
 43    "UpdateRequest",
 44    "Vector",
 45    "DeleteRequest",
 46    "UpdateRequest",
 47    "DescribeIndexStatsRequest",
 48    "SparseValues",
 49]
 50
 51from ..utils.error_handling import validate_and_convert_errors
 52
 53_OPENAPI_ENDPOINT_PARAMS = (
 54    "_return_http_data_only",
 55    "_preload_content",
 56    "_request_timeout",
 57    "_check_input_type",
 58    "_check_return_type",
 59    "_host_index",
 60    "async_req",
 61)
 62
 63
 64def parse_query_response(response: QueryResponse):
 65    response._data_store.pop("results", None)
 66    return response
 67
 68
 69class Index(ImportFeatureMixin):
 70    """
 71    A client for interacting with a Pinecone index via REST API.
 72    For improved performance, use the Pinecone GRPC index client.
 73    """
 74
 75    def __init__(
 76        self,
 77        api_key: str,
 78        host: str,
 79        pool_threads: Optional[int] = 1,
 80        additional_headers: Optional[Dict[str, str]] = {},
 81        openapi_config=None,
 82        **kwargs,
 83    ):
 84        super().__init__(
 85            api_key=api_key,
 86            host=host,
 87            pool_threads=pool_threads,
 88            additional_headers=additional_headers,
 89            openapi_config=openapi_config,
 90            **kwargs,
 91        )
 92
 93        self._config = ConfigBuilder.build(
 94            api_key=api_key,
 95            host=host,
 96            additional_headers=additional_headers,
 97            **kwargs,
 98        )
 99        openapi_config = ConfigBuilder.build_openapi_config(self._config, openapi_config)
100
101        self._vector_api = setup_openapi_client(
102            api_client_klass=ApiClient,
103            api_klass=DataPlaneApi,
104            config=self._config,
105            openapi_config=openapi_config,
106            pool_threads=pool_threads,
107            api_version=API_VERSION,
108        )
109
110    def __enter__(self):
111        return self
112
113    def __exit__(self, exc_type, exc_value, traceback):
114        self._vector_api.api_client.close()
115
116    @validate_and_convert_errors
117    def upsert(
118        self,
119        vectors: Union[List[Vector], List[tuple], List[dict]],
120        namespace: Optional[str] = None,
121        batch_size: Optional[int] = None,
122        show_progress: bool = True,
123        **kwargs,
124    ) -> UpsertResponse:
125        """
126        The upsert operation writes vectors into a namespace.
127        If a new value is upserted for an existing vector id, it will overwrite the previous value.
128
129        To upsert in parallel follow: https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel
130
131        A vector can be represented by a 1) Vector object, a 2) tuple or 3) a dictionary
132
133        If a tuple is used, it must be of the form `(id, values, metadata)` or `(id, values)`.
134        where id is a string, vector is a list of floats, metadata is a dict,
135        and sparse_values is a dict of the form `{'indices': List[int], 'values': List[float]}`.
136
137        Examples:
138            >>> ('id1', [1.0, 2.0, 3.0], {'key': 'value'}, {'indices': [1, 2], 'values': [0.2, 0.4]})
139            >>> ('id1', [1.0, 2.0, 3.0], None, {'indices': [1, 2], 'values': [0.2, 0.4]})
140            >>> ('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])
141
142        If a Vector object is used, a Vector object must be of the form
143        `Vector(id, values, metadata, sparse_values)`, where metadata and sparse_values are optional
144        arguments.
145
146        Examples:
147            >>> Vector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'})
148            >>> Vector(id='id2', values=[1.0, 2.0, 3.0])
149            >>> Vector(id='id3', values=[1.0, 2.0, 3.0], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]))
150
151        **Note:** the dimension of each vector must match the dimension of the index.
152
153        If a dictionary is used, it must be in the form `{'id': str, 'values': List[float], 'sparse_values': {'indices': List[int], 'values': List[float]}, 'metadata': dict}`
154
155        Examples:
156            >>> index.upsert([('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])])
157            >>>
158            >>> index.upsert([{'id': 'id1', 'values': [1.0, 2.0, 3.0], 'metadata': {'key': 'value'}},
159            >>>               {'id': 'id2', 'values': [1.0, 2.0, 3.0], 'sparse_values': {'indices': [1, 8], 'values': [0.2, 0.4]}])
160            >>> index.upsert([Vector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'}),
161            >>>               Vector(id='id2', values=[1.0, 2.0, 3.0], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]))])
162
163        API reference: https://docs.pinecone.io/reference/upsert
164
165        Args:
166            vectors (Union[List[Vector], List[Tuple]]): A list of vectors to upsert.
167            namespace (str): The namespace to write to. If not specified, the default namespace is used. [optional]
168            batch_size (int): The number of vectors to upsert in each batch.
169                               If not specified, all vectors will be upserted in a single batch. [optional]
170            show_progress (bool): Whether to show a progress bar using tqdm.
171                                  Applied only if batch_size is provided. Default is True.
172        Keyword Args:
173            Supports OpenAPI client keyword arguments. See pinecone.core.client.models.UpsertRequest for more details.
174
175        Returns: UpsertResponse, includes the number of vectors upserted.
176        """
177        _check_type = kwargs.pop("_check_type", True)
178
179        if kwargs.get("async_req", False) and batch_size is not None:
180            raise ValueError(
181                "async_req is not supported when batch_size is provided."
182                "To upsert in parallel, please follow: "
183                "https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel"
184            )
185
186        if batch_size is None:
187            return self._upsert_batch(vectors, namespace, _check_type, **kwargs)
188
189        if not isinstance(batch_size, int) or batch_size <= 0:
190            raise ValueError("batch_size must be a positive integer")
191
192        pbar = tqdm(
193            total=len(vectors),
194            disable=not show_progress,
195            desc="Upserted vectors",
196        )
197        total_upserted = 0
198        for i in range(0, len(vectors), batch_size):
199            batch_result = self._upsert_batch(vectors[i : i + batch_size], namespace, _check_type, **kwargs)
200            pbar.update(batch_result.upserted_count)
201            # we can't use here pbar.n for the case show_progress=False
202            total_upserted += batch_result.upserted_count
203
204        return UpsertResponse(upserted_count=total_upserted)
205
206    def _upsert_batch(
207        self,
208        vectors: Union[List[Vector], List[tuple], List[dict]],
209        namespace: Optional[str],
210        _check_type: bool,
211        **kwargs,
212    ) -> UpsertResponse:
213        args_dict = parse_non_empty_args([("namespace", namespace)])
214        vec_builder = lambda v: VectorFactory.build(v, check_type=_check_type)
215
216        return self._vector_api.upsert(
217            UpsertRequest(
218                vectors=list(map(vec_builder, vectors)),
219                **args_dict,
220                _check_type=_check_type,
221                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS},
222            ),
223            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
224        )
225
226    @staticmethod
227    def _iter_dataframe(df, batch_size):
228        for i in range(0, len(df), batch_size):
229            batch = df.iloc[i : i + batch_size].to_dict(orient="records")
230            yield batch
231
232    def upsert_from_dataframe(
233        self,
234        df,
235        namespace: Optional[str] = None,
236        batch_size: int = 500,
237        show_progress: bool = True,
238    ) -> UpsertResponse:
239        """Upserts a dataframe into the index.
240
241        Args:
242            df: A pandas dataframe with the following columns: id, values, sparse_values, and metadata.
243            namespace: The namespace to upsert into.
244            batch_size: The number of rows to upsert in a single batch.
245            show_progress: Whether to show a progress bar.
246        """
247        try:
248            import pandas as pd
249        except ImportError:
250            raise RuntimeError(
251                "The `pandas` package is not installed. Please install pandas to use `upsert_from_dataframe()`"
252            )
253
254        if not isinstance(df, pd.DataFrame):
255            raise ValueError(f"Only pandas dataframes are supported. Found: {type(df)}")
256
257        pbar = tqdm(
258            total=len(df),
259            disable=not show_progress,
260            desc="sending upsert requests",
261        )
262        results = []
263        for chunk in self._iter_dataframe(df, batch_size=batch_size):
264            res = self.upsert(vectors=chunk, namespace=namespace)
265            pbar.update(len(chunk))
266            results.append(res)
267
268        upserted_count = 0
269        for res in results:
270            upserted_count += res.upserted_count
271
272        return UpsertResponse(upserted_count=upserted_count)
273
274    @validate_and_convert_errors
275    def delete(
276        self,
277        ids: Optional[List[str]] = None,
278        delete_all: Optional[bool] = None,
279        namespace: Optional[str] = None,
280        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
281        **kwargs,
282    ) -> Dict[str, Any]:
283        """
284        The Delete operation deletes vectors from the index, from a single namespace.
285        No error raised if the vector id does not exist.
286        Note: for any delete call, if namespace is not specified, the default namespace is used.
287
288        Delete can occur in the following mutual exclusive ways:
289        1. Delete by ids from a single namespace
290        2. Delete all vectors from a single namespace by setting delete_all to True
291        3. Delete all vectors from a single namespace by specifying a metadata filter
292            (note that for this option delete all must be set to False)
293
294        API reference: https://docs.pinecone.io/reference/delete_post
295
296        Examples:
297            >>> index.delete(ids=['id1', 'id2'], namespace='my_namespace')
298            >>> index.delete(delete_all=True, namespace='my_namespace')
299            >>> index.delete(filter={'key': 'value'}, namespace='my_namespace')
300
301        Args:
302            ids (List[str]): Vector ids to delete [optional]
303            delete_all (bool): This indicates that all vectors in the index namespace should be deleted.. [optional]
304                                Default is False.
305            namespace (str): The namespace to delete vectors from [optional]
306                            If not specified, the default namespace is used.
307            filter (Dict[str, Union[str, float, int, bool, List, dict]]):
308                    If specified, the metadata filter here will be used to select the vectors to delete.
309                    This is mutually exclusive with specifying ids to delete in the ids param or using delete_all=True.
310                    See https://www.pinecone.io/docs/metadata-filtering/.. [optional]
311
312        Keyword Args:
313          Supports OpenAPI client keyword arguments. See pinecone.core.client.models.DeleteRequest for more details.
314
315
316          Returns: An empty dictionary if the delete operation was successful.
317        """
318        _check_type = kwargs.pop("_check_type", False)
319        args_dict = parse_non_empty_args(
320            [
321                ("ids", ids),
322                ("delete_all", delete_all),
323                ("namespace", namespace),
324                ("filter", filter),
325            ]
326        )
327
328        return self._vector_api.delete(
329            DeleteRequest(
330                **args_dict,
331                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS and v is not None},
332                _check_type=_check_type,
333            ),
334            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
335        )
336
337    @validate_and_convert_errors
338    def fetch(self, ids: List[str], namespace: Optional[str] = None, **kwargs) -> FetchResponse:
339        """
340        The fetch operation looks up and returns vectors, by ID, from a single namespace.
341        The returned vectors include the vector data and/or metadata.
342
343        API reference: https://docs.pinecone.io/reference/fetch
344
345        Examples:
346            >>> index.fetch(ids=['id1', 'id2'], namespace='my_namespace')
347            >>> index.fetch(ids=['id1', 'id2'])
348
349        Args:
350            ids (List[str]): The vector IDs to fetch.
351            namespace (str): The namespace to fetch vectors from.
352                             If not specified, the default namespace is used. [optional]
353        Keyword Args:
354            Supports OpenAPI client keyword arguments. See pinecone.core.client.models.FetchResponse for more details.
355
356
357        Returns: FetchResponse object which contains the list of Vector objects, and namespace name.
358        """
359        args_dict = parse_non_empty_args([("namespace", namespace)])
360        return self._vector_api.fetch(ids=ids, **args_dict, **kwargs)
361
362    @validate_and_convert_errors
363    def query(
364        self,
365        *args,
366        top_k: int,
367        vector: Optional[List[float]] = None,
368        id: Optional[str] = None,
369        namespace: Optional[str] = None,
370        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
371        include_values: Optional[bool] = None,
372        include_metadata: Optional[bool] = None,
373        sparse_vector: Optional[Union[SparseValues, Dict[str, Union[List[float], List[int]]]]] = None,
374        **kwargs,
375    ) -> QueryResponse:
376        """
377        The Query operation searches a namespace, using a query vector.
378        It retrieves the ids of the most similar items in a namespace, along with their similarity scores.
379
380        API reference: https://docs.pinecone.io/reference/query
381
382        Examples:
383            >>> index.query(vector=[1, 2, 3], top_k=10, namespace='my_namespace')
384            >>> index.query(id='id1', top_k=10, namespace='my_namespace')
385            >>> index.query(vector=[1, 2, 3], top_k=10, namespace='my_namespace', filter={'key': 'value'})
386            >>> index.query(id='id1', top_k=10, namespace='my_namespace', include_metadata=True, include_values=True)
387            >>> index.query(vector=[1, 2, 3], sparse_vector={'indices': [1, 2], 'values': [0.2, 0.4]},
388            >>>             top_k=10, namespace='my_namespace')
389            >>> index.query(vector=[1, 2, 3], sparse_vector=SparseValues([1, 2], [0.2, 0.4]),
390            >>>             top_k=10, namespace='my_namespace')
391
392        Args:
393            vector (List[float]): The query vector. This should be the same length as the dimension of the index
394                                  being queried. Each `query()` request can contain only one of the parameters
395                                  `id` or `vector`.. [optional]
396            id (str): The unique ID of the vector to be used as a query vector.
397                      Each `query()` request can contain only one of the parameters
398                      `vector` or  `id`. [optional]
399            top_k (int): The number of results to return for each query. Must be an integer greater than 1.
400            namespace (str): The namespace to fetch vectors from.
401                             If not specified, the default namespace is used. [optional]
402            filter (Dict[str, Union[str, float, int, bool, List, dict]):
403                    The filter to apply. You can use vector metadata to limit your search.
404                    See https://www.pinecone.io/docs/metadata-filtering/.. [optional]
405            include_values (bool): Indicates whether vector values are included in the response.
406                                   If omitted the server will use the default value of False [optional]
407            include_metadata (bool): Indicates whether metadata is included in the response as well as the ids.
408                                     If omitted the server will use the default value of False  [optional]
409            sparse_vector: (Union[SparseValues, Dict[str, Union[List[float], List[int]]]]): sparse values of the query vector.
410                            Expected to be either a SparseValues object or a dict of the form:
411                             {'indices': List[int], 'values': List[float]}, where the lists each have the same length.
412
413        Returns: QueryResponse object which contains the list of the closest vectors as ScoredVector objects,
414                 and namespace name.
415        """
416
417        if len(args) > 0:
418            raise ValueError(
419                "The argument order for `query()` has changed; please use keyword arguments instead of positional arguments. Example: index.query(vector=[0.1, 0.2, 0.3], top_k=10, namespace='my_namespace')"
420            )
421
422        if vector is not None and id is not None:
423            raise ValueError("Cannot specify both `id` and `vector`")
424
425        _check_type = kwargs.pop("_check_type", False)
426
427        sparse_vector = self._parse_sparse_values_arg(sparse_vector)
428        args_dict = parse_non_empty_args(
429            [
430                ("vector", vector),
431                ("id", id),
432                ("queries", None),
433                ("top_k", top_k),
434                ("namespace", namespace),
435                ("filter", filter),
436                ("include_values", include_values),
437                ("include_metadata", include_metadata),
438                ("sparse_vector", sparse_vector),
439            ]
440        )
441        response = self._vector_api.query(
442            QueryRequest(
443                **args_dict,
444                _check_type=_check_type,
445                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS},
446            ),
447            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
448        )
449        return parse_query_response(response)
450
451    @validate_and_convert_errors
452    def update(
453        self,
454        id: str,
455        values: Optional[List[float]] = None,
456        set_metadata: Optional[
457            Dict[
458                str,
459                Union[str, float, int, bool, List[int], List[float], List[str]],
460            ]
461        ] = None,
462        namespace: Optional[str] = None,
463        sparse_values: Optional[Union[SparseValues, Dict[str, Union[List[float], List[int]]]]] = None,
464        **kwargs,
465    ) -> Dict[str, Any]:
466        """
467        The Update operation updates vector in a namespace.
468        If a value is included, it will overwrite the previous value.
469        If a set_metadata is included,
470        the values of the fields specified in it will be added or overwrite the previous value.
471
472        API reference: https://docs.pinecone.io/reference/update
473
474        Examples:
475            >>> index.update(id='id1', values=[1, 2, 3], namespace='my_namespace')
476            >>> index.update(id='id1', set_metadata={'key': 'value'}, namespace='my_namespace')
477            >>> index.update(id='id1', values=[1, 2, 3], sparse_values={'indices': [1, 2], 'values': [0.2, 0.4]},
478            >>>              namespace='my_namespace')
479            >>> index.update(id='id1', values=[1, 2, 3], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]),
480            >>>              namespace='my_namespace')
481
482        Args:
483            id (str): Vector's unique id.
484            values (List[float]): vector values to set. [optional]
485            set_metadata (Dict[str, Union[str, float, int, bool, List[int], List[float], List[str]]]]):
486                metadata to set for vector. [optional]
487            namespace (str): Namespace name where to update the vector.. [optional]
488            sparse_values: (Dict[str, Union[List[float], List[int]]]): sparse values to update for the vector.
489                           Expected to be either a SparseValues object or a dict of the form:
490                           {'indices': List[int], 'values': List[float]} where the lists each have the same length.
491
492        Keyword Args:
493            Supports OpenAPI client keyword arguments. See pinecone.core.client.models.UpdateRequest for more details.
494
495        Returns: An empty dictionary if the update was successful.
496        """
497        _check_type = kwargs.pop("_check_type", False)
498        sparse_values = self._parse_sparse_values_arg(sparse_values)
499        args_dict = parse_non_empty_args(
500            [
501                ("values", values),
502                ("set_metadata", set_metadata),
503                ("namespace", namespace),
504                ("sparse_values", sparse_values),
505            ]
506        )
507        return self._vector_api.update(
508            UpdateRequest(
509                id=id,
510                **args_dict,
511                _check_type=_check_type,
512                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS},
513            ),
514            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
515        )
516
517    @validate_and_convert_errors
518    def describe_index_stats(
519        self,
520        filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None,
521        **kwargs,
522    ) -> DescribeIndexStatsResponse:
523        """
524        The DescribeIndexStats operation returns statistics about the index's contents.
525        For example: The vector count per namespace and the number of dimensions.
526
527        API reference: https://docs.pinecone.io/reference/describe_index_stats_post
528
529        Examples:
530            >>> index.describe_index_stats()
531            >>> index.describe_index_stats(filter={'key': 'value'})
532
533        Args:
534            filter (Dict[str, Union[str, float, int, bool, List, dict]]):
535            If this parameter is present, the operation only returns statistics for vectors that satisfy the filter.
536            See https://www.pinecone.io/docs/metadata-filtering/.. [optional]
537
538        Returns: DescribeIndexStatsResponse object which contains stats about the index.
539        """
540        _check_type = kwargs.pop("_check_type", False)
541        args_dict = parse_non_empty_args([("filter", filter)])
542
543        return self._vector_api.describe_index_stats(
544            DescribeIndexStatsRequest(
545                **args_dict,
546                **{k: v for k, v in kwargs.items() if k not in _OPENAPI_ENDPOINT_PARAMS},
547                _check_type=_check_type,
548            ),
549            **{k: v for k, v in kwargs.items() if k in _OPENAPI_ENDPOINT_PARAMS},
550        )
551
552    @validate_and_convert_errors
553    def list_paginated(
554        self,
555        prefix: Optional[str] = None,
556        limit: Optional[int] = None,
557        pagination_token: Optional[str] = None,
558        namespace: Optional[str] = None,
559        **kwargs,
560    ) -> ListResponse:
561        """
562        The list_paginated operation finds vectors based on an id prefix within a single namespace.
563        It returns matching ids in a paginated form, with a pagination token to fetch the next page of results.
564        This id list can then be passed to fetch or delete operations, depending on your use case.
565
566        Consider using the `list` method to avoid having to handle pagination tokens manually.
567
568        Examples:
569            >>> results = index.list_paginated(prefix='99', limit=5, namespace='my_namespace')
570            >>> [v.id for v in results.vectors]
571            ['99', '990', '991', '992', '993']
572            >>> results.pagination.next
573            eyJza2lwX3Bhc3QiOiI5OTMiLCJwcmVmaXgiOiI5OSJ9
574            >>> next_results = index.list_paginated(prefix='99', limit=5, namespace='my_namespace', pagination_token=results.pagination.next)
575
576        Args:
577            prefix (Optional[str]): The id prefix to match. If unspecified, an empty string prefix will
578                                    be used with the effect of listing all ids in a namespace [optional]
579            limit (Optional[int]): The maximum number of ids to return. If unspecified, the server will use a default value. [optional]
580            pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned
581                in the response if additional results are available. [optional]
582            namespace (Optional[str]): The namespace to fetch vectors from. If not specified, the default namespace is used. [optional]
583
584        Returns: ListResponse object which contains the list of ids, the namespace name, pagination information, and usage showing the number of read_units consumed.
585        """
586        args_dict = parse_non_empty_args(
587            [
588                ("prefix", prefix),
589                ("limit", limit),
590                ("namespace", namespace),
591                ("pagination_token", pagination_token),
592            ]
593        )
594        return self._vector_api.list(**args_dict, **kwargs)
595
596    @validate_and_convert_errors
597    def list(self, **kwargs):
598        """
599        The list operation accepts all of the same arguments as list_paginated, and returns a generator that yields
600        a list of the matching vector ids in each page of results. It automatically handles pagination tokens on your
601        behalf.
602
603        Examples:
604            >>> for ids in index.list(prefix='99', limit=5, namespace='my_namespace'):
605            >>>     print(ids)
606            ['99', '990', '991', '992', '993']
607            ['994', '995', '996', '997', '998']
608            ['999']
609
610        Args:
611            prefix (Optional[str]): The id prefix to match. If unspecified, an empty string prefix will
612                                    be used with the effect of listing all ids in a namespace [optional]
613            limit (Optional[int]): The maximum number of ids to return. If unspecified, the server will use a default value. [optional]
614            pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned
615                in the response if additional results are available. [optional]
616            namespace (Optional[str]): The namespace to fetch vectors from. If not specified, the default namespace is used. [optional]
617        """
618        done = False
619        while not done:
620            results = self.list_paginated(**kwargs)
621            if len(results.vectors) > 0:
622                yield [v.id for v in results.vectors]
623
624            if results.pagination:
625                kwargs.update({"pagination_token": results.pagination.next})
626            else:
627                done = True
628
629    @staticmethod
630    def _parse_sparse_values_arg(
631        sparse_values: Optional[Union[SparseValues, Dict[str, Union[List[float], List[int]]]]]
632    ) -> Optional[SparseValues]:
633        if sparse_values is None:
634            return None
635
636        if isinstance(sparse_values, SparseValues):
637            return sparse_values
638
639        if not isinstance(sparse_values, dict) or "indices" not in sparse_values or "values" not in sparse_values:
640            raise ValueError(
641                "Invalid sparse values argument. Expected a dict of: {'indices': List[int], 'values': List[float]}."
642                f"Received: {sparse_values}"
643            )
644
645        return SparseValues(indices=sparse_values["indices"], values=sparse_values["values"])