tests.system.aiplatform.test_matching_engine_index.TestMatchingEngine: test_create_get_list_matching_engine_index failed

Note: #2305 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

----

commit: 0fa47aea08b635159e9c6dc64f7d330f047dfb5d
buildURL: [Build Status](https://source.cloud.google.com/results/invocations/6a410ac8-ed78-42d5-81dc-9db12d6aa1fb), [Sponge](http://sponge2/6a410ac8-ed78-42d5-81dc-9db12d6aa1fb)
status: failed
Test output

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"

index_endpoint {

  display_name: "public_endpoint_name"

 ...ption: "my public endpoint"

  labels {

    key: "my_key"

    value: "my_value"

  }

  public_endpoint_enabled: true

}

,)

kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-client', 'model-builder/1.32.0 gl-python/3.11.1 grpc/1.58.0 gax/2.11.1 gapic/1.32.0')]}


    @functools.wraps(callable_)

    def error_remapped_callable(*args, **kwargs):

        try:

>           return callable_(*args, **kwargs)

.nox/system-3-11/lib/python3.11/site-packages/google/api_core/grpc_helpers.py:72: 

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

.nox/system-3-11/lib/python3.11/site-packages/grpc/_channel.py:1161: in __call__

    return _end_unary_response_blocking(state, call, False, None)

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = 

call = 

with_call = False, deadline = None

    def _end_unary_response_blocking(

        state: _RPCState,

        call: cygrpc.SegregatedCall,

        with_call: bool,

        deadline: Optional[float],

    ) -> Union[ResponseType, Tuple[ResponseType, grpc.Call]]:

        if state.code is grpc.StatusCode.OK:

            if with_call:

                rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)

                return state.response, rendezvous

            else:

                return state.response

        else:

>           raise _InactiveRpcError(state)  # pytype: disable=not-instantiable

E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.FAILED_PRECONDITION

E           	details = "Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."

E           	debug_error_string = "UNKNOWN:Error received from peer ipv4:74.125.197.95:443 {created_time:"2023-09-13T10:40:17.014736581+00:00", grpc_status:9, grpc_message:"Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."}"

E           >

.nox/system-3-11/lib/python3.11/site-packages/grpc/_channel.py:1004: _InactiveRpcError

The above exception was the direct cause of the following exception:

self = 

shared_state = {'index': ...dex object at 0x7ff89492d950> 

resource name: projects/580378083368/locations/us-central1/indexes/4011431834498891776]}

    def test_create_get_list_matching_engine_index(self, shared_state):

        aiplatform.init(

            project=e2e_base._PROJECT,

            location=e2e_base._LOCATION,

        )

    

        # Create an index

        index = aiplatform.MatchingEngineIndex.create_tree_ah_index(

            display_name=_TEST_INDEX_DISPLAY_NAME,

            contents_delta_uri=_TEST_CONTENTS_DELTA_URI,

            dimensions=_TEST_INDEX_CONFIG_DIMENSIONS,

            approximate_neighbors_count=_TEST_INDEX_APPROXIMATE_NEIGHBORS_COUNT,

            distance_measure_type=_TEST_INDEX_DISTANCE_MEASURE_TYPE,

            leaf_node_embedding_count=_TEST_LEAF_NODE_EMBEDDING_COUNT,

            leaf_nodes_to_search_percent=_TEST_LEAF_NODES_TO_SEARCH_PERCENT,

            description=_TEST_INDEX_DESCRIPTION,

            labels=_TEST_LABELS,

        )

    

        shared_state["resources"] = [index]

        shared_state["index"] = index

        shared_state["index_name"] = index.resource_name

    

        # Verify that the retrieved index is the same

        get_index = aiplatform.MatchingEngineIndex(index_name=index.resource_name)

        assert index.resource_name == get_index.resource_name

    

        # Create index and check that it is listed

        list_indexes = aiplatform.MatchingEngineIndex.list()

        assert get_index.resource_name in [

            index.resource_name for index in list_indexes

        ]

    

        # Update the index metadata

        updated_index = get_index.update_metadata(

            display_name=_TEST_DISPLAY_NAME_UPDATE,

            description=_TEST_DESCRIPTION_UPDATE,

            labels=_TEST_LABELS_UPDATE,

        )

    

        assert updated_index.name == get_index.name

        # TODO: Reinstate assertions once b/220005272 is fixed.

        # assert updated_index.display_name == _TEST_DISPLAY_NAME_UPDATE

        # assert updated_index.description == _TEST_DESCRIPTION_UPDATE

        # assert updated_index.labels == _TEST_LABELS_UPDATE

    

        # Update the index embeddings

        updated_index = get_index.update_embeddings(

            contents_delta_uri=_TEST_CONTENTS_DELTA_URI_UPDATE,

            is_complete_overwrite=_TEST_IS_COMPLETE_OVERWRITE,

        )

    

        assert updated_index.name == get_index.name

    

        # Create endpoint and check that it is listed

        my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(

            display_name=_TEST_INDEX_ENDPOINT_DISPLAY_NAME,

            description=_TEST_INDEX_ENDPOINT_DESCRIPTION,

            network=e2e_base._VPC_NETWORK_URI,

            labels=_TEST_LABELS,

        )

        assert my_index_endpoint.resource_name in [

            index_endpoint.resource_name

            for index_endpoint in aiplatform.MatchingEngineIndexEndpoint.list()

        ]

    

        assert my_index_endpoint.labels == _TEST_LABELS

        assert my_index_endpoint.display_name == _TEST_INDEX_ENDPOINT_DISPLAY_NAME

        assert my_index_endpoint.description == _TEST_INDEX_ENDPOINT_DESCRIPTION

    

        # Create endpoint and check that it is listed

>       public_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(

            display_name=_TEST_PUBLIC_INDEX_ENDPOINT_DISPLAY_NAME,

            description=_TEST_PUBLIC_INDEX_ENDPOINT_DESCRIPTION,

            public_endpoint_enabled=True,

            labels=_TEST_LABELS,

        )

tests/system/aiplatform/test_matching_engine_index.py:249: 

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:227: in create

    return cls._create(

google/cloud/aiplatform/base.py:819: in wrapper

    return method(*args, **kwargs)

google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:329: in _create

    create_lro = api_client.create_index_endpoint(

google/cloud/aiplatform_v1/services/index_endpoint_service/client.py:594: in create_index_endpoint

    response = rpc(

.nox/system-3-11/lib/python3.11/site-packages/google/api_core/gapic_v1/method.py:113: in __call__

    return wrapped_func(*args, **kwargs)

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"

index_endpoint {

  display_name: "public_endpoint_name"

 ...ption: "my public endpoint"

  labels {

    key: "my_key"

    value: "my_value"

  }

  public_endpoint_enabled: true

}

,)

kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-client', 'model-builder/1.32.0 gl-python/3.11.1 grpc/1.58.0 gax/2.11.1 gapic/1.32.0')]}

    @functools.wraps(callable_)

    def error_remapped_callable(*args, **kwargs):

        try:

            return callable_(*args, **kwargs)

        except grpc.RpcError as exc:

>           raise exceptions.from_grpc_error(exc) from exc

E           google.api_core.exceptions.FailedPrecondition: 400 Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15.

.nox/system-3-11/lib/python3.11/site-packages/google/api_core/grpc_helpers.py:74: FailedPrecondition