Skip to content

Commit cf1981e

Browse files
jsondaicopybara-github
jsondai
authored and
copybara-github
committed
chore: clean up legacy unit tests
PiperOrigin-RevId: 934647748
1 parent 585545f commit cf1981e

1 file changed

Lines changed: 0 additions & 311 deletions

File tree

‎tests/unit/aiplatform/test_language_models.py‎

Lines changed: 0 additions & 311 deletions
Original file line numberDiff line numberDiff line change
@@ -4883,317 +4883,6 @@ def teardown_method(self):
48834883
self._job_wait_patcher.stop()
48844884
self._log_wait_patcher.stop()
48854885

4886-
@pytest.mark.usefixtures(
4887-
"get_model_with_tuned_version_label_mock",
4888-
"get_endpoint_with_models_mock",
4889-
)
4890-
@pytest.mark.parametrize(
4891-
"job_spec",
4892-
[_TEST_EVAL_PIPELINE_SPEC_JSON, _TEST_EVAL_PIPELINE_JOB],
4893-
)
4894-
@pytest.mark.parametrize(
4895-
"mock_request_urlopen_eval",
4896-
["https://us-kfp.pkg.dev/proj/repo/pack/latest"],
4897-
indirect=True,
4898-
)
4899-
def test_model_evaluation_text_generation_task_with_gcs_input(
4900-
self,
4901-
job_spec,
4902-
mock_pipeline_service_create_eval,
4903-
mock_pipeline_job_get_eval,
4904-
mock_successfully_completed_eval_job,
4905-
mock_pipeline_bucket_exists,
4906-
mock_load_yaml_and_json,
4907-
mock_request_urlopen_eval,
4908-
):
4909-
aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
4910-
4911-
with mock.patch.object(
4912-
target=model_garden_service_client.ModelGardenServiceClient,
4913-
attribute="get_publisher_model",
4914-
return_value=gca_publisher_model.PublisherModel(
4915-
_TEXT_BISON_PUBLISHER_MODEL_DICT
4916-
),
4917-
):
4918-
my_model = preview_language_models.TextGenerationModel.get_tuned_model(
4919-
test_constants.ModelConstants._TEST_MODEL_RESOURCE_NAME
4920-
)
4921-
4922-
eval_metrics = my_model.evaluate(
4923-
task_spec=preview_language_models.EvaluationTextGenerationSpec(
4924-
ground_truth_data="gs://my-bucket/ground-truth.jsonl",
4925-
),
4926-
)
4927-
4928-
assert isinstance(eval_metrics, preview_language_models.EvaluationMetric)
4929-
assert eval_metrics.bleu == _TEST_TEXT_GENERATION_METRICS["bleu"]
4930-
4931-
@pytest.mark.usefixtures(
4932-
"get_model_with_tuned_version_label_mock",
4933-
"get_endpoint_with_models_mock",
4934-
)
4935-
@pytest.mark.parametrize(
4936-
"job_spec",
4937-
[_TEST_EVAL_PIPELINE_SPEC_JSON, _TEST_EVAL_PIPELINE_JOB],
4938-
)
4939-
def test_populate_eval_template_params(
4940-
self,
4941-
job_spec,
4942-
mock_pipeline_service_create,
4943-
mock_model_evaluate,
4944-
mock_pipeline_job_get,
4945-
mock_successfully_completed_eval_job,
4946-
mock_pipeline_bucket_exists,
4947-
mock_load_yaml_and_json,
4948-
):
4949-
aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
4950-
4951-
with mock.patch.object(
4952-
target=model_garden_service_client.ModelGardenServiceClient,
4953-
attribute="get_publisher_model",
4954-
return_value=gca_publisher_model.PublisherModel(
4955-
_TEXT_BISON_PUBLISHER_MODEL_DICT
4956-
),
4957-
):
4958-
my_model = preview_language_models.TextGenerationModel.get_tuned_model(
4959-
test_constants.ModelConstants._TEST_MODEL_RESOURCE_NAME
4960-
)
4961-
4962-
task_spec = preview_language_models.EvaluationTextGenerationSpec(
4963-
ground_truth_data="gs://my-bucket/ground-truth.jsonl",
4964-
)
4965-
4966-
formatted_template_params = (
4967-
_evaluatable_language_models._populate_eval_template_params(
4968-
task_spec=task_spec, model_name=my_model._model_resource_name
4969-
)
4970-
)
4971-
4972-
assert (
4973-
"batch_predict_gcs_destination_output_uri" in formatted_template_params
4974-
)
4975-
assert "model_name" in formatted_template_params
4976-
assert "evaluation_task" in formatted_template_params
4977-
4978-
# This should only be in the classification task pipeline template
4979-
assert "evaluation_class_labels" not in formatted_template_params
4980-
assert "target_column_name" not in formatted_template_params
4981-
4982-
@pytest.mark.usefixtures(
4983-
"get_model_with_tuned_version_label_mock",
4984-
"get_endpoint_with_models_mock",
4985-
)
4986-
@pytest.mark.parametrize(
4987-
"job_spec",
4988-
[_TEST_EVAL_PIPELINE_SPEC_JSON, _TEST_EVAL_PIPELINE_JOB],
4989-
)
4990-
def test_populate_template_params_for_classification_task(
4991-
self,
4992-
job_spec,
4993-
mock_pipeline_service_create,
4994-
mock_model_evaluate,
4995-
mock_pipeline_job_get,
4996-
mock_successfully_completed_eval_job,
4997-
mock_pipeline_bucket_exists,
4998-
mock_load_yaml_and_json,
4999-
):
5000-
aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
5001-
5002-
with mock.patch.object(
5003-
target=model_garden_service_client.ModelGardenServiceClient,
5004-
attribute="get_publisher_model",
5005-
return_value=gca_publisher_model.PublisherModel(
5006-
_TEXT_BISON_PUBLISHER_MODEL_DICT
5007-
),
5008-
):
5009-
my_model = preview_language_models.TextGenerationModel.get_tuned_model(
5010-
test_constants.ModelConstants._TEST_MODEL_RESOURCE_NAME
5011-
)
5012-
5013-
task_spec = preview_language_models.EvaluationTextClassificationSpec(
5014-
ground_truth_data="gs://my-bucket/ground-truth.jsonl",
5015-
target_column_name="test_targ_name",
5016-
class_names=["test_class_name_1", "test_class_name_2"],
5017-
)
5018-
5019-
formatted_template_params = (
5020-
_evaluatable_language_models._populate_eval_template_params(
5021-
task_spec=task_spec, model_name=my_model._model_resource_name
5022-
)
5023-
)
5024-
5025-
assert "evaluation_class_labels" in formatted_template_params
5026-
assert "target_field_name" in formatted_template_params
5027-
5028-
@pytest.mark.usefixtures(
5029-
"get_model_with_tuned_version_label_mock",
5030-
"get_endpoint_with_models_mock",
5031-
"mock_storage_blob_upload_from_filename",
5032-
)
5033-
@pytest.mark.parametrize(
5034-
"job_spec",
5035-
[_TEST_EVAL_PIPELINE_SPEC_JSON, _TEST_EVAL_PIPELINE_JOB],
5036-
)
5037-
def test_populate_template_params_with_dataframe_input(
5038-
self,
5039-
job_spec,
5040-
mock_pipeline_service_create,
5041-
mock_pipeline_job_get,
5042-
mock_successfully_completed_eval_job,
5043-
mock_pipeline_bucket_exists,
5044-
mock_load_yaml_and_json,
5045-
):
5046-
aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
5047-
5048-
with mock.patch.object(
5049-
target=model_garden_service_client.ModelGardenServiceClient,
5050-
attribute="get_publisher_model",
5051-
return_value=gca_publisher_model.PublisherModel(
5052-
_TEXT_BISON_PUBLISHER_MODEL_DICT
5053-
),
5054-
):
5055-
my_model = preview_language_models.TextGenerationModel.get_tuned_model(
5056-
test_constants.ModelConstants._TEST_MODEL_RESOURCE_NAME
5057-
)
5058-
5059-
task_spec = preview_language_models.EvaluationTextGenerationSpec(
5060-
ground_truth_data=_TEST_EVAL_DATA_DF,
5061-
)
5062-
5063-
formatted_template_params = (
5064-
_evaluatable_language_models._populate_eval_template_params(
5065-
task_spec=task_spec, model_name=my_model._model_resource_name
5066-
)
5067-
)
5068-
5069-
# The utility method should not modify task_spec
5070-
assert isinstance(task_spec.ground_truth_data, pd.DataFrame)
5071-
5072-
assert (
5073-
"batch_predict_gcs_destination_output_uri" in formatted_template_params
5074-
)
5075-
assert "model_name" in formatted_template_params
5076-
assert "evaluation_task" in formatted_template_params
5077-
5078-
# This should only be in the classification task pipeline template
5079-
assert "evaluation_class_labels" not in formatted_template_params
5080-
assert "target_column_name" not in formatted_template_params
5081-
5082-
def test_evaluate_raises_on_ga_language_model(
5083-
self,
5084-
):
5085-
aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
5086-
5087-
with mock.patch.object(
5088-
target=model_garden_service_client.ModelGardenServiceClient,
5089-
attribute="get_publisher_model",
5090-
return_value=gca_publisher_model.PublisherModel(
5091-
_TEXT_BISON_PUBLISHER_MODEL_DICT
5092-
),
5093-
):
5094-
model = language_models.TextGenerationModel.from_pretrained(
5095-
"text-bison@001"
5096-
)
5097-
5098-
with pytest.raises(AttributeError):
5099-
model.evaluate()
5100-
5101-
@pytest.mark.usefixtures(
5102-
"get_endpoint_with_models_mock",
5103-
)
5104-
@pytest.mark.parametrize(
5105-
"job_spec",
5106-
[_TEST_EVAL_PIPELINE_SPEC_JSON, _TEST_EVAL_PIPELINE_JOB],
5107-
)
5108-
@pytest.mark.parametrize(
5109-
"mock_request_urlopen_eval",
5110-
["https://us-kfp.pkg.dev/proj/repo/pack/latest"],
5111-
indirect=True,
5112-
)
5113-
def test_model_evaluation_text_generation_task_on_base_model(
5114-
self,
5115-
job_spec,
5116-
mock_pipeline_service_create_eval,
5117-
mock_pipeline_job_get_eval,
5118-
mock_successfully_completed_eval_job,
5119-
mock_pipeline_bucket_exists,
5120-
mock_load_yaml_and_json,
5121-
mock_request_urlopen_eval,
5122-
):
5123-
aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
5124-
5125-
with mock.patch.object(
5126-
target=model_garden_service_client.ModelGardenServiceClient,
5127-
attribute="get_publisher_model",
5128-
return_value=gca_publisher_model.PublisherModel(
5129-
_TEXT_BISON_PUBLISHER_MODEL_DICT
5130-
),
5131-
):
5132-
my_model = preview_language_models.TextGenerationModel.from_pretrained(
5133-
"text-bison@001"
5134-
)
5135-
5136-
eval_metrics = my_model.evaluate(
5137-
task_spec=preview_language_models.EvaluationTextGenerationSpec(
5138-
ground_truth_data="gs://my-bucket/ground-truth.jsonl",
5139-
),
5140-
)
5141-
5142-
assert isinstance(eval_metrics, preview_language_models.EvaluationMetric)
5143-
5144-
@pytest.mark.usefixtures(
5145-
"get_endpoint_with_models_mock",
5146-
)
5147-
@pytest.mark.parametrize(
5148-
"job_spec",
5149-
[
5150-
_TEST_EVAL_CLASSIFICATION_PIPELINE_SPEC_JSON,
5151-
_TEST_EVAL_CLASSIFICATION_PIPELINE_JOB,
5152-
],
5153-
)
5154-
@pytest.mark.parametrize(
5155-
"mock_request_urlopen_eval_classification",
5156-
["https://us-central1-kfp.pkg.dev/proj/repo/pack/latest"],
5157-
indirect=True,
5158-
)
5159-
def test_model_evaluation_text_classification_base_model_only_summary_metrics(
5160-
self,
5161-
job_spec,
5162-
mock_pipeline_service_create_eval_classification,
5163-
mock_pipeline_job_get_eval_classification,
5164-
mock_successfully_completed_eval_classification_job,
5165-
mock_pipeline_bucket_exists,
5166-
mock_load_yaml_and_json,
5167-
mock_request_urlopen_eval_classification,
5168-
):
5169-
aiplatform.init(project=_TEST_PROJECT, location=_TEST_LOCATION)
5170-
5171-
with mock.patch.object(
5172-
target=model_garden_service_client.ModelGardenServiceClient,
5173-
attribute="get_publisher_model",
5174-
return_value=gca_publisher_model.PublisherModel(
5175-
_TEXT_BISON_PUBLISHER_MODEL_DICT
5176-
),
5177-
):
5178-
my_model = preview_language_models.TextGenerationModel.from_pretrained(
5179-
"text-bison@001"
5180-
)
5181-
5182-
eval_metrics = my_model.evaluate(
5183-
task_spec=preview_language_models.EvaluationTextClassificationSpec(
5184-
ground_truth_data="gs://my-bucket/ground-truth.jsonl",
5185-
target_column_name="test_targ_name",
5186-
class_names=["test_class_name_1", "test_class_name_2"],
5187-
)
5188-
)
5189-
5190-
assert isinstance(
5191-
eval_metrics,
5192-
preview_language_models.EvaluationClassificationMetric,
5193-
)
5194-
assert eval_metrics.confidenceMetrics is None
5195-
assert eval_metrics.auPrc == _TEST_TEXT_CLASSIFICATION_METRICS["auPrc"]
5196-
51974886
@pytest.mark.parametrize(
51984887
"job_spec",
51994888
[

0 commit comments

Comments
 (0)