Tag legacy extractors and evaluators with legacy_ in preparation for deprecation. Renamed V2 MetricsAndPlotsEvaluator to MetricsPlotsAndValidationsEvaluator for consistency with recent updates to writer and to allow export without collision with legacy name.

PiperOrigin-RevId: 340565141

Author: mdreves

RELEASE.md

@@ -52,6 +52,11 @@
 
 ## Deprecations
 
+*   `tfma.extractors.FeatureExtractor`, `tfma.extractors.PredictExtractor`,
+    `tfma.extractors.InputExtractor`, and
+    `tfma.evaluators.MetricsAndPlotsEvaluator` are deprecated and may be
+    replaced with newer versions in upcoming releases.
+
 # Version 0.24.3
 
 ## Major Features and Improvements

tensorflow_model_analysis/addons/fairness/post_export_metrics/fairness_indicators_test.py

@@ -32,7 +32,7 @@
 from tensorflow_model_analysis.eval_saved_model import testutil
 from tensorflow_model_analysis.eval_saved_model.example_trainers import fixed_prediction_estimator_extra_fields
 from tensorflow_model_analysis.eval_saved_model.example_trainers import multi_head
-from tensorflow_model_analysis.evaluators import metrics_and_plots_evaluator
+from tensorflow_model_analysis.evaluators import legacy_metrics_and_plots_evaluator
 from tensorflow_model_analysis.post_export_metrics import post_export_metrics
 import tensorflow_model_analysis.post_export_metrics.metric_keys as metric_keys
 from tensorflow_model_analysis.proto import metrics_for_slice_pb2
@@ -79,7 +79,7 @@ def _runTestWithCustomCheck(self,
           | 'InputsToExtracts' >> model_eval_lib.InputsToExtracts()
           | 'Extract' >> tfma_unit.Extract(extractors=extractors)  # pylint: disable=no-value-for-parameter
           | 'ComputeMetricsAndPlots' >>
-          metrics_and_plots_evaluator.ComputeMetricsAndPlots(
+          legacy_metrics_and_plots_evaluator.ComputeMetricsAndPlots(
               eval_shared_model=eval_shared_model,
               compute_confidence_intervals=self.compute_confidence_intervals,
               random_seed_for_testing=self.deterministic_test_seed))

tensorflow_model_analysis/api/model_eval_lib.py

@@ -37,16 +37,16 @@
 from tensorflow_model_analysis import types
 from tensorflow_model_analysis.eval_saved_model import constants as eval_constants
 from tensorflow_model_analysis.evaluators import evaluator
-from tensorflow_model_analysis.evaluators import metrics_and_plots_evaluator
-from tensorflow_model_analysis.evaluators import metrics_and_plots_evaluator_v2
+from tensorflow_model_analysis.evaluators import legacy_metrics_and_plots_evaluator
+from tensorflow_model_analysis.evaluators import metrics_plots_and_validations_evaluator
 from tensorflow_model_analysis.extractors import batched_input_extractor
 from tensorflow_model_analysis.extractors import batched_predict_extractor_v2
 from tensorflow_model_analysis.extractors import extractor
-from tensorflow_model_analysis.extractors import input_extractor
-from tensorflow_model_analysis.extractors import predict_extractor
+from tensorflow_model_analysis.extractors import legacy_input_extractor
+from tensorflow_model_analysis.extractors import legacy_predict_extractor
+from tensorflow_model_analysis.extractors import legacy_tfjs_predict_extractor
+from tensorflow_model_analysis.extractors import legacy_tflite_predict_extractor
 from tensorflow_model_analysis.extractors import slice_key_extractor
-from tensorflow_model_analysis.extractors import tfjs_predict_extractor
-from tensorflow_model_analysis.extractors import tflite_predict_extractor
 from tensorflow_model_analysis.extractors import unbatch_extractor
 from tensorflow_model_analysis.post_export_metrics import post_export_metrics
 from tensorflow_model_analysis.proto import metrics_for_slice_pb2
@@ -445,7 +445,7 @@ def default_extractors(  # pylint: disable=invalid-name
       eval_config = config.EvalConfig(
           slicing_specs=[s.to_proto() for s in slice_spec])
     return [
-        custom_predict_extractor or predict_extractor.PredictExtractor(
+        custom_predict_extractor or legacy_predict_extractor.PredictExtractor(
             eval_shared_model, materialize=materialize),
         slice_key_extractor.SliceKeyExtractor(
             eval_config=eval_config, materialize=materialize)
@@ -465,9 +465,9 @@ def default_extractors(  # pylint: disable=invalid-name
       # TODO(b/163889779): Convert TFLite extractor to operate on batched
       # extracts. Then we can remove the input extractor.
       return [
-          input_extractor.InputExtractor(eval_config=eval_config),
+          legacy_input_extractor.InputExtractor(eval_config=eval_config),
           (custom_predict_extractor or
-           tflite_predict_extractor.TFLitePredictExtractor(
+           legacy_tflite_predict_extractor.TFLitePredictExtractor(
                eval_config=eval_config, eval_shared_model=eval_shared_model)),
           slice_key_extractor.SliceKeyExtractor(
               eval_config=eval_config, materialize=materialize)
@@ -479,9 +479,9 @@ def default_extractors(  # pylint: disable=invalid-name
 
     if model_types == set([constants.TF_JS]):
       return [
-          input_extractor.InputExtractor(eval_config=eval_config),
+          legacy_input_extractor.InputExtractor(eval_config=eval_config),
           (custom_predict_extractor or
-           tfjs_predict_extractor.TFJSPredictExtractor(
+           legacy_tfjs_predict_extractor.TFJSPredictExtractor(
                eval_config=eval_config, eval_shared_model=eval_shared_model)),
           slice_key_extractor.SliceKeyExtractor(
               eval_config=eval_config, materialize=materialize)
@@ -495,7 +495,7 @@ def default_extractors(  # pylint: disable=invalid-name
           all(eval_constants.EVAL_TAG in m.model_loader.tags
               for m in eval_shared_models)):
       return [
-          custom_predict_extractor or predict_extractor.PredictExtractor(
+          custom_predict_extractor or legacy_predict_extractor.PredictExtractor(
               eval_shared_model,
               materialize=materialize,
               eval_config=eval_config),
@@ -590,7 +590,7 @@ def default_evaluators(  # pylint: disable=invalid-name
       if eval_config.options.HasField('min_slice_size'):
         min_slice_size = eval_config.options.min_slice_size.value
     return [
-        metrics_and_plots_evaluator.MetricsAndPlotsEvaluator(
+        legacy_metrics_and_plots_evaluator.MetricsAndPlotsEvaluator(
             eval_shared_model,
             compute_confidence_intervals=compute_confidence_intervals,
             min_slice_size=min_slice_size,
@@ -599,7 +599,8 @@ def default_evaluators(  # pylint: disable=invalid-name
     ]
   else:
     return [
-        metrics_and_plots_evaluator_v2.MetricsAndPlotsEvaluator(
+        metrics_plots_and_validations_evaluator
+        .MetricsPlotsAndValidationsEvaluator(
             eval_config=eval_config,
             eval_shared_model=eval_shared_model,
             schema=schema,

tensorflow_model_analysis/api/model_eval_lib_test.py

@@ -35,13 +35,13 @@
 from tensorflow_model_analysis.eval_saved_model.example_trainers import fixed_prediction_estimator
 from tensorflow_model_analysis.eval_saved_model.example_trainers import linear_classifier
 from tensorflow_model_analysis.eval_saved_model.example_trainers import linear_regressor
-from tensorflow_model_analysis.evaluators import metrics_and_plots_evaluator
-from tensorflow_model_analysis.evaluators import metrics_and_plots_evaluator_v2
-from tensorflow_model_analysis.evaluators import query_based_metrics_evaluator
+from tensorflow_model_analysis.evaluators import legacy_metrics_and_plots_evaluator
+from tensorflow_model_analysis.evaluators import legacy_query_based_metrics_evaluator
+from tensorflow_model_analysis.evaluators import metrics_plots_and_validations_evaluator
 from tensorflow_model_analysis.evaluators.query_metrics import ndcg as legacy_ndcg
 from tensorflow_model_analysis.evaluators.query_metrics import query_statistics
-from tensorflow_model_analysis.extractors import feature_extractor
-from tensorflow_model_analysis.extractors import predict_extractor
+from tensorflow_model_analysis.extractors import legacy_feature_extractor
+from tensorflow_model_analysis.extractors import legacy_predict_extractor
 from tensorflow_model_analysis.extractors import slice_key_extractor
 from tensorflow_model_analysis.metrics import calibration_plot
 from tensorflow_model_analysis.metrics import metric_specs
@@ -230,9 +230,9 @@ def testRunModelAnalysisExtraFieldsPlusFeatureExtraction(self):
     eval_shared_model = model_eval_lib.default_eval_shared_model(
         eval_saved_model_path=model_location, example_weight_key='age')
     extractors_with_feature_extraction = [
-        predict_extractor.PredictExtractor(
+        legacy_predict_extractor.PredictExtractor(
             eval_shared_model, desired_batch_size=3, materialize=False),
-        feature_extractor.FeatureExtractor(
+        legacy_feature_extractor.FeatureExtractor(
             extract_source=constants.INPUT_KEY,
             extract_dest=constants.FEATURES_PREDICTIONS_LABELS_KEY),
         slice_key_extractor.SliceKeyExtractor(
@@ -392,7 +392,7 @@ def testRunModelAnalysisWithCustomizations(self):
     extractors = model_eval_lib.default_extractors(
         eval_shared_model=eval_shared_model,
         eval_config=eval_config,
-        custom_predict_extractor=predict_extractor.PredictExtractor(
+        custom_predict_extractor=legacy_predict_extractor.PredictExtractor(
             eval_shared_model=eval_shared_model, eval_config=eval_config))
     eval_result = model_eval_lib.run_model_analysis(
         eval_config=eval_config,
@@ -874,7 +874,8 @@ def testRunModelAnalysisWithQueryBasedMetrics(self):
         data_location=data_location,
         output_path=output_path,
         evaluators=[
-            metrics_and_plots_evaluator_v2.MetricsAndPlotsEvaluator(
+            metrics_plots_and_validations_evaluator
+            .MetricsPlotsAndValidationsEvaluator(
                 eval_config=eval_config, eval_shared_model=eval_shared_model)
         ],
         schema=schema)
@@ -933,9 +934,9 @@ def testRunModelAnalysisWithLegacyQueryExtractor(self):
         data_location=data_location,
         output_path=self._getTempDir(),
         evaluators=[
-            metrics_and_plots_evaluator.MetricsAndPlotsEvaluator(
+            legacy_metrics_and_plots_evaluator.MetricsAndPlotsEvaluator(
                 eval_shared_model),
-            query_based_metrics_evaluator.QueryBasedMetricsEvaluator(
+            legacy_query_based_metrics_evaluator.QueryBasedMetricsEvaluator(
                 query_id='language',
                 prediction_key='logistic',
                 combine_fns=[

tensorflow_model_analysis/api/tfma_unit.py

@@ -71,7 +71,7 @@ def testWithBeam(self):
 from tensorflow_model_analysis.api import model_eval_lib
 from tensorflow_model_analysis.eval_saved_model import load
 from tensorflow_model_analysis.eval_saved_model import testutil
-from tensorflow_model_analysis.evaluators import metrics_and_plots_evaluator
+from tensorflow_model_analysis.evaluators import legacy_metrics_and_plots_evaluator
 from tensorflow_model_analysis.extractors import extractor
 from tensorflow_model_analysis.slicer import slicer_lib as slicer
 
@@ -287,7 +287,7 @@ def check_metrics(got):
           | 'CreateExamples' >> beam.Create(serialized_examples)
           | 'InputsToExtracts' >> model_eval_lib.InputsToExtracts()
           | 'Extract' >> Extract(extractors=extractors)
-          | 'ComputeMetricsAndPlots' >> metrics_and_plots_evaluator
+          | 'ComputeMetricsAndPlots' >> legacy_metrics_and_plots_evaluator
           .ComputeMetricsAndPlots(eval_shared_model=eval_shared_model))
       # pylint: enable=no-value-for-parameter
 
@@ -378,7 +378,7 @@ def check_metrics(got):
         examples_pcollection
         | 'InputsToExtracts' >> model_eval_lib.InputsToExtracts()
         | 'Extract' >> Extract(extractors=extractors)
-        | 'ComputeMetricsAndPlots' >> metrics_and_plots_evaluator
+        | 'ComputeMetricsAndPlots' >> legacy_metrics_and_plots_evaluator
         .ComputeMetricsAndPlots(eval_shared_model=eval_shared_model))
     # pylint: enable=no-value-for-parameter
 

tensorflow_model_analysis/contrib/model_eval_lib.py

@@ -26,8 +26,8 @@
 from tensorflow_model_analysis.evaluators import analysis_table_evaluator
 from tensorflow_model_analysis.evaluators import evaluator
 from tensorflow_model_analysis.extractors import extractor
-from tensorflow_model_analysis.extractors import feature_extractor
-from tensorflow_model_analysis.extractors import predict_extractor
+from tensorflow_model_analysis.extractors import legacy_feature_extractor
+from tensorflow_model_analysis.extractors import legacy_predict_extractor
 from tensorflow_model_analysis.extractors import slice_key_extractor
 from tensorflow_model_analysis.slicer import slicer_lib as slicer
 
@@ -70,9 +70,9 @@ def BuildAnalysisTable(  # pylint: disable=invalid-name
 
   if not extractors:
     extractors = [
-        predict_extractor.PredictExtractor(eval_shared_model,
-                                           desired_batch_size),
-        feature_extractor.FeatureExtractor(),
+        legacy_predict_extractor.PredictExtractor(eval_shared_model,
+                                                  desired_batch_size),
+        legacy_feature_extractor.FeatureExtractor(),
         slice_key_extractor.SliceKeyExtractor(slice_spec)
     ]
   if not evaluators:

tensorflow_model_analysis/evaluators/__init__.py

@@ -18,4 +18,5 @@
 from tensorflow_model_analysis.evaluators.evaluator import Evaluation
 from tensorflow_model_analysis.evaluators.evaluator import Evaluator
 from tensorflow_model_analysis.evaluators.evaluator import verify_evaluator
-from tensorflow_model_analysis.evaluators.metrics_and_plots_evaluator import MetricsAndPlotsEvaluator
+from tensorflow_model_analysis.evaluators.legacy_metrics_and_plots_evaluator import MetricsAndPlotsEvaluator
+from tensorflow_model_analysis.evaluators.metrics_plots_and_validations_evaluator import MetricsPlotsAndValidationsEvaluator

tensorflow_model_analysis/evaluators/aggregate.py renamed to tensorflow_model_analysis/evaluators/legacy_aggregate.py

@@ -25,7 +25,7 @@
 from tensorflow_model_analysis import constants
 from tensorflow_model_analysis.eval_saved_model import testutil
 from tensorflow_model_analysis.eval_saved_model.example_trainers import linear_classifier
-from tensorflow_model_analysis.evaluators import aggregate
+from tensorflow_model_analysis.evaluators import legacy_aggregate as aggregate
 from tensorflow_model_analysis.evaluators import poisson_bootstrap
 
 

tensorflow_model_analysis/evaluators/aggregate_test.py renamed to tensorflow_model_analysis/evaluators/legacy_aggregate_test.py

@@ -24,9 +24,9 @@
 import apache_beam as beam
 from tensorflow_model_analysis import constants
 from tensorflow_model_analysis import types
-from tensorflow_model_analysis.evaluators import aggregate
 from tensorflow_model_analysis.evaluators import counter_util
 from tensorflow_model_analysis.evaluators import evaluator
+from tensorflow_model_analysis.evaluators import legacy_aggregate
 from tensorflow_model_analysis.evaluators import poisson_bootstrap
 from tensorflow_model_analysis.extractors import extractor
 from tensorflow_model_analysis.extractors import slice_key_extractor
@@ -155,7 +155,7 @@ def ComputeMetricsAndPlots(  # pylint: disable=invalid-name
       # plots if applicable.
       | 'ComputePerSliceMetrics' >>
       poisson_bootstrap.ComputeWithConfidenceIntervals(
-          aggregate.ComputePerSliceMetrics,
+          legacy_aggregate.ComputePerSliceMetrics,
           num_bootstrap_samples=(poisson_bootstrap.DEFAULT_NUM_BOOTSTRAP_SAMPLES
                                  if compute_confidence_intervals else 1),
           random_seed_for_testing=random_seed_for_testing,

tensorflow_model_analysis/evaluators/metrics_and_plots_evaluator.py renamed to tensorflow_model_analysis/evaluators/legacy_metrics_and_plots_evaluator.py

@@ -30,8 +30,8 @@
 from tensorflow_model_analysis.eval_saved_model.example_trainers import fixed_prediction_estimator
 from tensorflow_model_analysis.eval_saved_model.example_trainers import fixed_prediction_estimator_no_labels
 from tensorflow_model_analysis.eval_saved_model.example_trainers import linear_classifier
-from tensorflow_model_analysis.evaluators import metrics_and_plots_evaluator
-from tensorflow_model_analysis.extractors import predict_extractor
+from tensorflow_model_analysis.evaluators import legacy_metrics_and_plots_evaluator as metrics_and_plots_evaluator
+from tensorflow_model_analysis.extractors import legacy_predict_extractor
 from tensorflow_model_analysis.extractors import slice_key_extractor
 from tensorflow_model_analysis.post_export_metrics import metric_keys
 from tensorflow_model_analysis.post_export_metrics import metrics as metric_fns
@@ -94,7 +94,7 @@ def testEvaluateNoSlicing(self):
         eval_saved_model_path=eval_export_dir,
         add_metrics_callbacks=[_addExampleCountMetricCallback])
     extractors = [
-        predict_extractor.PredictExtractor(eval_shared_model),
+        legacy_predict_extractor.PredictExtractor(eval_shared_model),
         slice_key_extractor.SliceKeyExtractor()
     ]
 
@@ -143,7 +143,7 @@ def testEvaluateWithSlicingAndDifferentBatchSizes(self):
         eval_saved_model_path=eval_export_dir,
         add_metrics_callbacks=[_addExampleCountMetricCallback])
     extractors = [
-        predict_extractor.PredictExtractor(eval_shared_model),
+        legacy_predict_extractor.PredictExtractor(eval_shared_model),
         slice_key_extractor.SliceKeyExtractor([
             slicer.SingleSliceSpec(),
             slicer.SingleSliceSpec(columns=['slice_key'])
@@ -233,7 +233,7 @@ def testEvaluateWithSlicingAndUncertainty(self):
         eval_saved_model_path=eval_export_dir,
         add_metrics_callbacks=[_addExampleCountMetricCallback])
     extractors = [
-        predict_extractor.PredictExtractor(eval_shared_model),
+        legacy_predict_extractor.PredictExtractor(eval_shared_model),
         slice_key_extractor.SliceKeyExtractor([
             slicer.SingleSliceSpec(),
             slicer.SingleSliceSpec(columns=['slice_key'])
@@ -331,7 +331,7 @@ def testEvaluateNoSlicingAddPostExportAndCustomMetrics(self):
             post_export_metrics.example_weight(example_weight_key='age')
         ])
     extractors = [
-        predict_extractor.PredictExtractor(eval_shared_model),
+        legacy_predict_extractor.PredictExtractor(eval_shared_model),
         slice_key_extractor.SliceKeyExtractor()
     ]
 
@@ -392,7 +392,7 @@ def testEvaluateNoSlicingAddPostExportAndCustomMetricsUnsupervisedModel(self):
             post_export_metrics.example_weight(example_weight_key='prediction')
         ])
     extractors = [
-        predict_extractor.PredictExtractor(eval_shared_model),
+        legacy_predict_extractor.PredictExtractor(eval_shared_model),
         slice_key_extractor.SliceKeyExtractor()
     ]
 
@@ -441,7 +441,7 @@ def testEvaluateWithPlots(self):
             post_export_metrics.auc_plots()
         ])
     extractors = [
-        predict_extractor.PredictExtractor(eval_shared_model),
+        legacy_predict_extractor.PredictExtractor(eval_shared_model),
         slice_key_extractor.SliceKeyExtractor()
     ]
 

tensorflow_model_analysis/evaluators/metrics_and_plots_evaluator_test.py renamed to tensorflow_model_analysis/evaluators/legacy_metrics_and_plots_evaluator_test.py

@@ -28,11 +28,11 @@
 from tensorflow_model_analysis.api import tfma_unit
 from tensorflow_model_analysis.eval_saved_model import testutil
 from tensorflow_model_analysis.eval_saved_model.example_trainers import fixed_prediction_estimator_extra_fields
-from tensorflow_model_analysis.evaluators import query_based_metrics_evaluator
+from tensorflow_model_analysis.evaluators import legacy_query_based_metrics_evaluator as query_based_metrics_evaluator
 from tensorflow_model_analysis.evaluators.query_metrics import min_label_position
 from tensorflow_model_analysis.evaluators.query_metrics import ndcg
 from tensorflow_model_analysis.evaluators.query_metrics import query_statistics
-from tensorflow_model_analysis.extractors import predict_extractor
+from tensorflow_model_analysis.extractors import legacy_predict_extractor
 from tensorflow_model_analysis.extractors import slice_key_extractor
 
 
@@ -122,7 +122,7 @@ def testEvaluateQueryBasedMetrics(self):
     eval_shared_model = self.createTestEvalSharedModel(
         eval_saved_model_path=eval_export_dir)
     extractors = [
-        predict_extractor.PredictExtractor(eval_shared_model),
+        legacy_predict_extractor.PredictExtractor(eval_shared_model),
         slice_key_extractor.SliceKeyExtractor()
     ]