Internal change.
PiperOrigin-RevId: 573318330
This commit is contained in:
parent
8993073f35
commit
a1e1b5d34c
|
@ -145,7 +145,6 @@ class AverageWordEmbeddingClassifierPreprocessor:
|
||||||
"""Returns the vocab of the AverageWordEmbeddingClassifierPreprocessor."""
|
"""Returns the vocab of the AverageWordEmbeddingClassifierPreprocessor."""
|
||||||
return self._vocab
|
return self._vocab
|
||||||
|
|
||||||
# TODO: Align with MediaPipe's RegexTokenizer.
|
|
||||||
def _regex_tokenize(self, text: str) -> Sequence[str]:
|
def _regex_tokenize(self, text: str) -> Sequence[str]:
|
||||||
"""Splits `text` by words but does not split on single quotes.
|
"""Splits `text` by words but does not split on single quotes.
|
||||||
|
|
||||||
|
|
|
@ -353,7 +353,6 @@ class _AverageWordEmbeddingClassifier(TextClassifier):
|
||||||
return text_classifier_writer.MetadataWriter.create_for_regex_model(
|
return text_classifier_writer.MetadataWriter.create_for_regex_model(
|
||||||
model_buffer=tflite_model,
|
model_buffer=tflite_model,
|
||||||
regex_tokenizer=metadata_writer.RegexTokenizer(
|
regex_tokenizer=metadata_writer.RegexTokenizer(
|
||||||
# TODO: Align with MediaPipe's RegexTokenizer.
|
|
||||||
delim_regex_pattern=self._DELIM_REGEX_PATTERN,
|
delim_regex_pattern=self._DELIM_REGEX_PATTERN,
|
||||||
vocab_file_path=vocab_filepath),
|
vocab_file_path=vocab_filepath),
|
||||||
labels=metadata_writer.Labels().add(list(self._label_names)))
|
labels=metadata_writer.Labels().add(list(self._label_names)))
|
||||||
|
|
Loading…
Reference in New Issue
Block a user