tensorflow 2 TextVectorization process tensor and dataset error

Question

I would like to process text with tensorflow 2.8 on Jupyter notebook.

my code:

import re
import string
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import tensorflow_text as tf_text

def standardize(input_data):
    lowercase_str = tf.strings.lower(input_data)
    a_str = tf.strings.regex_replace(lowercase_str, f"[{re.escape(string.punctuation)}]", "")
    tokenizer = tf_text.WhitespaceTokenizer()
    tokens = tokenizer.tokenize(a_str)
    return tokens

# the input data loaded from text files by TfRecordDataset(file_paths, "GZIP")
# each file can be 200+MB, totally about 300 files
# each file hold the data with multiple columns
# some columns are text
# after loading, the dataset will be accessed by column name 
# e.g. one column is "sports", so the input_dataset["sports"] 
# return a tensor, which is like the following example
my_data_tensor = tf.constant([["SWIM 2008-07 Baseball"], ["Football"]])

tf.print(my_data_tensor)
tf.print(my_data_tensor.shape)
tf.print(f"type is {type(my_data_tensor)}")
text_layer = layers.TextVectorization(
                        standardize = standardize,
                        max_tokens = 10,
                        output_mode = 'int',
                        output_sequence_length=10
                       )

my_dataset = tf.data.Dataset.from_tensor_slices(my_data_tensor)
text_layer.adapt(my_dataset.batch(2)) # error         
processed_text = text_layer(my_dataset)

error:
 ValueError: Exception encountered when calling layer "query_tower" (type QueryTower).
 When using `TextVectorization` to tokenize strings, the input rank must be 1 or the last shape dimension must be 1. Received: inputs.shape=(2, 1, None) with rank=3

I have tried tf.unstack() and tf.reshape, tf.unbatch, but none of them work. For the given example:

[["SWIM 2008-07 Baseball"], ["Football"]]

what I need:

[["swim 200807 baseball"], ["football"]]
then
it will be encoded as int by the "text_layer"

these data (bach_size=2) will be used for a machine learning model as features.

Did I do something wrong ? thanks

Answer 1

You could try something like this:

import re
import string
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import tensorflow_text as tf_text

def standardize(input_data):
    lowercase_str = tf.strings.lower(input_data)
    a_str = tf.strings.regex_replace(lowercase_str, f"[{re.escape(string.punctuation)}]", "")
    tokenizer = tf_text.WhitespaceTokenizer()
    tokens = tokenizer.tokenize(a_str)
    return tokens

# the input data loaded from text files by TfRecordDataset(file_paths, "GZIP")
# each file can be 200+MB, totally about 300 files
# each file hold the data with multiple columns
# some columns are text
# after loading, the dataset will be accessed by column name 
# e.g. one column is "sports", so the input_dataset["sports"] 
# return a tensor, which is like the following example
my_data_tensor = tf.constant([["SWIM 2008-07 Baseball"], ["Football"]])

tf.print(my_data_tensor)
tf.print(my_data_tensor.shape)
tf.print(f"type is {type(my_data_tensor)}")
text_layer = layers.TextVectorization(
                        standardize = standardize,
                        max_tokens = 10,
                        output_mode = 'int',
                        output_sequence_length=10
                       )

my_dataset = tf.data.Dataset.from_tensor_slices(my_data_tensor)
my_dataset = tf.data.Dataset.from_tensor_slices((tf.concat(list(my_dataset.map(lambda x: x)), axis=0)))

text_layer.adapt(my_dataset) 
my_dataset = my_dataset.batch(2)    
processed_text = my_dataset.map(lambda x: text_layer(tf.squeeze(x, axis=-1)))

for p in process_text:
  print(p)

[["SWIM 2008-07 Baseball"]
 ["Football"]]
TensorShape([2, 1])
type is <class 'tensorflow.python.framework.ops.EagerTensor'>
(<tf.Tensor: shape=(2, 10), dtype=int64, numpy=
array([[2, 5, 6, 4, 0, 0, 0, 0, 0, 0],
       [3, 0, 0, 0, 0, 0, 0, 0, 0, 0]])>, <tf.Tensor: shape=(2,), dtype=int32, numpy=array([1, 0], dtype=int32)>)

Answer 2

the callable function returns the target text value. enter link description here You may adapts to use Bytes_split for target values.

[ Sample ]:

import tensorflow as tf
import tensorflow_text as tf_text

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
Functions
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
def standardize(input_data):
    input_data = tf.strings.lower(input_data)
    input_data = tf.strings.regex_replace(input_data,  "<[^>]+>", " ")
    
    return input_data

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Varibles
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
input_data = tf.constant([["SWIM 2008-07 Baseball"], ["Football"]], shape=(2, 1), dtype=tf.string)
text_layer = tf.keras.layers.TextVectorization( standardize = standardize, max_tokens = 10, output_mode = 'int', output_sequence_length=10 )

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Working
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
print("")
print("")
print("")
dataset = tf.data.Dataset.from_tensors( standardize(input_data) )
dataset = dataset.batch(2)
process_text = text_layer.adapt(dataset)
print( "standardize: " + str(standardize(input_data)) ) 
print( "process_text: " + str(process_text) )

[ Output ]:

standardize: tf.Tensor(
[[b'swim 2008-07 baseball']
 [b'football']], shape=(2, 1), dtype=string)
process_text: None

Reference