Google Cloud ML Engine + Tensorflow在input_fn()中执行预处理/标记化

pip install tensorflow

import tensorflow as tf
import numpy as np

def input_fn(data_file, batch_size, num_epochs, shuffle):
    """Input function for training and evaluation.
    
    Args:
      data_file: File path to the CSV file containing the data.
      batch_size: The number of samples per batch.
      num_epochs: The number of epochs to repeat the dataset.
      shuffle: Boolean, whether to shuffle the data.
    
    Returns:
      A tuple (features, labels) where features is a dictionary of input features,
      and labels is the target tensor.
    """
    
    # Load and preprocess the data
    def parse_csv(value):
        columns = tf.io.decode_csv(value, record_defaults=[[0]] * 3)
        features = {'text': columns[0]}
        labels = columns[1:]
        return features, labels
    
    # Read the CSV file
    dataset = tf.data.TextLineDataset(data_file)
    
    if shuffle:
        dataset = dataset.shuffle(buffer_size=10000)
    
    dataset = dataset.map(parse_csv, num_parallel_calls=tf.data.experimental.AUTOTUNE)
    
    # Tokenization and preprocessing
    tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=10000, oov_token='<OOV>')
    
    def tokenize_text(features, labels):
        text = features['text']
        text = tf.strings.lower(text)  # Convert to lowercase
        text = tf.strings.regex_replace(text, '[%s]' % re.escape(string.punctuation), '')  # Remove punctuation
        sequences = tokenizer.texts_to_sequences([text.numpy()[0]])[0]  # Tokenize
        padded = tf.keras.preprocessing.sequence.pad_sequences([sequences], maxlen=100)  # Pad sequences
        features['text'] = padded
        return features, labels
    
    dataset = dataset.map(tokenize_text, num_parallel_calls=tf.data.experimental.AUTOTUNE)
    
    dataset = dataset.padded_batch(batch_size, padded_shapes=({'text': [None]}, [None]))
    dataset = dataset.repeat(num_epochs)
    
    return dataset