Tensorflow:从大于2 GB的numpy数组创建minibatch

import tensorflow as tf
import numpy as np

# 假设有一个大于2 GB的NumPy数组data
data = np.random.rand(10000, 1000)  # 示例数据，10000个样本，每个样本有1000个特征
labels = np.random.randint(0, 2, size=(10000,))  # 示例标签，二分类问题，0或1

# 将数据集划分为训练集和测试集
train_data = data[:8000]
train_labels = labels[:8000]
test_data = data[8000:]
test_labels = labels[8000:]

# 使用from_tensor_slices方法将NumPy数组转换为数据集对象
train_dataset = tf.data.Dataset.from_tensor_slices((train_data, train_labels))
test_dataset = tf.data.Dataset.from_tensor_slices((test_data, test_labels))

# 对数据集进行预处理和转换，例如标准化、缩放等
train_dataset = train_dataset.map(preprocess_function)
test_dataset = test_dataset.map(preprocess_function)

# 使用batch方法将数据集划分为小批量数据
batch_size = 32
train_dataset = train_dataset.batch(batch_size)
test_dataset = test_dataset.batch(batch_size)