mysql数据迁移到hive

# 安装必要的库
!pip install pandas mysql-connector-python pyhive

import pandas as pd
import mysql.connector
from pyhive import hive

# 连接MySQL
mysql_conn = mysql.connector.connect(host='localhost', user='user', password='password', database='database')
mysql_cursor = mysql_conn.cursor()

# 查询数据
mysql_cursor.execute("SELECT * FROM table")
data = mysql_cursor.fetchall()

# 转换为DataFrame
df = pd.DataFrame(data, columns=[i[0] for i in mysql_cursor.description])

# 连接Hive
hive_conn = hive.Connection(host='localhost', port=10000, username='user')
hive_cursor = hive_conn.cursor()

# 创建Hive表
create_table_sql = """
CREATE TABLE IF NOT EXISTS hive_table (
    column1 STRING,
    column2 INT,
    ...
)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE
"""
hive_cursor.execute(create_table_sql)

# 将数据写入Hive
df.to_csv('temp_data.csv', index=False, header=False)
with open('temp_data.csv', 'r') as f:
    hive_cursor.copy_from(f, 'hive_table', sep=',')

# 关闭连接
mysql_cursor.close()
mysql_conn.close()
hive_cursor.close()
hive_conn.close()