python_使用多线程来处理数据写入Excel文件_threading.Thread

python_使用多线程来处理数据写入Excel文件

因为pandas的ExcelWriter不支持并发写入同一个文件,所以如果需要的话，后期还要合并。

import pandas as pd from io import BytesIO import threading import multiprocessing import time import math  # 假设我们有12万行的数据 data = pd.DataFrame({     'A': range(120001),     'B': range(120001),     'C': range(120001) })   def write_to_excel(df, filename):     with pd.ExcelWriter(filename) as writer:         df.to_excel(writer, index=False)  def thread_write(df, filename):     thread = threading.Thread(target=write_to_excel, args=(df, filename))     thread.start()     return thread  def test_threading(data):     threads = []     # 分割数据     split_size=30000     #向上取整     num_threads = math.ceil(120001/split_size)          data_splits = [data[i:i + split_size] for i in range(0, len(data), split_size)]     filenames = [f'D:\\desktop\\new\\thread_{i}.xlsx' for i in range(num_threads)]          start_time = time.time()     for i in range(num_threads):         threads.append(thread_write(data_splits[i], filenames[i]))          for thread in threads:         #join方法用于等待线程完成         thread.join()          print(f"Threaded execution took {time.time() - start_time:.2f} seconds")  # 模拟主函数入口 if __name__ == '__main__':     test_threading(data)