阅读量:1
MINIO服务器基于AWS S3 SDK 文件分片上传及下载（C++实现）
基于aws-s3安装
安装
安装环境依赖：
Debian/Ubuntu-based systems : sudo apt-get install libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev cmake Redhat/Fedora-based systems : sudo dnf install libcurl-devel openssl-devel libuuid-devel pulseaudio-devel cmake 获取SDK源码并安装：
git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git cd aws-sdk-cpp mkdir build cd build cmake .. make -j8 sudo make install 项目中CMakeLists.txt配置：
cmake_minimum_required(VERSION 3.19) project(project_name) set(CMAKE_CXX_STANDARD 11) # Find the AWS SDK for C++ package. find_package(AWSSDK CONFIG COMPONENTS core s3 REQUIRED) include_directories(${AWSSDK_INCLUDE_DIRS}) add_executable(project_name mian.cpp) target_link_libraries(project_name PRIVATE aws-cpp-sdk-s3) 初始化S3连接
S3Client InitS3Client(const Aws::String IPPort, const Aws::String& accessKeyId, const Aws::String& secretKey, std::string securityToken){ // 初始化 S3 Client Aws::Client::ClientConfiguration cfg; cfg.endpointOverride = IPPort; cfg.scheme = Aws::Http::Scheme::HTTP; cfg.verifySSL = false; Aws::Auth::AWSCredentials cred(accessKeyId, secretKey, securityToken); S3Client client(cred, cfg, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false); return client; } 上传文件
一般文件上传
/// 判断待上传的文件是否存在 inline bool file_exists(const string& name) { struct stat buffer; return (stat(name.c_str(), &buffer) == 0); } /// 上传文件 bool doPutObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const string& file_name, const Aws::String ®ion) { // 判断文件是否存在 if (!file_exists(file_name)) { cout << "ERROR: 找不到这个文件，这个文件不存在" << endl; return false; } /// 初始化上传请求 Aws::S3::Model::PutObjectRequest object_request; object_request.SetBucket(fromBucket); object_request.SetKey(objectKey); /// 建立文件输入串流，可为任何串流 const shared_ptr<Aws::IOStream> input_data = Aws::MakeShared<Aws::FStream>("SampleAllocationTag", file_name.c_str(), ios_base::in | ios_base::binary); object_request.SetBody(input_data); /// 开始上传至S3 Aws::S3::Model::PutObjectOutcome put_object_outcome = client.PutObject(object_request); if (!put_object_outcome.IsSuccess()) { auto error = put_object_outcome.GetError(); cout << "ERROR: " << error.GetExceptionName() << ": " << error.GetMessage() << endl; return false; }else { cout << "success" << endl; return true; } } /// 启动上传程序 bool StartUpLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){ // 初始化 Aws::SDKOptions options; options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; Aws::InitAPI(options); /// 设置ip、acccessKey、secretKey Aws::String IP = Ips; Aws::String accessKeyId = acccessKey; Aws::String secretKey = secretKeyS; S3Client client; client = InitS3Client(IP, accessKeyId, secretKey, securityToken); Aws::InitAPI(options); { /// 设置上传的目标桶和文件路径 const Aws::String bucket_name = bucket_names; const Aws::String object_name = object_names; if (!doPutObject(object_name, bucket_name, client, imagename, " ")) { std::string errorInfo = object_name + " 上传失败"; writeLog(L_ERROR, errorInfo.c_str()); return false; } } writeLog(L_INFO, "====> 文件上传成功"); return true; } 分块上传
/// MINIO 切片上传 bool mainCutUpload(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){ Aws::SDKOptions options; options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; Aws::InitAPI(options); Aws::String IP = Ips; Aws::String accessKeyId = acccessKey; Aws::String secretKey = secretKeyS; /// 初始化S3服务 S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken); Aws::InitAPI(options); { /// 设置上传的目标桶和文件路径 const Aws::String bucket_name = bucket_names; const Aws::String object_name = object_names; std::string bucket = bucket_name; std::string key = object_name; /// 初始化分块上传任务 Aws::S3::Model::CreateMultipartUploadRequest create_request; create_request.SetBucket(bucket.c_str()); create_request.SetKey(key.c_str()); Aws::S3::Model::CreateMultipartUploadOutcome outcome = client.CreateMultipartUpload(create_request); /// 获取上传ID std::string upload_id = outcome.GetResult().GetUploadId(); std::cout << "upload id is:" << upload_id << std::endl; /// 创建分段上传输出 if (!outcome.IsSuccess()) { auto err = outcome.GetError(); std::cout << "Error: CreateMultipartUpload: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl; return false; } else std::cout << "Success: CreateMultipartUpload: " << bucket << std::endl; std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetBucket() << std::endl; std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetKey() << std::endl; std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetUploadId() << std::endl; Aws::String UploadId = upload_id; /// 启动分块上传 Aws::S3::Model::UploadPartRequest request1; request1.SetBucket(bucket); request1.SetKey(key); /// 每个分片大小 long partSize = 5 * 1024 * 1024; /// 读取文件 std::fstream file(imagename.c_str(),std::ios::in | std::ios::binary); file.seekg(0,std::ios::end); long fileSize = file.tellg(); /// 获取文件大小 file.seekg(0, std::ios::beg); char* buffer = new char[partSize]; // 定义缓存 /// 初始化上传变量 long filePosition = 0; Aws::Vector<Aws::S3::Model::CompletedPart> completeParts; int partNumber = 1; /// 开始上传 while(filePosition < fileSize){ partSize = std::min(partSize,(fileSize - filePosition)); file.read(buffer,partSize); Aws::S3::Model::UploadPartRequest uploadPartRequest; uploadPartRequest.WithBucket(bucket).WithKey(key).WithUploadId(UploadId).WithPartNumber(partNumber).WithContentLength(partSize); Aws::String str(buffer,partSize); auto input_data = Aws::MakeShared<Aws::StringStream>("UploadPartStream",str); uploadPartRequest.SetBody(input_data); filePosition += partSize; auto uploadPartResult = client.UploadPart(uploadPartRequest); completeParts.push_back(Aws::S3::Model::CompletedPart().WithETag(uploadPartResult.GetResult().GetETag()).WithPartNumber(partNumber)); memset(buffer, 0, partSize); ++partNumber; } // 完成分块上传 Aws::S3::Model::CompleteMultipartUploadRequest request2; request2.SetBucket(bucket); request2.SetKey(key); request2.SetUploadId(UploadId); Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload; completed_multipart_upload.SetParts(completeParts); request2.SetMultipartUpload(completed_multipart_upload); Aws::S3::Model::CompleteMultipartUploadOutcome outcome3 = client.CompleteMultipartUpload(request2); if (outcome3.IsSuccess()) { std::cout << "Success: CompleteMultipartUpload " << std::endl; return true; } else { std::cout << "Error: CompleteMultipartUpload: " << outcome.GetError().GetMessage() << std::endl; return false; } } } 下载文件
/// 从MINIO中下载文件或数据 bool doGetObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const Aws::String savePath,const Aws::String ®ion) { /// 初始化下载请求 Aws::S3::Model::GetObjectRequest object_request; object_request.SetBucket(fromBucket); object_request.SetKey(objectKey); /// 从S3服务器中下载数据 Aws::S3::Model::GetObjectOutcome get_object_outcome = client.GetObject(object_request); if (get_object_outcome.IsSuccess()) { /// 保存文件 Aws::OFStream local_file; local_file.open(savePath, std::ios::out | std::ios::binary); local_file << get_object_outcome.GetResultWithOwnership().GetBody().rdbuf(); std::cout << "Done!" << std::endl; return true; } else { auto err = get_object_outcome.GetError(); std::cout << "Error: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl; return false; } } /// 启动下载程序 bool StartDownLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_name, std::string downPath, std::string imagePath, std::string securityToken, bool &flags){ // 初始化Aws API Aws::SDKOptions options; options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; Aws::InitAPI(options); /// 设置ip、acccessKey、secretKey Aws::String IP = Ips; Aws::String accessKeyId = acccessKey; Aws::String secretKey = secretKeyS; /// 初始化S3服务 S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken); /// 根据get接口的桶名称设置文件所在的桶 Aws::S3::Model::Bucket temps{}; std::string stringK = bucket_name; temps.SetName(stringK); /// 查找MINIO相关数据 Aws::S3::Model::ListObjectsRequest objects_request; objects_request.WithBucket(temps.GetName()); /// 设置桶路径 objects_request.SetPrefix(downPath); /// 设置文件夹路径 auto list_objects_outcome = client.ListObjects(objects_request); /// 判断存储桶及相关连接是否有效 if (flags){ writeLog(L_INFO, "====> 文件下载终止"); return false; } if (list_objects_outcome.IsSuccess()) { /// 获取该目录下的前一千个文件 Aws::Vector<Aws::S3::Model::Object> object_list = list_objects_outcome.GetResult().GetContents(); if (object_list.size() == 1000){ /// 当下载数量大于一千时 循环下载文件 while(object_list.size() == 1000){ auto finally_object = object_list[0]; for (auto const &s3_object : object_list) { if (flags){ writeLog(L_INFO, "====> 文件下载终止"); return false; } /// 设置桶名称 const Aws::String bucket_name = stringK; /// 设置待下载的文件名称 const Aws::String object_name = s3_object.GetKey(); /// 设置保存文件路径及文件名称 std::string NewPath; size_t indexNew = object_name.find_last_of('/'); NewPath = object_name.substr(indexNew + 1, object_name.length()); const Aws::String SavePath = imagePath + NewPath; size_t indexK = downPath.find_last_of('/'); std::string newPathEs = downPath.substr(0, indexK); std::string strK; strK = imagePath + "/" + NewPath; const Aws::String SavePaths = strK; //筛选数据 std::string fileName; size_t index = object_name.find_last_of('.'); fileName = object_name.substr(index + 1, object_name.length()); if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){ /// 启动下载 if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) { std::string errorInfo = object_name + " 下载失败"; writeLog(L_ERROR, errorInfo.c_str()); continue; } } } finally_object = object_list[999]; /// 重新配置相关路径 objects_request.WithBucket(temps.GetName()); /// 设置桶路径 objects_request.SetPrefix(finally_object.GetKey()); /// 设置新文件夹路径 list_objects_outcome = client.ListObjects(objects_request); /// 判断存储桶及相关连接是否有效 if (list_objects_outcome.IsSuccess()){ object_list = list_objects_outcome.GetResult().GetContents(); }else{ writeLog(L_ERROR, "====> 再次连接失败"); } } for (auto const &s3_object : object_list) { if (flags){ writeLog(L_INFO, "====> 文件下载终止"); return false; } /// 设置桶名称 const Aws::String bucket_name = stringK; /// 设置待下载的文件名称 const Aws::String object_name = s3_object.GetKey(); /// 设置保存文件路径及文件名称 std::string NewPath; size_t indexNew = object_name.find_last_of('/'); NewPath = object_name.substr(indexNew + 1, object_name.length()); const Aws::String SavePath = imagePath + NewPath; size_t indexK = downPath.find_last_of('/'); std::string newPathEs = downPath.substr(0, indexK); std::string strK; strK = imagePath + "/" + NewPath; const Aws::String SavePaths = strK; //筛选数据 std::string fileName; size_t index = object_name.find_last_of('.'); fileName = object_name.substr(index + 1, object_name.length()); if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){ /// 启动下载 if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) { std::string errorInfo = object_name + " 下载失败"; writeLog(L_ERROR, errorInfo.c_str()); continue; } } } }else{ for (auto const &s3_object : object_list) { if (flags){ writeLog(L_INFO, "====> 文件下载终止"); return false; } /// 设置桶名称 const Aws::String bucket_name = stringK; /// 设置待下载的文件名称 const Aws::String object_name = s3_object.GetKey(); /// 设置保存文件路径及文件名称 std::string NewPath; size_t indexNew = object_name.find_last_of('/'); NewPath = object_name.substr(indexNew + 1, object_name.length()); const Aws::String SavePath = imagePath + NewPath; size_t indexK = downPath.find_last_of('/'); std::string newPathEs = downPath.substr(0, indexK); std::string strK; strK = imagePath + "/" + NewPath; const Aws::String SavePaths = strK; //筛选数据 std::string fileName; size_t index = object_name.find_last_of('.'); fileName = object_name.substr(index + 1, object_name.length()); if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){ /// 启动下载 if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) { std::string errorInfo = object_name + " 下载失败"; writeLog(L_ERROR, errorInfo.c_str()); continue; } } } } } writeLog(L_INFO, "====> 文件下载成功"); return true; } 参考资料
下载地址
实现下载的整体类代码下载
具体内容如下
