MINIO服务器基于AWS S3 SDK 文件分片上传及下载（C++实现）

• 基于aws-s3安装
• • 安装
• 初始化S3连接
• 上传文件
• • 一般文件上传
  • 分块上传
• 下载文件
• 参考资料
• 下载地址

基于aws-s3安装

安装

安装环境依赖：

Debian/Ubuntu-based systems : sudo apt-get install libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev cmake Redhat/Fedora-based systems : sudo dnf install libcurl-devel openssl-devel libuuid-devel pulseaudio-devel cmake 

获取SDK源码并安装：

git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git cd aws-sdk-cpp mkdir build cd build cmake .. make -j8 sudo make install 

项目中CMakeLists.txt配置：

cmake_minimum_required(VERSION 3.19) project(project_name) set(CMAKE_CXX_STANDARD 11)   # Find the AWS SDK for C++ package. find_package(AWSSDK CONFIG COMPONENTS core s3 REQUIRED)   include_directories(${AWSSDK_INCLUDE_DIRS}) add_executable(project_name mian.cpp) target_link_libraries(project_name PRIVATE  aws-cpp-sdk-s3) 

初始化S3连接

S3Client InitS3Client(const Aws::String IPPort, const Aws::String& accessKeyId, const Aws::String& secretKey, std::string securityToken){     // 初始化 S3 Client     Aws::Client::ClientConfiguration cfg;     cfg.endpointOverride = IPPort;     cfg.scheme = Aws::Http::Scheme::HTTP;     cfg.verifySSL = false;      Aws::Auth::AWSCredentials cred(accessKeyId, secretKey, securityToken);     S3Client client(cred, cfg,                     Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);     return client; } 

上传文件

一般文件上传

/// 判断待上传的文件是否存在 inline bool file_exists(const string& name) {     struct stat buffer;     return (stat(name.c_str(), &buffer) == 0); }  /// 上传文件 bool doPutObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const string& file_name, const Aws::String ®ion) {     // 判断文件是否存在     if (!file_exists(file_name)) {         cout << "ERROR: 找不到这个文件，这个文件不存在"              << endl;         return false;     }      /// 初始化上传请求     Aws::S3::Model::PutObjectRequest object_request;     object_request.SetBucket(fromBucket);     object_request.SetKey(objectKey);      /// 建立文件输入串流，可为任何串流     const shared_ptr input_data =             Aws::MakeShared("SampleAllocationTag",                                           file_name.c_str(),                                           ios_base::in | ios_base::binary);     object_request.SetBody(input_data);      /// 开始上传至S3     Aws::S3::Model::PutObjectOutcome put_object_outcome = client.PutObject(object_request);      if (!put_object_outcome.IsSuccess()) {         auto error = put_object_outcome.GetError();         cout << "ERROR: " << error.GetExceptionName() << ": "              << error.GetMessage() << endl;         return false;     }else {         cout << "success" << endl;         return true;     } }  /// 启动上传程序 bool StartUpLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){     // 初始化     Aws::SDKOptions options;     options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;     Aws::InitAPI(options);      /// 设置ip、acccessKey、secretKey     Aws::String IP = Ips;     Aws::String accessKeyId = acccessKey;     Aws::String secretKey = secretKeyS;      S3Client client;     client = InitS3Client(IP, accessKeyId, secretKey, securityToken);      Aws::InitAPI(options);     {         /// 设置上传的目标桶和文件路径         const Aws::String bucket_name = bucket_names;         const Aws::String object_name = object_names;         if (!doPutObject(object_name, bucket_name, client, imagename, " ")) {             std::string errorInfo =  object_name + " 上传失败";             writeLog(L_ERROR, errorInfo.c_str());             return false;         }     }     writeLog(L_INFO, "====> 文件上传成功");     return true; } 

分块上传

/// MINIO 切片上传 bool mainCutUpload(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){       Aws::SDKOptions options;     options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;     Aws::InitAPI(options);      Aws::String IP = Ips;     Aws::String accessKeyId = acccessKey;     Aws::String secretKey = secretKeyS;      /// 初始化S3服务     S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);      Aws::InitAPI(options);     {         /// 设置上传的目标桶和文件路径         const Aws::String bucket_name = bucket_names;         const Aws::String object_name = object_names;         std::string bucket = bucket_name;         std::string key = object_name;          /// 初始化分块上传任务         Aws::S3::Model::CreateMultipartUploadRequest create_request;         create_request.SetBucket(bucket.c_str());         create_request.SetKey(key.c_str());         Aws::S3::Model::CreateMultipartUploadOutcome outcome = client.CreateMultipartUpload(create_request);          /// 获取上传ID         std::string upload_id = outcome.GetResult().GetUploadId();         std::cout << "upload id is:" << upload_id << std::endl;          /// 创建分段上传输出         if (!outcome.IsSuccess()) {             auto err = outcome.GetError();             std::cout << "Error: CreateMultipartUpload: " <<                       err.GetExceptionName() << ": " << err.GetMessage() << std::endl;             return false;         }         else             std::cout << "Success: CreateMultipartUpload: " << bucket << std::endl;          std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetBucket() << std::endl;         std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetKey() << std::endl;         std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetUploadId() << std::endl;          Aws::String UploadId = upload_id;          /// 启动分块上传         Aws::S3::Model::UploadPartRequest request1;         request1.SetBucket(bucket);         request1.SetKey(key);          /// 每个分片大小         long partSize = 5 * 1024 * 1024;          /// 读取文件         std::fstream file(imagename.c_str(),std::ios::in | std::ios::binary);         file.seekg(0,std::ios::end);         long fileSize = file.tellg();   /// 获取文件大小         file.seekg(0, std::ios::beg);         char* buffer = new char[partSize];  // 定义缓存          /// 初始化上传变量         long filePosition = 0;         Aws::Vector completeParts;         int partNumber = 1;          /// 开始上传         while(filePosition < fileSize){             partSize = std::min(partSize,(fileSize - filePosition));             file.read(buffer,partSize);             Aws::S3::Model::UploadPartRequest uploadPartRequest;             uploadPartRequest.WithBucket(bucket).WithKey(key).WithUploadId(UploadId).WithPartNumber(partNumber).WithContentLength(partSize);             Aws::String str(buffer,partSize);             auto input_data = Aws::MakeShared("UploadPartStream",str);             uploadPartRequest.SetBody(input_data);             filePosition += partSize;             auto uploadPartResult = client.UploadPart(uploadPartRequest);             completeParts.push_back(Aws::S3::Model::CompletedPart().WithETag(uploadPartResult.GetResult().GetETag()).WithPartNumber(partNumber));             memset(buffer, 0, partSize);             ++partNumber;         }          // 完成分块上传         Aws::S3::Model::CompleteMultipartUploadRequest request2;         request2.SetBucket(bucket);         request2.SetKey(key);         request2.SetUploadId(UploadId);         Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload;          completed_multipart_upload.SetParts(completeParts);         request2.SetMultipartUpload(completed_multipart_upload);         Aws::S3::Model::CompleteMultipartUploadOutcome outcome3 = client.CompleteMultipartUpload(request2);          if (outcome3.IsSuccess()) {             std::cout << "Success: CompleteMultipartUpload " << std::endl;             return true;         }         else         {             std::cout << "Error: CompleteMultipartUpload: " <<                       outcome.GetError().GetMessage() << std::endl;             return false;         }      } } 

下载文件

/// 从MINIO中下载文件或数据 bool doGetObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const Aws::String savePath,const Aws::String ®ion) {     /// 初始化下载请求     Aws::S3::Model::GetObjectRequest object_request;     object_request.SetBucket(fromBucket);     object_request.SetKey(objectKey);      /// 从S3服务器中下载数据     Aws::S3::Model::GetObjectOutcome get_object_outcome = client.GetObject(object_request);      if (get_object_outcome.IsSuccess()) {         /// 保存文件         Aws::OFStream local_file;         local_file.open(savePath, std::ios::out | std::ios::binary);         local_file << get_object_outcome.GetResultWithOwnership().GetBody().rdbuf();         std::cout << "Done!" << std::endl;         return true;     } else {         auto err = get_object_outcome.GetError();         std::cout << "Error: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl;         return false;     } }   /// 启动下载程序 bool StartDownLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_name, std::string downPath, std::string imagePath, std::string securityToken,     bool &flags){     // 初始化Aws API     Aws::SDKOptions options;     options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;     Aws::InitAPI(options);      /// 设置ip、acccessKey、secretKey     Aws::String IP = Ips;     Aws::String accessKeyId = acccessKey;     Aws::String secretKey = secretKeyS;      /// 初始化S3服务     S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);      /// 根据get接口的桶名称设置文件所在的桶     Aws::S3::Model::Bucket temps{};     std::string stringK = bucket_name;     temps.SetName(stringK);      /// 查找MINIO相关数据     Aws::S3::Model::ListObjectsRequest objects_request;     objects_request.WithBucket(temps.GetName());   /// 设置桶路径     objects_request.SetPrefix(downPath);   /// 设置文件夹路径      auto list_objects_outcome = client.ListObjects(objects_request);  /// 判断存储桶及相关连接是否有效      if (flags){         writeLog(L_INFO, "====> 文件下载终止");         return false;     }     if (list_objects_outcome.IsSuccess())     {         /// 获取该目录下的前一千个文件         Aws::Vector object_list =                 list_objects_outcome.GetResult().GetContents();          if (object_list.size() == 1000){              /// 当下载数量大于一千时 循环下载文件             while(object_list.size() == 1000){                 auto finally_object = object_list[0];                 for (auto const &s3_object : object_list)                 {                     if (flags){                         writeLog(L_INFO, "====> 文件下载终止");                         return false;                     }                     /// 设置桶名称                     const Aws::String bucket_name = stringK;                     /// 设置待下载的文件名称                     const Aws::String object_name = s3_object.GetKey();                     /// 设置保存文件路径及文件名称                     std::string NewPath;                     size_t indexNew = object_name.find_last_of('/');                     NewPath = object_name.substr(indexNew + 1, object_name.length());                     const Aws::String SavePath = imagePath + NewPath;                     size_t indexK = downPath.find_last_of('/');                     std::string newPathEs = downPath.substr(0, indexK);                     std::string strK;                     strK = imagePath + "/" + NewPath;                     const Aws::String SavePaths = strK;                      //筛选数据                     std::string fileName;                     size_t index = object_name.find_last_of('.');                     fileName = object_name.substr(index + 1, object_name.length());                     if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){                         /// 启动下载                         if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {                             std::string errorInfo =  object_name + " 下载失败";                             writeLog(L_ERROR, errorInfo.c_str());                             continue;                         }                     }                 }                 finally_object = object_list[999];                 /// 重新配置相关路径                 objects_request.WithBucket(temps.GetName());   /// 设置桶路径                 objects_request.SetPrefix(finally_object.GetKey());   /// 设置新文件夹路径                  list_objects_outcome = client.ListObjects(objects_request);  /// 判断存储桶及相关连接是否有效                 if (list_objects_outcome.IsSuccess()){                     object_list = list_objects_outcome.GetResult().GetContents();                 }else{                     writeLog(L_ERROR, "====> 再次连接失败");                 }             }             for (auto const &s3_object : object_list)             {                 if (flags){                     writeLog(L_INFO, "====> 文件下载终止");                     return false;                 }                 /// 设置桶名称                 const Aws::String bucket_name = stringK;                 /// 设置待下载的文件名称                 const Aws::String object_name = s3_object.GetKey();                 /// 设置保存文件路径及文件名称                 std::string NewPath;                 size_t indexNew = object_name.find_last_of('/');                 NewPath = object_name.substr(indexNew + 1, object_name.length());                 const Aws::String SavePath = imagePath + NewPath;                 size_t indexK = downPath.find_last_of('/');                 std::string newPathEs = downPath.substr(0, indexK);                  std::string strK;                 strK = imagePath + "/" + NewPath;                 const Aws::String SavePaths = strK;                  //筛选数据                 std::string fileName;                 size_t index = object_name.find_last_of('.');                 fileName = object_name.substr(index + 1, object_name.length());                 if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){                     /// 启动下载                     if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {                         std::string errorInfo =  object_name + " 下载失败";                         writeLog(L_ERROR, errorInfo.c_str());                         continue;                     }                 }             }         }else{             for (auto const &s3_object : object_list)             {                 if (flags){                     writeLog(L_INFO, "====> 文件下载终止");                     return false;                 }                 /// 设置桶名称                 const Aws::String bucket_name = stringK;                 /// 设置待下载的文件名称                 const Aws::String object_name = s3_object.GetKey();                 /// 设置保存文件路径及文件名称                 std::string NewPath;                 size_t indexNew = object_name.find_last_of('/');                 NewPath = object_name.substr(indexNew + 1, object_name.length());                 const Aws::String SavePath = imagePath +  NewPath;                 size_t indexK = downPath.find_last_of('/');                 std::string newPathEs = downPath.substr(0, indexK);                 std::string strK;                 strK = imagePath + "/" + NewPath;                 const Aws::String SavePaths = strK;                  //筛选数据                 std::string fileName;                 size_t index = object_name.find_last_of('.');                 fileName = object_name.substr(index + 1, object_name.length());                 if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){                     /// 启动下载                     if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {                         std::string errorInfo =  object_name + " 下载失败";                         writeLog(L_ERROR, errorInfo.c_str());                         continue;                     }                 }              }         }      }     writeLog(L_INFO, "====> 文件下载成功");     return true; } 

参考资料

SDK文档资料
C++_SDK.pdf

下载地址

实现下载的整体类代码下载
具体内容如下