阅读量:2
本文使用spring boot 结合minio文件服务做的大文件分片上传，思路：①：初始化文件调用后端接口，后端再调用minio把文件分片成几份，生成每个分片的minio上传url②：把提前分片好的文件依次调用上一次返回的url ③：调用合并分片文件接口，完成上传 PS：文件并未经过后端服务器，而是直接上传到minio文件服务的
文章目录
第一步：pom引入minio
<!--minio大文件上传--> <dependency> <groupId>io.minio</groupId> <artifactId>minio</artifactId> <version>8.0.3</version> </dependency> 第二步：配置minio
# minio 配置（大文件分片上传使用） minio: endpoint: ${MINIO_ENDPOINT:http://minio服务IP:端口号} accesskey: ${MINIO_ASSESSKEY:密钥} secretkey: ${MINIO_SECRETKEY:密匙} bucket: ${MINIO_BUCKET:桶名称} 
第三步：自定义minio客户端与工具类
1. 自定义客户端
package com.gcddd.visual.uav.largeFileUpload.util; import com.google.common.collect.Multimap; import io.minio.CreateMultipartUploadResponse; import io.minio.ListPartsResponse; import io.minio.MinioClient; import io.minio.ObjectWriteResponse; import io.minio.errors.*; import io.minio.messages.Part; import java.io.IOException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; /** * 自定义minio * @date 2023-09-15 */ public class CustomMinioClient extends MinioClient { public CustomMinioClient(MinioClient client) { super(client); } public String initMultiPartUpload(String bucket, String region, String object, Multimap<String, String> headers, Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException { CreateMultipartUploadResponse response = this.createMultipartUpload(bucket, region, object, headers, extraQueryParams); return response.result().uploadId(); } public ObjectWriteResponse mergeMultipartUpload(String bucketName, String region, String objectName, String uploadId, Part[] parts, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws IOException, InvalidKeyException, NoSuchAlgorithmException, InsufficientDataException, ServerException, InternalException, XmlParserException, InvalidResponseException, ErrorResponseException { return this.completeMultipartUpload(bucketName, region, objectName, uploadId, parts, extraHeaders, extraQueryParams); } public ListPartsResponse listMultipart(String bucketName, String region, String objectName, Integer maxParts, Integer partNumberMarker, String uploadId, Multimap<String, String> extraHeaders, Multimap<String, String> extraQueryParams) throws NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException { return this.listParts(bucketName, region, objectName, maxParts, partNumberMarker, uploadId, extraHeaders, extraQueryParams); } } 2.封装minio配置
package com.gcddd.visual.uav.largeFileUpload.configurer; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; /** * @Description: MinioProperties * minio配置类 */ @ConfigurationProperties(prefix = "minio") @Getter @Setter public class MinioProperties { private String endpoint; private String accesskey; private String secretkey; private String bucket; } 3.上传工具类
package com.gcddd.visual.uav.largeFileUpload.configurer; import cn.hutool.core.util.StrUtil; import com.gcddd.visual.uav.largeFileUpload.util.CustomMinioClient; import com.google.common.collect.HashMultimap; import io.minio.*; import io.minio.http.Method; import io.minio.messages.Part; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; /** * @Description: MinIoUtils * 工具类 */ @Component @Configuration @EnableConfigurationProperties({MinioProperties.class}) public class MinIoUtils { private final MinioProperties minioProperties; private CustomMinioClient customMinioClient; public MinIoUtils(MinioProperties minioProperties) { this.minioProperties = minioProperties; } @PostConstruct public void init() { MinioClient minioClient = MinioClient.builder() .endpoint(minioProperties.getEndpoint()) .credentials(minioProperties.getAccesskey(), minioProperties.getSecretkey()) .build(); customMinioClient = new CustomMinioClient(minioClient); } /** * 单文件签名上传 * * @param objectName 文件全路径名称 * @return / */ public String getUploadObjectUrl(String objectName) { try { return customMinioClient.getPresignedObjectUrl( GetPresignedObjectUrlArgs.builder() .method(Method.PUT) .bucket(minioProperties.getBucket()) .object(objectName) .expiry(1, TimeUnit.DAYS) //.extraHeaders(headers) .build() ); } catch (Exception e) { e.printStackTrace(); return null; } } /** * 初始化分片上传 * * @param objectName 文件全路径名称 * @param partCount 分片数量 * @param contentType 类型，如果类型使用默认流会导致无法预览 * @return / */ public Map<String, Object> initMultiPartUpload(String objectName, int partCount, String contentType) { Map<String, Object> result = new HashMap<>(); try { if (StrUtil.isBlank(contentType)) { contentType = "application/octet-stream"; } HashMultimap<String, String> headers = HashMultimap.create(); headers.put("Content-Type", contentType); String uploadId = customMinioClient.initMultiPartUpload(minioProperties.getBucket(), null, objectName, headers, null); result.put("uploadId", uploadId); List<String> partList = new ArrayList<>(); Map<String, String> reqParams = new HashMap<>(); //reqParams.put("response-content-type", "application/json"); reqParams.put("uploadId", uploadId); for (int i = 1; i <= partCount; i++) { reqParams.put("partNumber", String.valueOf(i)); String uploadUrl = customMinioClient.getPresignedObjectUrl( GetPresignedObjectUrlArgs.builder() .method(Method.PUT) .bucket(minioProperties.getBucket()) .object(objectName) .expiry(1, TimeUnit.DAYS) .extraQueryParams(reqParams) .build()); partList.add(uploadUrl); } result.put("uploadUrls", partList); } catch (Exception e) { e.printStackTrace(); return null; } return result; } /** * 分片上传完后合并 * * @param objectName 文件全路径名称 * @param uploadId 返回的uploadId * @return / */ public boolean mergeMultipartUpload(String objectName, String uploadId) { try { //TODO::目前仅做了最大1000分片 Part[] parts = new Part[1000]; ListPartsResponse partResult = customMinioClient.listMultipart(minioProperties.getBucket(), null, objectName, 1000, 0, uploadId, null, null); int partNumber = 1; for (Part part : partResult.result().partList()) { parts[partNumber - 1] = new Part(partNumber, part.etag()); partNumber++; } customMinioClient.mergeMultipartUpload(minioProperties.getBucket(), null, objectName, uploadId, parts, null, null); } catch (Exception e) { e.printStackTrace(); return false; } return true; } } 4.service
package com.gcddd.visual.uav.largeFileUpload.service; import java.util.Map; /** * @date 2023-09-13 */ public interface UploadService { /** * 分片上传初始化 * * @param path 路径 * @param filename 文件名 * @param partCount 分片数量 * @param contentType / * @return / */ Map<String, Object> initMultiPartUpload(String path, String filename, Integer partCount, String contentType); /** * 完成分片上传 * * @param objectName 文件名 * @param uploadId 标识 * @return / */ boolean mergeMultipartUpload(String objectName, String uploadId); } 5.service实现
package com.gcddd.visual.uav.largeFileUpload.service.impl; import com.gcddd.visual.uav.largeFileUpload.configurer.MinIoUtils; import com.gcddd.visual.uav.largeFileUpload.service.UploadService; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import java.util.Map; /** * @date 2023-09-13 */ @Service @RequiredArgsConstructor public class UploadServiceImpl implements UploadService { private final MinIoUtils minIoUtils; @Override public Map<String, Object> initMultiPartUpload(String path, String filename, Integer partCount, String contentType) { path = path.replaceAll("/+", "/"); if (path.indexOf("/") == 0) { path = path.substring(1); } String filePath = path + "/" + filename; Map<String, Object> result; // 单文件，直接上传 if (partCount == 1) { String uploadObjectUrl = minIoUtils.getUploadObjectUrl(filePath); result = ImmutableMap.of("uploadUrls", ImmutableList.of(uploadObjectUrl)); } else {//多文件，分片上传 result = minIoUtils.initMultiPartUpload(filePath, partCount, contentType); } return result; } @Override public boolean mergeMultipartUpload(String objectName, String uploadId) { return minIoUtils.mergeMultipartUpload(objectName, uploadId); } } 6.接口控制器
import cn.hutool.core.lang.Assert; import cn.hutool.json.JSONObject; import com.gcddd.visual.uav.largeFileUpload.service.UploadService; import com.gcddd.visual.uav.util.DateUtils; import com.google.common.collect.ImmutableMap; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import java.util.Map; /** * 大文件上传控制器 */ @RestController @RequiredArgsConstructor @RequestMapping("/largeFileUpload" ) @Tag(description = "largeFileUpload" , name = "大文件分片上传" ) @SecurityRequirement(name = HttpHeaders.AUTHORIZATION) public class uploadController { private final UploadService uploadService; // 1.用户调用初始化接口，后端调用minio初始化，得到uploadId，生成每个分片的minio上传url //初始化 示例： $ curl --location --request POST 'http://127.0.0.1:9998/admin/largeFileUpload/multipart/init' \ // --header 'Content-Type: application/json' \ // --data-raw '{ // "filename": "result.png", // "partCount": 2 // }' // 返回示例： //{ // "uploadId": "eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3", // "uploadUrls": [ // "http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=9e7bd28d6f4d42d0ed447750cadcd8782ccc2648a66ecb25a1e7639593ad7dd6", // "http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=60d31989b9dc3bdacf6bbc9c13cb5c982f79b1e3a0bbcb948e0ec90fede2f9f3" // ] //} // 2.用户调用对应分片的上传地址，多次上传会覆盖 // 示例： // curl --location --request PUT 'http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=9e7bd28d6f4d42d0ed447750cadcd8782ccc2648a66ecb25a1e7639593ad7dd6' \ // --header 'Content-Type: application/octet-stream' \ // --data-binary '@/D:/jpgone' // curl --location --request PUT 'http://ip:端口号/uav/20230913/result.png?uploadId=eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3&partNumber=2&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=lengleng%2F20230913%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230913T031427Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=60d31989b9dc3bdacf6bbc9c13cb5c982f79b1e3a0bbcb948e0ec90fede2f9f3' \ // --header 'Content-Type: application/octet-stream' \ // --data-binary '@/D:/jpgtwo' // 3.调用完成接口，后端查询所有上传的分片并合并，uploadId为第一步返回的uploadId // 示例： // curl --location --request PUT '127.0.0.1:8006/multipart/complete' \ // --header 'Content-Type: application/json' \ // --data-raw '{ // "objectName":"20230913/result.png", // "uploadId":"eb163e3d-bbf5-48ae-aa56-85cf2a3c0fa3" // }' /** * 分片初始化 * * @param requestParam 请求参数 * @return / */ @PostMapping("/multipart/init") public ResponseEntity<Object> initMultiPartUpload(@RequestBody JSONObject requestParam) { // 路径（存入minio的路径） String pathText = DateUtils.getTodays("yyyyMMdd");//当前日期 String path = requestParam.getStr("path", pathText); // 文件名 String filename = requestParam.getStr("filename", "test.obj"); // content-type String contentType = requestParam.getStr("contentType", "application/octet-stream"); // md5-可进行秒传判断 String md5 = requestParam.getStr("md5", ""); // 分片数量 Integer partCount = requestParam.getInt("partCount", 100); //TODO::业务判断+秒传判断 Map<String, Object> result = uploadService.initMultiPartUpload(path, filename, partCount, contentType); return new ResponseEntity<>(result, HttpStatus.OK); } /** * 合并分片文件，完成上传 * * @param requestParam 用户参数 * @return / */ @PutMapping("/multipart/complete") public ResponseEntity<Object> completeMultiPartUpload( @RequestBody JSONObject requestParam ) { // 文件名完整路径 String objectName = requestParam.getStr("objectName"); //初始化返回的uploadId String uploadId = requestParam.getStr("uploadId"); Assert.notNull(objectName, "objectName must not be null"); Assert.notNull(uploadId, "uploadId must not be null"); boolean result = uploadService.mergeMultipartUpload(objectName, uploadId); return new ResponseEntity<>(ImmutableMap.of("success", result), HttpStatus.OK); } } 第四步：文件切片
把result.png图片以500M大小切片，重命名为result.png.后缀
split -b 500m result.png result.png. 
第五步：示例截图
这里测试为把result.png切为2片，返回了uploadId 和两个minio文件上传地址
分片上传的地址上传文件，请求方式为put
合并文件，完成上传
检查minio是否成功
