简介
面向对象存储
xml
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>8.2.1</version>
</dependency>使用原生的aws sdk或者minio都可以 [https://docs.aws.amazon.com/zh_cn/sdk-for-java/v1/developer-guide/examples-s3.html](https://docs.aws.amazon.com/zh_cn/sdk-for-java/v1/developer-guide/examples-s3.html)[doc](https://docs.min.io/docs/java-client-api-reference.html)
配置
java
@Component
public class AwsS3Component implements InitializingBean {
@Value("${custom.aws.access-key}")
private String accessKey;
@Value("${custom.aws.secret-key}")
private String accessSecret;
@Value("${custom.aws.endpoint}")
private String endpoint;
private AmazonS3 client;
public AmazonS3 getClient() {
return client;
}
@Override
public void afterPropertiesSet() {
ClientConfiguration config = new ClientConfiguration();
config.setProtocol(Protocol.HTTP);
config.disableSocketProxy();
this.client = AmazonS3ClientBuilder
.standard()
.withClientConfiguration(config)
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, accessSecret)))
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, Regions.CN_NORTH_1.getName()))
.enablePathStyleAccess()
.build();
}
}java
@Component
public class MinIOFactory implements InitializingBean {
@Value("${custom.aws.access-key}")
private String accessKey;
@Value("${custom.aws.secret-key}")
private String accessSecret;
@Value("${custom.aws.endpoint}")
private String endpoint;
private MinioClient minioClient;
public MinioClient getMinioClient() {
return minioClient;
}
@Override
public void afterPropertiesSet() {
this.minioClient = new MinioClient.Builder()
.credentials(accessKey,accessSecret)
.endpoint(endpoint,80,false)
.region("cn-north-1")
.build();
}
}测试
java
@Autowired
private AwsS3Component awsS3Component;
//读取S3中某个具体的文件,需要传递 租户id,项目id和文件名称fileName
@Test
void contextLoads() throws Exception{
// bucketId,path
S3Object o = awsS3Component.getClient().getObject("1111","a/1.json");
S3ObjectInputStream s3is = o.getObjectContent();
InputStream inputStream = s3is.getDelegateStream();
}
//创建某个桶
@Test
void a(){
String bucketId = UUID.randomUUID().toString();
Bucket b ;
AmazonS3 amazonS3 = awsS3Component.getClient();
if (amazonS3.doesBucketExistV2(bucketId)) {
System.out.format("Bucket %s already exists.\n", bucketId);
// b = s3.getBucket(bucketId);
} else {
try {
b = amazonS3.createBucket(bucketId);
} catch (AmazonS3Exception e) {
System.err.println(e.getErrorMessage());
}
}
}
//删除桶以及桶内所有对象
@Test
void b(){
String bucketName = "1111";
AmazonS3 s3Client = awsS3Component.getClient();
ObjectListing objectListing = s3Client.listObjects(bucketName);
while (true) {
Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
while (objIter.hasNext()) {
s3Client.deleteObject(bucketName, objIter.next().getKey());
}
if (objectListing.isTruncated()) {
objectListing = s3Client.listNextBatchOfObjects(objectListing);
} else {
break;
}
}
VersionListing versionList = s3Client.listVersions(new ListVersionsRequest().withBucketName(bucketName));
while (true) {
Iterator<S3VersionSummary> versionIter = versionList.getVersionSummaries().iterator();
while (versionIter.hasNext()) {
S3VersionSummary vs = versionIter.next();
s3Client.deleteVersion(bucketName, vs.getKey(), vs.getVersionId());
}
if (versionList.isTruncated()) {
versionList = s3Client.listNextBatchOfVersions(versionList);
} else {
break;
}
}
s3Client.deleteBucket(bucketName);
}
//删除某个对象
@Test
void c (){
AmazonS3 s3Client = awsS3Component.getClient();
s3Client.deleteObject("1111","1.md");
}
//创建对象,根据JSONObject
@Test
void d(){
AmazonS3 s3Client = awsS3Component.getClient();
String bucketId = "1111";
String key ="a/1.json";
JSONObject jsonObject = new JSONObject();
jsonObject.put("id","2");
InputStream result = new ByteArrayInputStream(jsonObject.toString().getBytes(StandardCharsets.UTF_8));
ObjectMetadata objectMetadata = new ObjectMetadata();
s3Client.putObject(bucketId,key,result,objectMetadata);
}java
@Component
@Slf4j
public class MinioUtil {
private static MinioClient minioClient;
private final MinioConfig minioConfig;
public MinioUtil(MinioConfig minioConfig) {
this.minioConfig = minioConfig;
minioClient = minioConfig.getMinioClient();
}
/**
* 检查该租户是否已存在
* @param bucketId
* @return
* @throws Exception
*/
public static boolean bucketExists(String bucketId) throws Exception{
return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketId).build());
}
/**
* 检查对象是否存在
* @param bucketId
* @param key
* @return
*/
public static boolean objectExists(String bucketId,String key){
Iterable<Result<Item>> results = minioClient.listObjects(
ListObjectsArgs.builder()
.bucket(bucketId)
.prefix(key)
.build());
return results.iterator().hasNext();
}
/**
* 获取对象内容
* @param bucketId
* @param key
* @return
* @throws Exception
*/
public static String getObjectContent(String bucketId,String key) throws Exception{
GetObjectArgs getObjectArgs = GetObjectArgs.builder()
.bucket(bucketId).object(key)
.build();
InputStream inputStream = null;
try {
inputStream = minioClient.getObject(getObjectArgs);
return new String(ByteStreams.toByteArray(inputStream));
} finally {
if(inputStream!=null){
inputStream.close();
}
}
}
/**
* 创建桶
* @param bucketId 租户id
* @return
*/
public static void createBucketIfNotExists(String bucketId) throws Exception{
if(!bucketExists(bucketId)){
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketId).build());
}
}
/**
* 删除桶内某个文件夹
* @param bucketId
* @param key
* @throws Exception
*/
public static void removeFolder(String bucketId,String key) throws Exception{
Iterable<Result<Item>> results = minioClient.listObjects(
ListObjectsArgs.builder()
.bucket(bucketId)
.prefix(key)
.build());
if(results.iterator().hasNext()){
List<DeleteObject> objects = new LinkedList<>();
for (Result<Item> result : results) {
objects.add(new DeleteObject(result.get().objectName()));
}
Iterable<Result<DeleteError>> results1 = minioClient.removeObjects(
RemoveObjectsArgs.builder().bucket(bucketId).objects(objects).build());
for (Result<DeleteError> result : results1) {
DeleteError error = result.get();
log.error("Error in deleting object " + error.objectName() + "; " + error.message());
}
}
}
/**
* 添加对象
* @param bucketId 租户id
* @param content 文件内容
* @return
*/
public static void putObject(String bucketId,String key,String content) throws Exception{
createBucketIfNotExists(bucketId);
InputStream inputStream = null;
try {
inputStream = new ByteArrayInputStream(content.toString().getBytes(StandardCharsets.UTF_8));
minioClient.putObject(
PutObjectArgs.builder().bucket(bucketId).object(key)
.stream(inputStream, inputStream.available(), -1)
.build());
} finally {
if(inputStream!=null){
inputStream.close();
}
}
}
/**
* 删除某个对象
* @param bucketId
* @param key
* @throws Exception
*/
public static void removeObject(String bucketId,String key) throws Exception{
minioClient.removeObject(
RemoveObjectArgs.builder()
.bucket(bucketId)
.object(key)
.build(批量下载
网上的例子大多数仅针对 一个bucket下面不存在多层级目录的情况,下面的代码实现了多层级目录递归下载的目的. 比如: 下载 bucket-A中 目录python下面的所有对象
- /bucket-A/python/demo.py
- /bucket-A/python/lib/A.py
java
public static void downloadFiles(MinioClient minioClient,String bucketId,String path,String downloadPath) throws Exception {
boolean found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketId).build());
if (found) {
ListObjectsArgs listObjectsArgs = ListObjectsArgs.builder()
.bucket(bucketId).startAfter(path).recursive(true).build();
Iterable<Result<Item>> listObjects = minioClient.listObjects(listObjectsArgs);
for (Result<Item> result : listObjects) {
Item item = result.get();
String objectName = item.objectName();
if(!item.objectName().endsWith("/")){
// 获取父级目录,并创建,避免 minioClient.downloadObject 异常
Paths.get(downloadPath + objectName).normalize().toFile().getParentFile().mkdirs();
minioClient.downloadObject(
DownloadObjectArgs.builder()
.bucket(bucketId)
.object(item.objectName())
.filename(downloadPath+item.objectName())
.build());
}else{
Paths.get(downloadPath + path).normalize().toFile().mkdirs();
}
}
}
}读取文件内容
java
GetObjectResponse ylx = minioClient.getObject(GetObjectArgs.builder().bucket(s3Param.getBucket()).object(s3Param.getObjectName()).build());
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(ylx));
String collect = bufferedReader.lines().collect(Collectors.joining());