Java 操作阿里 OSS

一、上代码

package com.xiaoqiang.demo;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.ListObjectsRequest;
import com.aliyun.oss.model.OSSObject;
import com.aliyun.oss.model.OSSObjectSummary;
import com.aliyun.oss.model.ObjectListing;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;

@Component
public class test1Task implements CommandLineRunner {
    private static final Logger logger = LoggerFactory.getLogger(test1Task.class);

    @Override
    public void run(String... args) throws IOException {
        String accessKeyId = "XXXXX";
        String accessKeySecret = "XXXXX";
        String endpoint = "XXXXX";
        OSSClient ossClient = new OSSClient(endpoint, accessKeyId, accessKeySecret);

        String bucketName = "heheda-data";
        
        // TODO 读取文件
        String objectName = "test.txt";
        OSSObject ossObject = ossClient.getObject(bucketName, objectName);

        InputStream inputStream = ossObject.getObjectContent();
        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
        String line;
        StringBuilder content = new StringBuilder();
        while ((line = reader.readLine()) != null) {
            content.append(line);
        }
        System.out.println(content.toString());


        // TODO 读取gz文件解压后的内容
        String objectName = "xiaoqiang/heheda-data-2023-11-15 11-0.log.gz";
        OSSObject ossObject = ossClient.getObject(bucketName, objectName);
        InputStream inputStream = ossObject.getObjectContent();
        GZIPInputStream gzipIS = new GZIPInputStream(inputStream);

        InputStreamReader streamReader = new InputStreamReader(gzipIS);
        BufferedReader reader = new BufferedReader(streamReader);
        String line;
        while ((line = reader.readLine()) != null) {
            System.out.println(line);
        }


        // TODO 获取目录下的文件信息
        List<JSONObject> listPath = new ArrayList<>();
        try {
            // 构造请求对象
            ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName);
            listObjectsRequest.setPrefix("2023-11-15/");
            listObjectsRequest.setMaxKeys(1000); // 设置每次返回的最大文件数，默认值为1000
            ObjectListing objectListing = ossClient.listObjects(listObjectsRequest);

            // 遍历文件列表（包括文件夹和文件）
            List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
            for (OSSObjectSummary s : sums) {
                JSONObject jsonObject = new JSONObject();
                String key = s.getKey();
                int length = key.length();
                int i = key.lastIndexOf("/") + 1;
                if (i != length) {
                    jsonObject.put("path",key);
                    jsonObject.put("size",s.getSize());
                }
                listPath.add(jsonObject);
            }

            // 如果文件列表有更多数据，继续遍历
            while (objectListing.isTruncated()) {
                ListObjectsRequest listObjectsRequest1 = new ListObjectsRequest(bucketName);
                listObjectsRequest1.setPrefix(objectListing.getPrefix());
                listObjectsRequest1.setMaxKeys(1000); // 设置每次返回的最大文件数，默认值为1000
                objectListing = ossClient.listObjects(listObjectsRequest1);
                sums.addAll(objectListing.getObjectSummaries());
                for (OSSObjectSummary s : sums) {
                    JSONObject jsonObject = new JSONObject();
                    String key = s.getKey();
                    int length = key.length();
                    int i = key.lastIndexOf("/") + 1;
                    if (i != length) {
                        jsonObject.put("path",key);
                        jsonObject.put("size",s.getSize());
                    }
                    listPath.add(jsonObject);
                }
            }
        } catch (Exception exception) {
            System.out.println(exception);
            ossClient.shutdown();
        }

        for (JSONObject path : listPath) {
            System.out.println(path);
        }
        // 输出结果：
        // {"path":"2023-11-15/heheda-xiaoqiang-data-2023-11-15 11-0.log.gz","size":36729173}
        // {"path":"2023-11-15/heheda-xiaoqiang-data-2023-11-15 12-0.log.gz","size":36989628}


        // TODO 下载并压缩为ZIP
        // 创建临时文件
        File zipFile = File.createTempFile("test", ".zip");

        FileOutputStream f = new FileOutputStream(zipFile);
        /**
         * 作用是为任何OutputStream产生校验和
         * 第一个参数是制定产生校验和的输出流，第二个参数是指定Checksum的类型 （Adler32（较快）和CRC32两种）
         */
        CheckedOutputStream csum = new CheckedOutputStream(f, new Adler32());
        // 用于将数据压缩成Zip文件格式
        ZipOutputStream zos = new ZipOutputStream(csum);
        List<String> keyList = new ArrayList<>();
        keyList.add("2023-11-15/heheda-data-2023-11-15 11-0.log.gz");
        for (String ossFile : keyList) {
            // 获取Object，返回结果为OSSObject对象

            OSSObject ossObject = ossClient.getObject(bucketName, ossFile);
            // 读去Object内容  返回
            InputStream inputStream = ossObject.getObjectContent();
            // 对于每一个要被存放到压缩包的文件，都必须调用ZipOutputStream对象的putNextEntry()方法，确保压缩包里面文件不同名
            String name = ossFile.substring(ossFile.lastIndexOf("/") + 1);
            zos.putNextEntry(new ZipEntry(name));
            int bytesRead = 0;
            // 向压缩文件中输出数据
            while ((bytesRead = inputStream.read()) != -1) {
                zos.write(bytesRead);
            }
            inputStream.close();
            zos.closeEntry(); // 当前文件写完，定位为写入下一条项目
        }
        zos.close();

        FileInputStream fis = new FileInputStream(zipFile);
        BufferedInputStream buff = new BufferedInputStream(fis);
        BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream("D:\\huiqdata.zip"));
        byte[] car = new byte[1024];
        int l = 0;
        while (l < zipFile.length()) {
            int j = buff.read(car, 0, 1024);
            l += j;
            out.write(car, 0, j);
        }
        // 关闭流
        fis.close();
        buff.close();
        out.close();
        // 删除临时文件
        zipFile.delete();

        ossClient.shutdown();
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165

二、Maven 配置

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0modelVersion>
    <parent>
        <groupId>org.springframework.bootgroupId>
        <artifactId>spring-boot-starter-parentartifactId>
        <version>2.0.4.RELEASEversion>
        <relativePath/> 
    parent>
    <groupId>com.examplegroupId>
    <artifactId>demoartifactId>
    <version>0.0.1-SNAPSHOTversion>
    <name>demoname>
    <description>Demo project for Spring Bootdescription>

    <dependencies>
        <dependency>
            <groupId>org.springframework.bootgroupId>
            <artifactId>spring-boot-starter-webartifactId>
        dependency>

        <dependency>
            <groupId>com.aliyun.odpsgroupId>
            <artifactId>hadoop-fs-ossartifactId>
            <version>3.3.8-publicversion>
        dependency>
        
        <dependency>
            <groupId>com.taosdata.jdbcgroupId>
            <artifactId>taos-jdbcdriverartifactId>
            <version>2.0.22version>
        dependency>
    dependencies>

    <build>
        <finalName>SpringBootTestfinalName>
        <plugins>
            <plugin>
                <groupId>org.springframework.bootgroupId>
                <artifactId>spring-boot-maven-pluginartifactId>
            plugin>
        plugins>
    build>

project>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

注意：在没有 com.taosdata.jdbc 依赖时会报错：

java.lang.NoClassDefFoundError: org/apache/http/ssl/SSLContextBuilder
	at com.aliyun.oss.common.comm.DefaultServiceClient.createHttpClientConnectionManager(DefaultServiceClient.java:217) ~[aliyun-sdk-oss-2.2.1.jar:na]
	at com.aliyun.oss.common.comm.DefaultServiceClient.(DefaultServiceClient.java:78) ~[aliyun-sdk-oss-2.2.1.jar:na]
	at com.aliyun.oss.OSSClient.(OSSClient.java:268) ~[aliyun-sdk-oss-2.2.1.jar:na]
	at com.aliyun.oss.OSSClient.(OSSClient.java:193) ~[aliyun-sdk-oss-2.2.1.jar:na]
	at com.hui.demo.test1Task.run(test1Task.java:26) ~[classes/:na]
	at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:800) [spring-boot-2.0.4.RELEASE.jar:2.0.4.RELEASE]
	at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:784) [spring-boot-2.0.4.RELEASE.jar:2.0.4.RELEASE]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:338) [spring-boot-2.0.4.RELEASE.jar:2.0.4.RELEASE]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1258) [spring-boot-2.0.4.RELEASE.jar:2.0.4.RELEASE]
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1246) [spring-boot-2.0.4.RELEASE.jar:2.0.4.RELEASE]
	at com.hui.Application.main(Application.java:13) [classes/:na]
Caused by: java.lang.ClassNotFoundException: org.apache.http.ssl.SSLContextBuilder
	at java.net.URLClassLoader.findClass(URLClassLoader.java:381) ~[na:1.8.0_131]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[na:1.8.0_131]
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[na:1.8.0_131]
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[na:1.8.0_131]
	... 11 common frames omitted
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

参考：
java 阿里云oss 图片，文件上传，获取文件夹 ，文件列表
java OSS批量下载，并压缩为ZIP
Java 批量下载OSS文件到本地为压缩文件

三、使用 Java 进行文件压缩和解压缩

  GZIPInputStream 类可以用于解压缩 GZIP 格式的文件。

3.1 解压缩 GZIP 文件

package com.xiaoqiang.demo;

import java.io.*;
import java.util.zip.*;

public class UnGZIPExample {

    public static void main(String[] args) {
        byte[] buffer = new byte[1024];

        try {
            GZIPInputStream gzipIS = new GZIPInputStream(new FileInputStream("D:\\xiaoqiang\\heheda-data-2023-11-15 11-0.log.gz"));

            String fileName = "D:\\xiaoqiang\\heheda-data-2023-11-15 11-0.log";
            FileOutputStream fos = new FileOutputStream(fileName);

            int length;
            while ((length = gzipIS.read(buffer)) > 0) {
                fos.write(buffer, 0, length);
            }

            gzipIS.close();
            fos.close();

        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

3.2 压缩 GZIP 文件

import java.io.*;
import java.util.zip.*;

public class GZIPExample {

  public static void main(String[] args) {
    byte[] buffer = new byte[1024];

    try {
      FileInputStream fis = new FileInputStream("D:\\test.txt");
      FileOutputStream fos = new FileOutputStream("D:\\output.gz");
      GZIPOutputStream gzipOS = new GZIPOutputStream(fos);

      int length;
      while ((length = fis.read(buffer)) > 0) {
        gzipOS.write(buffer, 0, length);
      }

      fis.close();
      gzipOS.finish();
      gzipOS.close();

    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

参考：
如何使用Java进行文件压缩和解压缩

遇到的问题：在压缩 GZIP 文件后发现里面的文件名和外面的不一致。


解决：java怎么gz压缩,但是里面文件名和外面不一直

遇到的问题：使用上面地址中的方法无法重写 writeHeader()，具体原因不明，我看 GZIPOutputStream 类中该方法是私有的，难道低版本的 Java 这块是公有的吗，等有时间再研究吧。