Harbor中镜像清理

背景

项目的每日构建中，由于微服务比较多，每天会产生比较多的镜像，导致Harbor服务器很快被占满。所以要定期去清理不需要的镜像来释放存储空间。

使用的Harbor版本是2.5.0

镜像清理

镜像清理可以手动在Harbor页面上操作。根据Harbor官方文档，当我们删除某个镜像时，存储空间不会被真正的释放，必须要手动去执行垃圾清理来释放不再被引用的blobs文件。

页面操作

1. 首先在页面删除不需要的镜像信息。

2. 然后在系统管理->垃圾清理，选择立即清理垃圾，手动GC。
   

3. 在立即清理右边有一个模拟运行，运行这个会打印出符合删除条件的blobs，并且会计算将会释放多少存储空间，不会真正的去删除文件，每次清理之前可以先执行一下模拟运行来防止误操作。

4. 清理操作触发之后，在历史记录，查看任务日志可以看到释放了多少的存储空间，完成清理。
   

定时清理工具

上面的手动清理不够灵活，我们可以根据Harbor提供的restful接口编写定时清理镜像的程序。

Harbor的接口需要提供一个basic认证的信息。我这边根据我们的业务场景编写了一个定时清理的程序，具体场景：每天定时扫描Harbor上我们CI构建的项目，只保留最近10次的构建推包镜像，其余的全部删除。

代码参考如下：

/**
 * 定时扫描harbor中的项目 只保留最近10次构建的镜像信息
 *
 * @author yuanzhihao
 * @since 2022/6/29
 */
@Slf4j
public class Main {
    private static final String USERNAME = "admin";
    private static final String PASSWORD = "Harbor12345";
    // 默认保存镜像的最大数量
    private static final int DEFAULT_REMAIN_ARTIFACT_COUNT = 10;
    // 需要清理的项目列表
    private static final List<String> PROJECTS = Arrays.asList("common", "library");
    private static final String GET_PROJECTS_INFO_URL = "http://192.168.1.103/api/v2.0/projects/%s";
    private static final String GET_REPOSITORIES_URL = "http://192.168.1.103/api/v2.0/projects/%s/repositories?page_size=%s";
    // 根据镜像push时间降序
    private static final String GET_ARTIFACTS_URL = "http://192.168.1.103/api/v2.0/projects/%s/repositories/%s/artifacts?page_size=%s&sort=-push_time";
    private static final String DELETE_ARTIFACTS_URL = "http://192.168.1.103/api/v2.0/projects/%s/repositories/%s/artifacts/%s";
    private static final String GC_URL = "http://192.168.1.103/api/v2.0/system/gc/schedule";


    public static void main(String[] args) {
        ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
        // 每天定时清理
        executorService.scheduleWithFixedDelay(new Runnable() {
            @SneakyThrows
            @Override
            public void run() {
                log.info("Garbage Collection Start");
                for (String project : PROJECTS) {
                    final int repoCount = getRepoCount(project);
                    final List<Repository> repositories = getRepositories(project, repoCount);
                    for (Repository repository : repositories) {
                        String repoName = repository.name.split("/")[1];
                        final List<String> artifacts = getArtifacts(project, repoName, repository.artifact_count);
                        // 只保留最近10次构建镜像
                        final int size = artifacts.size();
                        if (size > DEFAULT_REMAIN_ARTIFACT_COUNT) {
                            log.info("Project [{}] Repo [{}] Need GC", project, repoName);
                            for (int i = DEFAULT_REMAIN_ARTIFACT_COUNT; i < size; i++) {
                                deleteSpecificArtifact(project, repoName, artifacts.get(i));
                                log.info("Project [{}] Repo [{}] Artifact [{}] Delete Success", project, repoName, artifacts.get(i));
                            }
                        }
                    }
                }
                // 最后执行GC释放空间
                garbageCollect();
                log.info("Garbage Collection End");
            }
        }, 0, 1, TimeUnit.DAYS);
    }

    // 根据项目名称获取仓库数量
    private static int getRepoCount(String project) throws IOException {
        String url = String.format(GET_PROJECTS_INFO_URL, project);
        final String result = httpRequest(url, HttpMethod.GET, null);
        return JsonParser.parseString(result).getAsJsonObject().get("repo_count").getAsInt();
    }

    // 获取指定项目下所有的仓库列表
    private static List<Repository> getRepositories(String project, int size) throws IOException {
        String url = String.format(GET_REPOSITORIES_URL, project, size);
        final String result = httpRequest(url, HttpMethod.GET, null);
        return new Gson().fromJson(result, new TypeToken<List<Repository>>() {}.getType());
    }

    // 获取指定镜像仓库中的镜像列表 按照镜像push的时间排序
    private static List<String> getArtifacts(String project, String repository, int size) throws IOException {
        String url = String.format(GET_ARTIFACTS_URL, project, repository, size);
        final String result = httpRequest(url, HttpMethod.GET, null);
        List<String> artifactList = new ArrayList<>();
        JsonParser.parseString(result).getAsJsonArray().forEach(item -> artifactList.add(item.getAsJsonObject().get("digest").getAsString()));
        return artifactList;
    }

    // 根据ArtifactId删除镜像
    private static void deleteSpecificArtifact(String project, String repository, String artifactId) throws IOException {
        String url = String.format(DELETE_ARTIFACTS_URL, project, repository, artifactId);
        httpRequest(url, HttpMethod.DELETE, null);
    }

    // 触发gc
    private static void garbageCollect() throws IOException {
        String requestParameters = "{\"parameters\":{\"delete_untagged\":false,\"dry_run\":false},\"schedule\":{\"type\":\"Manual\"}}";
        httpRequest(GC_URL, HttpMethod.POST, requestParameters);
    }

    private static String httpRequest(String url, HttpMethod method, String requestBody) throws IOException {
        HttpRequestBase request = getRequest(url, method);
        request.addHeader("authorization", getAuthorization());
        if (StringUtils.isNotEmpty(requestBody)) {
            StringEntity stringEntity = new StringEntity(requestBody, "UTF-8");
            stringEntity.setContentType("application/json;charset=UTF-8");
            ((HttpEntityEnclosingRequestBase) request).setEntity(stringEntity);
        }
        try (CloseableHttpClient client = HttpClients.createDefault()) {
            try (CloseableHttpResponse response = client.execute(request)) {
                int statusCode = response.getStatusLine().getStatusCode();
                if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_CREATED) {
                    return EntityUtils.toString(response.getEntity(), "UTF-8");
                }
                log.error("Http Request error, HttpStatus is [{}]", statusCode);
            }
        }
        return "";
    }

    private static String getAuthorization() {
        byte[] encodeAuth = Base64.getEncoder().encode((USERNAME + ":" + PASSWORD).getBytes());
        return "Basic " + new String(encodeAuth);
    }

    private static HttpRequestBase getRequest(String url, HttpMethod method) {
        switch (method) {
            case GET:
                return new HttpGet(url);
            case POST:
                return new HttpPost(url);
            case DELETE:
                return new HttpDelete(url);
            default:
                throw new IllegalArgumentException("Unsupported method");
        }
    }

    // 仓库信息
    static class Repository {
        int artifact_count;
        String creation_time;
        int id;
        String name;
        int project_id;
        int pull_count;
        String update_time;
    }

    enum HttpMethod {
        GET, POST, DELETE
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

结语

参考：https://goharbor.io/docs/2.5.0/administration/garbage-collection

代码地址：https://github.com/yzh19961031/blogDemo/tree/master/harborImageClean