• 阿里云oss迁移到AWS S3


    0.项目背景

    公司迁移要求:从阿里云oss到亚马逊s3,数据量大概500G-2T左右。
    开启阿里云oss 加速模式,这样能够跨机房和区域加速。
    主要采用以下两种方式同步数据,根据每个公司情况自行选择即可。

    1.rclone 方式

    #s3 挂载
    s3fs ad-amazon-ad /ad-amazon-ad -o url=https://s3.us-east-1.amazonaws.com,endpoint=us-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    s3fs yinta-website /yinta-website -o url=https://s3.us-east-1.amazonaws.com,endpoint=us-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    s3fs erp-info-test /erp-info-test -o url=https://s3.ap-east-1.amazonaws.com,endpoint=ap-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    s3fs erp-info /erp-info -o url=https://s3.ap-east-1.amazonaws.com,endpoint=ap-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    s3fs erp-usinfo /erp-usinfo -o url=https://s3.us-east-1.amazonaws.com,endpoint=us-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    #rclone 配置信息
    [aws-s3]
    type = s3
    provider = AWS
    access_key_id = xxxxx
    secret_access_key = xxxxx
    region = ap-east-1
    endpoint = s3.ap-east-1.amazonaws.com.cn
    location_constraint = ap-east-1

    [ali-oss]
    type = s3
    provider = Alibaba
    access_key_id = xxxx
    secret_access_key = xxxxx
    endpoint = oss-accelerate.aliyuncs.com

    #传输命令

    nohup rclone sync -P --transfers 8 --checkers 8 --s3-chunk-size 8M --s3-upload-concurrency 8 --log-file=/tmp/rclone.txt --log-level INFO ali-oss:ad-amazon-ad aws-s3-us:ad-amazon-ad>/tmp/ad-amazon-ad.log &

    nohup rclone sync -P --transfers 8 --checkers 8 --s3-chunk-size 8M --s3-upload-concurrency 8 --log-file=/tmp/rclone.txt --log-level INFO ali-oss:yinta-website aws-s3-us:yinta-website>/tmp/yinta-website.log &

    nohup rclone sync -P --transfers 8 --checkers 8 --s3-chunk-size 8M --s3-upload-concurrency 8 --log-file=/tmp/rclone.txt --log-level INFO ali-oss:erp-info-test aws-s3:erp-info-test >/tmp/rc-test.log &

    nohup rclone sync -P --transfers 32 --checkers 16 --s3-chunk-size 15M --s3-upload-concurrency 16 --log-file=/tmp/rclone.txt --log-level INFO ali-oss:erp-usinfo aws-s3-us:erp-usinfo>/tmp/rc-us.log &

    nohup rclone sync -P --transfers 32 --checkers 16 --s3-chunk-size 15M --s3-upload-concurrency 16 --log-file=/tmp/rclone.txt --log-level INFO ali-oss:erp-info aws-s3:erp-info >/tmp/rc.log &

    #统计s3桶大小
    aws s3 ls --summarize --human-readable --recursive s3://erp-info

    s3cmd du s3://erp-info / --human-readable

    erp-info-test-us 美国西部(俄勒冈) us-west-2 公开 换成弗吉尼亚地区。

    参考:https://www.awstweaker.com/2022/02/16/rclone%e5%b7%a5%e5%85%b7%e4%bd%bf%e7%94%a8%e6%89%8b%e5%86%8c/

    2.rsync方式

    #1.挂载oss所需工具: ossfs
    echo erp-info:akxxx:skxxxx > /etc/passwd-ossfs
    chmod 600 /etc/passwd-ossfs

    #挂载oss
    ossfs erp-info /erp-info -ourl=oss-cn-hongkong-internal.aliyuncs.com

    #2.挂载S3:所需工具s3fs
    echo xxxxxx:xxxxxx/ > ~/.passwd-s3fs
    chmod 600 ~/.passwd-s3fs

    #挂载
    s3fs erp-info-test /erp-info-test -o url=https://s3.ap-east-1.amazonaws.com,endpoint=ap-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    s3fs bi-zyb /bi-zyb -o url=https://s3.ap-east-1.amazonaws.com,endpoint=ap-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    s3fs yinta-website /yinta-website -o url=https://s3.us-east-1.amazonaws.com,endpoint=us-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    s3fs ad-amazon-ad /ad-amazon-ad -o url=https://s3.us-east-1.amazonaws.com,endpoint=us-east-1,allow_other,uid=1000,gid=1000,mp_umask=007,use_cache=/tmp

    #sync
    [root@docker-yinta ~]# nohup rsync -av /erp-info-test/ root@16.163.255.57:/erp-info-test/ >/dev/null 2>&1 &
    [1] 21449
    [root@docker-yinta ~]# nohup rsync -av /erp-usinfo/ root@16.163.255.57:/erp-usinfo/ >/dev/null 2>&1 &
    [2] 21562
    [root@docker-yinta ~]# nohup rsync -av /erp-info/ root@16.163.255.57:/erp-info/ >/dev/null 2>&1 &
    [3] 21759

    3.注意

    oss中带/ 的目录,用rsync ,rclone ,oss等都无法同步其中文件!!

  • 相关阅读:
    LVS负载均衡
    QSS 自定义QLineEdit
    zookeeper详解
    2022年编程语言排名,官方数据来了,让人大开眼界
    Flutter 安装教程 + 运行教程
    python实现excel的数据提取
    逻辑运算符
    C++ 内存模型
    JVM内存结构
    微信小程序 java网上购物商城系统
  • 原文地址:https://blog.csdn.net/meijinmeng/article/details/134251609