aliyun-oss-go-sdk-1.5.0/000077500000000000000000000000001311621456000150015ustar00rootroot00000000000000aliyun-oss-go-sdk-1.5.0/.travis.yml000066400000000000000000000162211311621456000171140ustar00rootroot00000000000000language: go go: - 1.4 - 1.5 - 1.6 - 1.7 install: - go get golang.org/x/tools/cmd/cover - go get github.com/mattn/goveralls - go get gopkg.in/check.v1 script: - cd oss - travis_wait 30 go test -v -covermode=count -coverprofile=coverage.out -timeout=30m - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci env: global: - secure: CaUx0qMkOw3GhlGJmLQBcwiLmz7U4WoQeB0MtsLpGNat6JbgP/E3E+7ZiKv17rxxLpk1yNr4pScIwm7a1gv44qmjdOk506aBK/3exVkcntMU2q1i2IhWIiNUlr9g14CR5O3xB7yaErRy/p5ILbfw8wHYqPKRBfF03p7SUmTzWsiG1BKkjwKqQrGJadXIeL45EwPCIekpE/omDFMOQvYeE8jfvSJC3mcViE3LPaSzFa5FRO4xrBpMtvdBgbAW4xXyKbQ0xjLDOwlz7lT0CwOqrKQaf8+kB7qTuvViJdRBG9PJM9jFOIOQynzuUVI8RzCp0/+DC/fGmabzP62yCzguzasp78BkzTj0TxEutfxBPKimiONjiO8yVitqG4xEB9Qh6ya4i/6ktZXfQcSKFYw8AWCHKLqzC+pEc+Um6DeeLIOWJFEO1WhZ2L1orMoH1s445xVy35KZuJBUluWZyOL3Cgwd95RXhGZ/yhfCQwkIVscHSGfavkc2RL418jTjOB+0d/fES14jkPJsQ02E8ZAIZ6hI6QBbsorHLiiSMd2CwHZG2T0n4a1d6OKWFU5ftKVDQDKGLTaWYqStCjJCxiG7g4X3uvYzEdPBDctwNFT1HxeQ9UcjNs+2VLyLXpmJCuPDf3AK5FpoCOWVa99hYhFzbzHxMpPmPTC2MT7H8orxtno= - secure: F4pWK/B4irPi57wDQMKie1f8VaQBDjAFrvsX0of16+eaBaR/+DaqiRNuvYKTs1JLxgVvK3WOQNIwVcILibXr04/KrIrva44QYazfZ1eeR+TtEdy2dLzjVFHSGx/hKX0LSTXGRb2wOMsufSSGqfh8ngsr6LFp2RflGTCJ0HRyJ+p7BNL4R7rIR31STpqiV5Z2i26TMRjWDp177h5XHA1V0wOv/ZDMQUhRvdYH0uHDs76oVxZmZ3OG0EmSLSB7BuXoYT1aCkTpnZZXo39Rb5mjgObr3MWW7iCbaApNjscQ7IQ3OIUvW+RjsZ4BDVA2F88+HkC3gSnUa5Ka7gMFMAXtdC01Zxtpx8m2h18eHQOgpGcmjtU18QHDtv8AsoidgiMj9L7ZYmzo9ioXuZ9/UvBTARLJ/2+mI14VKCYintJPnWp3tXCyOrQIfr8nudpl380y0PJ+t7UMYm+PFEeUiC9MsMeFv2I6MZfx5CZBTIKr5XoIbuBC9BwFj5AIeuvXsw/qGy6gi/e/Ys5mLRTw5g9N+KFcZAsbnueoyNjlKaLP1nyrf3XJXMEuunRePpI4ABW4D+j3z2kW19f6XKNnHvBEEg2yCthk4dNouUKTL5fEv8IXXgtFoG/ADsdgDbWnOdIPmr7HcG1/VjJvhr7Yu9roSPH00J7pqABCDNebgEGxGcY= - secure: dcjezQ3DMMcyFwfG64GfQVwv4C2MjnJ6Sf4jdRac9XLDXd+VS5YbjuulzOdidHdvxcDI9x6XnNb8P9lErzR7URIcnHOF5p+7kwpXPBpd4E4DZV7GBYrbHBdQTnrYjs6gWCIzdWQxCT7wju6qSF3D0XtHNG8aM2qUCMCGfI0SBewbnDXpHtmHG/Ei/0DWfQeBW+pwV5OyAfZ2r0/H1ZVA6+z15eGW9u3gCN8E73U2ly19ZLQ8mu5lNHhFM2b2AdfzT9BDmCBmRp2vaqptivt1XVQ/ugCpWXtLdNtNSV51WIjbsLrmSRCb8yoQMVFAKsEC/zZU374ZZZocBhdJ2RXD46u1iW1hmivQgg+aNFQ6rQtByyhFGcpygRd063YvYWutfL6pa0il7PHLkaOfDC33G+GNcS+zfL/XnSbF/wH+Expv+OfrgRpcamBUGO3nvJFXekqzBiDWJWP4kJDazdUYT1kWIPdjxxI4zzjw4QXsIiOKy+7ihigkRe0VjzNJUJ8FLQFCvWXmNr67dKDjrLiM8NrzYep1u6lI0dCjOJhNWfMS3wt8rGIXqD+OfLQbFQiUBJYl1QEWEzZ1EeS/i+bGLDRW85+g/BNPc2V/o0V4TJy/I0yksvOHqFU5yqmMfTnlew28dQok7OZHGOFQpXyvpTLsbEHMvad8IaTx0MJ0yKE= - secure: GwuE+UMGD3NuU3f/6OPcRpT8iw8letEpfUeUIsw4C8LZsjNQAwR43oo3uCvNygjPew6+zpXqFIsggkcgFv8kC8rr6Jxq9rrxDCCQko3chKNRiB25lI10JaJ2VJxtrsHDmlC0LQgdq+RS4d8ejDCBTor1VtyCg7qqVawNYhMGv8QONs4CwiHf5ve/y9YRkNF6WH0F2iGsOJOCxcpVXo21KKCDA9M96zON2EmrElvKgfwo9OsIBaIilGqPR6lYCBvbN7KGV0wRqPTowGNfLtuOYkxBZRSkf/RJaFc10uEXEClkye5WcpumHgVfp9PoAg5Q+JYYgOInI0Gk7iyG77i9ObmkucccZ3OyFUUjRuCWBlQgzwBfKb3R8Or4VMGQ9n8bSPu50WLSRdizw2PWzFzU9p8+qFkTIJ6JCuMk8GnFiyayu8OjOxkKjKl5e5fQyynvbz397H7rnUqkl1y4mpeQI5FJz4p54cKDDuxPSqUvd15qQAhg5oDBgsWD3nj5fkI5C5tuFYBaPdgnnY7LiYqme7wu/g590HdPBfM0PCw5NquNheDuF4wn9BKlgxUUdGmCWAPmIpULIKlaPhP2VH29FXmR2oRATC3mOlorQnpXpvxaDfkNmCmMK5QcbqSRr2IJlsrApSlUhZI8FvIQwjG19MlbypGGaXxF8iNtoJfiSDE= - secure: ig/rYcJNIB0nWviO+fAsv83p6yMqcA7oHQfOawZxv0fJlBUOLDUzKgvqjQZoOCNZWT7eo6VESW3cKj1DMev1jpr7H6bzsSPG7CJ+BvsWmngLBy8kkJYXUf5UGV+CyFQmAPzdaotQ699Z8Mu/X1uYCvBoMwfvyFhuStCwZSusGt+iPjUqY5qyc4zM2yOXPVLK7YdkiGodC+MlEbgOk75+D0FSU0yHZ0kKiCcOtFxDjd2/yHjd1usSNp/9m2eE7QrSNSi+62RTvBSxDEZrjmBE1jYVeg0pBFHs4DrxcKGTiGTlALkDsak/acwlFZhKS97g1DnZC/jI883t2PcrJnORrww5tZNY87/kMB3BOjOTCTHNuWvBgHgmZ0VJ5ndu1FWwCiGZsBIJIZnzp/ZygcFvalcbqKA1WlegKtyT1x9J0zqS3is4Pfnc8yFC4GVoDu4H4iF+Cb8Uo5tZrQCXYEi7HXXn4JYkohwQw4JMX4ciI/o3hkmRP8sVKn5FdqGkpkXpJ7tITd50F7mNWejxbn2kSJmEHyQuG19MQ/Bf52D71AUpZiFxMqmk513UfilozkOVgvGjhtdD7hmIgtNQ2ttzz5NukXrTshTtrE/EzA5mYLsXnhC0EXB8cLZ1kpZs72ezYNud2XgidJI63GXd/1ZSFdjlDpmv4h6vWabkDzt/sm8= - secure: cn+C2FoHhBYw36yx2ObzQaeJjAEVr+cIwiFJAnRpMOusKL8v6vOQk0AGa+fJYXqCLoXJr4uliZytqos6ot4Kdkiw/XEq2fiIYi1ZG2oFyOANPjq00Rug6Ar5VpNu7XCaYcliA9bqRpGaQDDWINzyNWoI8iwmfZ58R/m27qEeoXDFNaiAZWIM3ZPvZTkKE+8FEepBTotBIRKyf7Rs7MKjWZqOxeg2cSRap6F8SvsC79CyPEu28NL+uRSxlu4EfgfvLEis6zuFxk68PqyQUo5BDiqDRYYnD9MS22tL3YV5Kkzm85bWWDdixect5u/LQFerGLg7l563dwfwn2XS8SMpm6H8S+TbKUQ9KYQb+uOERoCs0WQUTodRi74J9cGf7IoAOKn6ysceG23TBy9B6b8Umyx94sGko7/t2CE13FCC76VL2ONBlhP6err21Twi0KEr7hXRolVEp0sRGxDKwCeM/pVORv682wGh996Mp47NRzyfLnifCsIF4rD2ROVTeESy5UtTRxbSLvYkcS0SWbr9j4NHvF2qf3df/o1mRMsTa8ef+hZrOAKMEi44vZQbO2UQnLDv1njOQ3tdHbbY8NtS3eqpPqErl4mbYjM8Jry+UtV1JKxk7Yi00TiU803V+LLtm9+94A96tAKYMR5MfbsN8bF11Zz5Crk4zXGXF5jM5zw= - secure: EoTW6GuLT/WNnEAB7KF/Eb7Ne2DbMQ6auQaPwwrhIrpsgioq7MfOzmv+OE+wtiCUExR0CruvyhRSBvnXYgBjJ+AMyPqkC73ZwFKcQGhXQkuC+LLO7U7aFyHbX2UnMSXFIwjBMrdcIzAQgthAbQk4MTO0NcAxi9Mu9DrrBR5seFrzSWVEJuhEbLMTBpMHjFJNji37XcqQXKPiqdQMAYRUb5QBJ8mV8+39dxm/5IL8Rp6UOHUhZ0bmkO2ccBnoUlAtGOgWaTLPCQgidI++4yFSA1zJYp6Z6EYQHZgIUzdBtnPNCuw8PAhwQvcu+jWFNQaS/hDWDQ1ue27u/ZKTOYmxKJAwLmkG5Ega+D+sqXuih/03PMASL3O9b1sUVc1ilukFVRB2B7EuW+ujlyYaeed3b0aUUJI6Aphtd3tJoFd79zWcQ0XJNNvMFoRqGqKcE/MwIVjNyyP97m1KZ/PQVO4UiAhFnTxhojBvmk/3XX0kLgfUH4ujfvQXyB0G4fawCZZKE3AukZkUz2yd/C7bUyEMn2nnUe0PJHPG6XL9WUZZzKPSNUHDc19TQqDROYu+iB8AvoIwy/VqzGQjw+FE+kCXvpHa/9V91cURr1mtvA5wmNIRTI/HMxBRJ2khCm2/jidRw+Z2sh1hJIHBuT1DvBrb0OqqKJh4wDhlIkT6yGx5F2A= - secure: Wxp4SIJrGetnf1G1qvg5LLWPc2JDkqf2pQfXV+nZr76lyhkrerEDBcslzuAxYQMe3YCEgio5Ayk9owZjyIHjoO6Kef77cBL3QsXq51qYJT5GIhdaSGk5p7BCwy9lYaV90ljebEKSnpRO9zElh6grc3THlqUe4VScPmoMVeUP6RGRp15F+L8ecy+kEEbtMpks25htOQttqGZqvIn6uM64gdjFi/ZNZi2PN0FH0MFP2+440PvbAJpUl8Ev+tkaohiuWSv4gg1jxUO7QA0g+tNaCfstUadxxnDvAItTnFW3RzZU0Kpj3xtjf8TzaI2Q9mTiZ/GcN3RsUSF4OZefcc3M5T4C9iD5kYHP1WXvGC6Z8HOW4aHSZluD8g+l+I5kvCr1sQ/Ae/1k/dX1NlSZQr9UEaq2tsx/lNhI4y30Hp3OSC9wzeswcZSiA4P9G8LMoD2p3y18V+XCis8Nt8Eg7kjc0TqeMu2Ltx04ZtNB28faEPBNfrdSEFsPt3+H5fgRUDkfK7n7IW54i7AUnQx/UG+ewL+IoQLkDeXUxlytwvDYyZAKMf5ZSV15uiNHz/Kq6++TPcDpeO3SSoFAWlUj8xeQC2R6a9n4o03T/J0d/hTS7A0p7VRW0b/VIiZ7QPeo6JIosMUDkjwpon0otTio2MtH83tdVEKHDjMy7lkKJgSIQwU= - secure: a5Rkh6GjYQFILfnhVfX0ubdngpuCD2e0rXicD1+wzOBw46PoyWqzJshAFOJHS2C8IWfxe42KWD1uckzt1mxw2eETOM7r5Pp3DAeYchTWziHL3EEL1WG62wMdvxQRGSBz20N08DDfyK65g/2a+DQv8x8ZayZivg7wZc22fPOFEp8qDBetKs+LaEnAnxLBnuwhEKbGrXJ2kFBhKBLeAq7De759SFKBca8K5SVkRfgwqjnh/3i+oPBUketwztiXee5m36vFVjWoMxEF7t2kQO8I18Eew2syD/nOm6BzH06PyStsadoYcK381h7HUVUYNqMQk6l4mDFXNbNIrap8/spuOeVDsYI+O68Zh1mfnJfJDxFuiFzXVfvp+YxeT1aY2Yk4gstCWPUBUxHdLNuZrdMUO18boHXqQDiRKXHQl26pq7WTddNC/l3MTpufBN2EOQKqrUvKSb0YaEwit9Wq1IuIZsC0+UnqyhMTIhZIuD3HY/Mdm9FmwF0v8/a40I+aRIiqc6FSG20Warn2pKFNULRTGwgTn2JXdtsqNov+jD4gHLYLu1AHssHDZD8IcFXxylXF4VvF2Y030i22rUfP10kmfBqphFyGRs+vB3E01S02mlXnRXzRf95RzqXEzK1ezyb6vfZciZ966HPhybbbtoNBamvm/rWVHg5xNod5oeu1OI4= - secure: MK7zDaxpLwT+ItLDPMzc5SQOmtVrWAsc+cbCtPaYlFdXb8zKc4U3W8oIDNHajvwGPW+jB3vN0xarfiC7H9+q+I/JEViIgHGXwOt/P75tdIf7IkUHmOeEEbOHBsuV8roFOzDu4XkrYNkJG5qWxOvLtG+f4jW3bCZGL5FxdS2QIn5qBqy1v85QBrV1FKNKw+DOvHieGRul7MlGHAPCq5WWx/xyoxD1y1mDdTlFPduetcjOstsHd110Kr9SFUVu7jmvTU13S3mdkRGGN1FX3tWHiBLwhyIOOI0otuLfzq/9gsLtEoWH32/IKtMwH+Uu45gFn2ddP1cL4J42JxQcEJnE9e23E/XDuwYjV1NJBa+YgpOwUM98yBi+9AQ4+CQ+gLr+TQ6hbPyf9DqGSH0hXbY76+mnGZYSd33iVEjUdJe+82xnK/BXsqgAXXq2cIgNIjebUcgyiMHIt6s5ECzHRLcE4Hsj9lSRpEhv7fjvKq80mWsOoQa5tSoIEQx1pP/R8cyQCnX7eJBs/kbePg1Nwi3pC7WsCFxotCdxKajM7jXtdYBdHf2Fbuna3prIJEwP8lLJlBuQfwijq9DROvUovJN0wPPXuPOYX8O1LNY4wPsgX38PWzUCu2LQ6u0ztAWPio/7ZnxvbPysVMrMI2ENAadUotlDuGaSA7UhT0CAmbdexjo= aliyun-oss-go-sdk-1.5.0/CHANGELOG.md000066400000000000000000000030641311621456000166150ustar00rootroot00000000000000# ChangeLog - Aliyun OSS SDK for Go ## 版本号：1.4.0 日期：2017-05-23 ### 变更内容 - 增加：支持符号链接symlink - 增加：支持RestoreObject - 增加：CreateBucket支持StorageClass - 增加：支持范围读NormalizedRange - 修复：IsObjectExist使用GetObjectMeta实现 ## 版本号：1.3.0 日期：2017-01-13 ### 变更内容 - 增加：上传下载支持进度条功能 ## 版本号：1.2.3 日期：2016-12-28 ### 变更内容 - 修复：每次请求使用一个http.Client修改为共用http.Client ## 版本号：1.2.2 日期：2016-12-10 ### 变更内容 - 修复：GetObjectToFile/DownloadFile使用临时文件下载，成功后重命名成下载文件 - 修复：新建的下载文件权限修改为0664 ## 版本号：1.2.1 日期：2016-11-11 ### 变更内容 - 修复：只有当OSS返回x-oss-hash-crc64ecma头部时，才对上传的文件进行CRC64完整性校验 ## 版本号：1.2.0 日期：2016-10-18 ### 变更内容 - 支持CRC64校验 - 修复计算MD5占用内存大的问题 - 修复CopyObject时Object名称没有URL编码的问题 - 支持指定Useragent ## 版本号：1.1.0 日期：2016-08-09 ### 变更内容 - 支持代理服务器 ## 版本号：1.0.0 日期：2016-06-24 ### 变更内容 - 增加断点分片复制接口Bucket.CopyFile - 增加Bucket间复制接口Bucket.CopyObjectTo、Bucket.CopyObjectFrom - 增加Client.GetBucketInfo接口 - Bucket.UploadPartCopy支持Bucket间复制 - 修复断点上传、断点下载出错后，协程不退出的Bug - 去除接口Bucket.CopyObjectToBucket aliyun-oss-go-sdk-1.5.0/README-CN.md000066400000000000000000000117551311621456000165670ustar00rootroot00000000000000# Aliyun OSS SDK for Go [![GitHub version](https://badge.fury.io/gh/aliyun%2Faliyun-oss-go-sdk.svg)](https://badge.fury.io/gh/aliyun%2Faliyun-oss-go-sdk) [![Build Status](https://travis-ci.org/aliyun/aliyun-oss-go-sdk.svg?branch=master)](https://travis-ci.org/aliyun/aliyun-oss-go-sdk) [![Coverage Status](https://coveralls.io/repos/github/aliyun/aliyun-oss-go-sdk/badge.svg?branch=master)](https://coveralls.io/github/aliyun/aliyun-oss-go-sdk?branch=master) ## [README of English](https://github.com/aliyun/aliyun-oss-go-sdk/blob/master/README.md) ## 关于 > - 此Go SDK基于[阿里云对象存储服务](http://www.aliyun.com/product/oss/)官方API构建。 > - 阿里云对象存储（Object Storage Service，简称OSS），是阿里云对外提供的海量，安全，低成本，高可靠的云存储服务。 > - OSS适合存放任意文件类型，适合各种网站、开发企业及开发者使用。 > - 使用此SDK，用户可以方便地在任何应用、任何时间、任何地点上传，下载和管理数据。 ## 版本 > - 当前版本：1.4.0 ## 运行环境 > - 推荐使用Go 1.4及以上。 ## 安装方法 ### GitHub安装 > - 执行命令`go get github.com/aliyun/aliyun-oss-go-sdk/oss`获取远程代码包。 > - 在您的代码中使用`import "github.com/aliyun/aliyun-oss-go-sdk/oss"`引入OSS Go SDK的包。 ## 快速使用 #### 获取存储空间列表（List Bucket） ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } lsRes, err := client.ListBuckets() if err != nil { // HandleError(err) } for _, bucket := range lsRes.Buckets { fmt.Println("Buckets:", bucket.Name) } ``` #### 创建存储空间（Create Bucket） ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } err = client.CreateBucket("my-bucket") if err != nil { // HandleError(err) } ``` #### 删除存储空间（Delete Bucket） ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } err = client.DeleteBucket("my-bucket") if err != nil { // HandleError(err) } ``` #### 上传文件（Put Object） ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } bucket, err := client.Bucket("my-bucket") if err != nil { // HandleError(err) } err = bucket.PutObjectFromFile("my-object", "LocalFile") if err != nil { // HandleError(err) } ``` #### 下载文件 (Get Object) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } bucket, err := client.Bucket("my-bucket") if err != nil { // HandleError(err) } err = bucket.GetObjectToFile("my-object", "LocalFile") if err != nil { // HandleError(err) } ``` #### 获取文件列表（List Objects） ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } bucket, err := client.Bucket("my-bucket") if err != nil { // HandleError(err) } lsRes, err := bucket.ListObjects() if err != nil { // HandleError(err) } for _, object := range lsRes.Objects { fmt.Println("Objects:", object.Key) } ``` #### 删除文件(Delete Object) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } bucket, err := client.Bucket("my-bucket") if err != nil { // HandleError(err) } err = bucket.DeleteObject("my-object") if err != nil { // HandleError(err) } ``` #### 其它 更多的示例程序，请参看OSS Go SDK安装路径（即GOPATH变量中的第一个路径）下的`src\github.com\aliyun\aliyun-oss-go-sdk\sample`，该目录下为示例程序， 或者参看`https://github.com/aliyun/aliyun-oss-go-sdk`下sample目录中的示例文件。 ## 注意事项 ### 运行sample > - 拷贝示例文件。到OSS Go SDK的安装路径（即GOPATH变量中的第一个路径），进入OSS Go SDK的代码目录`src\github.com\aliyun\aliyun-oss-go-sdk`， 把其下的sample目录和sample.go复制到您的测试工程src目录下。 > - 修改sample/config.go里的endpoint、AccessKeyId、AccessKeySecret、BucketName等配置。 > - 请在您的工程目录下执行`go run src/sample.go`。 ## 联系我们 > - [阿里云OSS官方网站](http://oss.aliyun.com) > - [阿里云OSS官方论坛](http://bbs.aliyun.com) > - [阿里云OSS官方文档中心](http://www.aliyun.com/product/oss#Docs) > - 阿里云官方技术支持：[提交工单](https://workorder.console.aliyun.com/#/ticket/createIndex) ## 作者 > - Yubin Bai > - Hǎiliàng Wáng ## License > - Apache License 2.0 aliyun-oss-go-sdk-1.5.0/README.md000066400000000000000000000123401311621456000162600ustar00rootroot00000000000000# Alibaba Cloud OSS SDK for Go [![GitHub Version](https://badge.fury.io/gh/aliyun%2Faliyun-oss-go-sdk.svg)](https://badge.fury.io/gh/aliyun%2Faliyun-oss-go-sdk) [![Build Status](https://travis-ci.org/aliyun/aliyun-oss-go-sdk.svg?branch=master)](https://travis-ci.org/aliyun/aliyun-oss-go-sdk) [![Coverage Status](https://coveralls.io/repos/github/aliyun/aliyun-oss-go-sdk/badge.svg?branch=master)](https://coveralls.io/github/aliyun/aliyun-oss-go-sdk?branch=master) ## [README of Chinese](https://github.com/aliyun/aliyun-oss-go-sdk/blob/master/README-CN.md) ## About > - This Go SDK is based on the official APIs of [Alibaba Cloud OSS](http://www.aliyun.com/product/oss/). > - Alibaba Cloud Object Storage Service (OSS) is a cloud storage service provided by Alibaba Cloud, featuring massive capacity, security, a low cost, and high reliability. > - The OSS can store any type of files and therefore applies to various websites, development enterprises and developers. > - With this SDK, you can upload, download and manage data on any app anytime and anywhere conveniently. ## Version > - Current version: 1.4.0. ## Run environment > - Go 1.4 or above is recommended. ## Install OSS Go SDK ### Install the SDK through GitHub > - Run the 'go get github.com/aliyun/aliyun-oss-go-sdk/oss' command to get the remote code package. > - Use 'import "github.com/aliyun/aliyun-oss-go-sdk/oss"' in your code to introduce OSS Go SDK package. ## Quick use #### Get the bucket list (List Bucket) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } lsRes, err := client.ListBuckets() if err != nil { // HandleError(err) } for _, bucket := range lsRes.Buckets { fmt.Println("Buckets:", bucket.Name) } ``` #### Create a bucket (Create Bucket) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } err = client.CreateBucket("my-bucket") if err != nil { // HandleError(err) } ``` #### Delete a bucket (Delete Bucket) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } err = client.DeleteBucket("my-bucket") if err != nil { // HandleError(err) } ``` #### Upload a file (Put Object) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } bucket, err := client.Bucket("my-bucket") if err != nil { // HandleError(err) } err = bucket.PutObjectFromFile("my-object", "LocalFile") if err != nil { // HandleError(err) } ``` #### Download an object (Get Object) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } bucket, err := client.Bucket("my-bucket") if err != nil { // HandleError(err) } err = bucket.GetObjectToFile("my-object", "LocalFile") if err != nil { // HandleError(err) } ``` #### Get the object list (List Objects) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } bucket, err := client.Bucket("my-bucket") if err != nil { // HandleError(err) } lsRes, err := bucket.ListObjects() if err != nil { // HandleError(err) } for _, object := range lsRes.Objects { fmt.Println("Objects:", object.Key) } ``` #### Delete an object (Delete Object) ```go client, err := oss.New("Endpoint", "AccessKeyId", "AccessKeySecret") if err != nil { // HandleError(err) } bucket, err := client.Bucket("my-bucket") if err != nil { // HandleError(err) } err = bucket.DeleteObject("my-object") if err != nil { // HandleError(err) } ``` #### Others More example projects can be found at 'src\github.com\aliyun\aliyun-oss-go-sdk\sample' under the installation path of the OSS Go SDK (the first path of the GOPATH variable). The directory contains example projects. Or you can refer to the example objects in the sample directory under 'https://github.com/aliyun/aliyun-oss-go-sdk'. ## Notes ### Run a sample project > - Copy the example file. Go to the installation path of OSS Go SDK (the first path of the GOPATH variable), enter the code directory of the OSS Go SDK, namely 'src\github.com\aliyun\aliyun-oss-go-sdk', and copy the sample directory and sample.go to the src directory of your test project. > - Modify the endpoint, AccessKeyId, AccessKeySecret and BucketName configuration settings in sample/config.go. > - Run 'go run src/sample.go' under your project directory. ## Contact us > - [Alibaba Cloud OSS official website](http://oss.aliyun.com). > - [Alibaba Cloud OSS official forum](http://bbs.aliyun.com). > - [Alibaba Cloud OSS official documentation center](http://www.aliyun.com/product/oss#Docs). > - Alibaba Cloud official technical support: [Submit a ticket](https://workorder.console.aliyun.com/#/ticket/createIndex). ## Author > - Yubin Bai. > - Hǎiliàng Wáng. ## License > - Apache License 2.0. aliyun-oss-go-sdk-1.5.0/oss/000077500000000000000000000000001311621456000156055ustar00rootroot00000000000000aliyun-oss-go-sdk-1.5.0/oss/auth.go000066400000000000000000000045661311621456000171100ustar00rootroot00000000000000package oss import ( "bytes" "crypto/hmac" "crypto/sha1" "encoding/base64" "hash" "io" "net/http" "sort" "strings" ) // 用于signHeader的字典排序存放容器。 type headerSorter struct { Keys []string Vals []string } // 生成签名方法（直接设置请求的Header）。 func (conn Conn) signHeader(req *http.Request, canonicalizedResource string) { // Find out the "x-oss-"'s address in this request'header temp := make(map[string]string) for k, v := range req.Header { if strings.HasPrefix(strings.ToLower(k), "x-oss-") { temp[strings.ToLower(k)] = v[0] } } hs := newHeaderSorter(temp) // Sort the temp by the Ascending Order hs.Sort() // Get the CanonicalizedOSSHeaders canonicalizedOSSHeaders := "" for i := range hs.Keys { canonicalizedOSSHeaders += hs.Keys[i] + ":" + hs.Vals[i] + "\n" } // Give other parameters values date := req.Header.Get(HTTPHeaderDate) contentType := req.Header.Get(HTTPHeaderContentType) contentMd5 := req.Header.Get(HTTPHeaderContentMD5) signStr := req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + canonicalizedResource h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(conn.config.AccessKeySecret)) io.WriteString(h, signStr) signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil)) // Get the final Authorization' string authorizationStr := "OSS " + conn.config.AccessKeyID + ":" + signedStr // Give the parameter "Authorization" value req.Header.Set(HTTPHeaderAuthorization, authorizationStr) } // Additional function for function SignHeader. func newHeaderSorter(m map[string]string) *headerSorter { hs := &headerSorter{ Keys: make([]string, 0, len(m)), Vals: make([]string, 0, len(m)), } for k, v := range m { hs.Keys = append(hs.Keys, k) hs.Vals = append(hs.Vals, v) } return hs } // Additional function for function SignHeader. func (hs *headerSorter) Sort() { sort.Sort(hs) } // Additional function for function SignHeader. func (hs *headerSorter) Len() int { return len(hs.Vals) } // Additional function for function SignHeader. func (hs *headerSorter) Less(i, j int) bool { return bytes.Compare([]byte(hs.Keys[i]), []byte(hs.Keys[j])) < 0 } // Additional function for function SignHeader. func (hs *headerSorter) Swap(i, j int) { hs.Vals[i], hs.Vals[j] = hs.Vals[j], hs.Vals[i] hs.Keys[i], hs.Keys[j] = hs.Keys[j], hs.Keys[i] } aliyun-oss-go-sdk-1.5.0/oss/bucket.go000066400000000000000000000546771311621456000174340ustar00rootroot00000000000000package oss import ( "bytes" "crypto/md5" "encoding/base64" "encoding/xml" "hash" "hash/crc64" "io" "io/ioutil" "net/http" "net/url" "os" "strconv" ) // Bucket implements the operations of object. type Bucket struct { Client Client BucketName string } // // PutObject 新建Object，如果Object已存在，覆盖原有Object。 // // objectKey 上传对象的名称，使用UTF-8编码、长度必须在1-1023字节之间、不能以“/”或者“\”字符开头。 // reader io.Reader读取object的数据。 // options 上传对象时可以指定对象的属性，可用选项有CacheControl、ContentDisposition、ContentEncoding、 // Expires、ServerSideEncryption、ObjectACL、Meta，具体含义请参看 // https://help.aliyun.com/document_detail/oss/api-reference/object/PutObject.html // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) PutObject(objectKey string, reader io.Reader, options ...Option) error { opts := addContentType(options, objectKey) request := &PutObjectRequest{ ObjectKey: objectKey, Reader: reader, } resp, err := bucket.DoPutObject(request, opts) if err != nil { return err } defer resp.Body.Close() return err } // // PutObjectFromFile 新建Object，内容从本地文件中读取。 // // objectKey 上传对象的名称。 // filePath 本地文件，上传对象的值为该文件内容。 // options 上传对象时可以指定对象的属性。详见PutObject的options。 // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) PutObjectFromFile(objectKey, filePath string, options ...Option) error { fd, err := os.Open(filePath) if err != nil { return err } defer fd.Close() opts := addContentType(options, filePath, objectKey) request := &PutObjectRequest{ ObjectKey: objectKey, Reader: fd, } resp, err := bucket.DoPutObject(request, opts) if err != nil { return err } defer resp.Body.Close() return err } // // DoPutObject 上传文件。 // // request 上传请求。 // options 上传选项。 // // Response 上传请求返回值。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) DoPutObject(request *PutObjectRequest, options []Option) (*Response, error) { isOptSet, _, _ := isOptionSet(options, HTTPHeaderContentType) if !isOptSet { options = addContentType(options, request.ObjectKey) } listener := getProgressListener(options) resp, err := bucket.do("PUT", request.ObjectKey, "", "", options, request.Reader, listener) if err != nil { return nil, err } if bucket.getConfig().IsEnableCRC { err = checkCRC(resp, "DoPutObject") if err != nil { return resp, err } } err = checkRespCode(resp.StatusCode, []int{http.StatusOK}) return resp, err } // // GetObject 下载文件。 // // objectKey 下载的文件名称。 // options 对象的属性限制项，可选值有Range、IfModifiedSince、IfUnmodifiedSince、IfMatch、 // IfNoneMatch、AcceptEncoding，详细请参考 // https://help.aliyun.com/document_detail/oss/api-reference/object/GetObject.html // // io.ReadCloser reader，读取数据后需要close。error为nil时有效。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) GetObject(objectKey string, options ...Option) (io.ReadCloser, error) { result, err := bucket.DoGetObject(&GetObjectRequest{objectKey}, options) if err != nil { return nil, err } return result.Response.Body, nil } // // GetObjectToFile 下载文件。 // // objectKey 下载的文件名称。 // filePath 下载对象的内容写到该本地文件。 // options 对象的属性限制项。详见GetObject的options。 // // error 操作无错误时返回error为nil，非nil为错误说明。 // func (bucket Bucket) GetObjectToFile(objectKey, filePath string, options ...Option) error { tempFilePath := filePath + TempFileSuffix // 读取Object内容 result, err := bucket.DoGetObject(&GetObjectRequest{objectKey}, options) if err != nil { return err } defer result.Response.Body.Close() // 如果文件不存在则创建，存在则清空 fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, FilePermMode) if err != nil { return err } // 存储数据到文件 _, err = io.Copy(fd, result.Response.Body) fd.Close() if err != nil { return err } // 比较CRC值 hasRange, _, _ := isOptionSet(options, HTTPHeaderRange) if bucket.getConfig().IsEnableCRC && !hasRange { result.Response.ClientCRC = result.ClientCRC.Sum64() err = checkCRC(result.Response, "GetObjectToFile") if err != nil { os.Remove(tempFilePath) return err } } return os.Rename(tempFilePath, filePath) } // // DoGetObject 下载文件 // // request 下载请求 // options 对象的属性限制项。详见GetObject的options。 // // GetObjectResult 下载请求返回值。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) DoGetObject(request *GetObjectRequest, options []Option) (*GetObjectResult, error) { resp, err := bucket.do("GET", request.ObjectKey, "", "", options, nil, nil) if err != nil { return nil, err } result := &GetObjectResult{ Response: resp, } // crc var crcCalc hash.Hash64 hasRange, _, _ := isOptionSet(options, HTTPHeaderRange) if bucket.getConfig().IsEnableCRC && !hasRange { crcCalc = crc64.New(crcTable()) result.ServerCRC = resp.ServerCRC result.ClientCRC = crcCalc } // progress listener := getProgressListener(options) contentLen, _ := strconv.ParseInt(resp.Headers.Get(HTTPHeaderContentLength), 10, 64) resp.Body = ioutil.NopCloser(TeeReader(resp.Body, crcCalc, contentLen, listener, nil)) return result, nil } // // CopyObject 同一个bucket内拷贝Object。 // // srcObjectKey Copy的源对象。 // destObjectKey Copy的目标对象。 // options Copy对象时，您可以指定源对象的限制条件，满足限制条件时copy，不满足时返回错误，您可以选择如下选项CopySourceIfMatch、 // CopySourceIfNoneMatch、CopySourceIfModifiedSince、CopySourceIfUnmodifiedSince、MetadataDirective。 // Copy对象时，您可以指定目标对象的属性，如CacheControl、ContentDisposition、ContentEncoding、Expires、 // ServerSideEncryption、ObjectACL、Meta，选项的含义请参看 // https://help.aliyun.com/document_detail/oss/api-reference/object/CopyObject.html // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) CopyObject(srcObjectKey, destObjectKey string, options ...Option) (CopyObjectResult, error) { var out CopyObjectResult options = append(options, CopySource(bucket.BucketName, url.QueryEscape(srcObjectKey))) resp, err := bucket.do("PUT", destObjectKey, "", "", options, nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // CopyObjectTo bucket间拷贝object。 // // srcObjectKey 源Object名称。源Bucket名称为Bucket.BucketName。 // destBucketName 目标Bucket名称。 // destObjectKey 目标Object名称。 // options Copy选项，详见CopyObject的options。 // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) CopyObjectTo(destBucketName, destObjectKey, srcObjectKey string, options ...Option) (CopyObjectResult, error) { return bucket.copy(srcObjectKey, destBucketName, destObjectKey, options...) } // // CopyObjectFrom bucket间拷贝object。 // // srcBucketName 源Bucket名称。 // srcObjectKey 源Object名称。 // destObjectKey 目标Object名称。目标Bucket名称为Bucket.BucketName。 // options Copy选项，详见CopyObject的options。 // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) CopyObjectFrom(srcBucketName, srcObjectKey, destObjectKey string, options ...Option) (CopyObjectResult, error) { destBucketName := bucket.BucketName var out CopyObjectResult srcBucket, err := bucket.Client.Bucket(srcBucketName) if err != nil { return out, err } return srcBucket.copy(srcObjectKey, destBucketName, destObjectKey, options...) } func (bucket Bucket) copy(srcObjectKey, destBucketName, destObjectKey string, options ...Option) (CopyObjectResult, error) { var out CopyObjectResult options = append(options, CopySource(bucket.BucketName, url.QueryEscape(srcObjectKey))) headers := make(map[string]string) err := handleOptions(headers, options) if err != nil { return out, err } resp, err := bucket.Client.Conn.Do("PUT", destBucketName, destObjectKey, "", "", headers, nil, 0, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // AppendObject 追加方式上传。 // // AppendObject参数必须包含position，其值指定从何处进行追加。首次追加操作的position必须为0， // 后续追加操作的position是Object的当前长度。例如，第一次Append Object请求指定position值为0， // content-length是65536；那么，第二次Append Object需要指定position为65536。 // 每次操作成功后，响应头部x-oss-next-append-position也会标明下一次追加的position。 // // objectKey 需要追加的Object。 // reader io.Reader，读取追的内容。 // appendPosition object追加的起始位置。 // destObjectProperties 第一次追加时指定新对象的属性，如CacheControl、ContentDisposition、ContentEncoding、 // Expires、ServerSideEncryption、ObjectACL。 // // int64 下次追加的开始位置，error为nil空时有效。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) AppendObject(objectKey string, reader io.Reader, appendPosition int64, options ...Option) (int64, error) { request := &AppendObjectRequest{ ObjectKey: objectKey, Reader: reader, Position: appendPosition, } result, err := bucket.DoAppendObject(request, options) return result.NextPosition, err } // // DoAppendObject 追加上传。 // // request 追加上传请求。 // options 追加上传选项。 // // AppendObjectResult 追加上传请求返回值。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) DoAppendObject(request *AppendObjectRequest, options []Option) (*AppendObjectResult, error) { params := "append&position=" + strconv.FormatInt(request.Position, 10) headers := make(map[string]string) opts := addContentType(options, request.ObjectKey) handleOptions(headers, opts) var initCRC uint64 isCRCSet, initCRCOpt, _ := isOptionSet(options, initCRC64) if isCRCSet { initCRC = initCRCOpt.(uint64) } listener := getProgressListener(options) handleOptions(headers, opts) resp, err := bucket.Client.Conn.Do("POST", bucket.BucketName, request.ObjectKey, params, params, headers, request.Reader, initCRC, listener) if err != nil { return nil, err } defer resp.Body.Close() nextPosition, _ := strconv.ParseInt(resp.Headers.Get(HTTPHeaderOssNextAppendPosition), 10, 64) result := &AppendObjectResult{ NextPosition: nextPosition, CRC: resp.ServerCRC, } if bucket.getConfig().IsEnableCRC && isCRCSet { err = checkCRC(resp, "AppendObject") if err != nil { return result, err } } return result, nil } // // DeleteObject 删除Object。 // // objectKey 待删除Object。 // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) DeleteObject(objectKey string) error { resp, err := bucket.do("DELETE", objectKey, "", "", nil, nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusNoContent}) } // // DeleteObjects 批量删除object。 // // objectKeys 待删除object类表。 // options 删除选项，DeleteObjectsQuiet，是否是安静模式，默认不使用。 // // DeleteObjectsResult 非安静模式的的返回值。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) DeleteObjects(objectKeys []string, options ...Option) (DeleteObjectsResult, error) { out := DeleteObjectsResult{} dxml := deleteXML{} for _, key := range objectKeys { dxml.Objects = append(dxml.Objects, DeleteObject{Key: key}) } isQuiet, _ := findOption(options, deleteObjectsQuiet, false) dxml.Quiet = isQuiet.(bool) encode := "&encoding-type=url" bs, err := xml.Marshal(dxml) if err != nil { return out, err } buffer := new(bytes.Buffer) buffer.Write(bs) contentType := http.DetectContentType(buffer.Bytes()) options = append(options, ContentType(contentType)) sum := md5.Sum(bs) b64 := base64.StdEncoding.EncodeToString(sum[:]) options = append(options, ContentMD5(b64)) resp, err := bucket.do("POST", "", "delete"+encode, "delete", options, buffer, nil) if err != nil { return out, err } defer resp.Body.Close() if !dxml.Quiet { if err = xmlUnmarshal(resp.Body, &out); err == nil { err = decodeDeleteObjectsResult(&out) } } return out, err } // // IsObjectExist object是否存在。 // // bool object是否存在，true存在，false不存在。error为nil时有效。 // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) IsObjectExist(objectKey string) (bool, error) { _, err := bucket.GetObjectMeta(objectKey) if err == nil { return true, nil } switch err.(type) { case ServiceError: if err.(ServiceError).StatusCode == 404 && err.(ServiceError).Code == "NoSuchKey" { return false, nil } } return false, err } // // ListObjects 获得Bucket下筛选后所有的object的列表。 // // options ListObject的筛选行为。Prefix指定的前缀、MaxKeys最大数目、Marker第一个开始、Delimiter对Object名字进行分组的字符。 // // 您有如下8个object，my-object-1, my-object-11, my-object-2, my-object-21, // my-object-22, my-object-3, my-object-31, my-object-32。如果您指定了Prefix为my-object-2, // 则返回my-object-2, my-object-21, my-object-22三个object。如果您指定了Marker为my-object-22， // 则返回my-object-3, my-object-31, my-object-32三个object。如果您指定MaxKeys则每次最多返回MaxKeys个， // 最后一次可能不足。这三个参数可以组合使用，实现分页等功能。如果把prefix设为某个文件夹名，就可以罗列以此prefix开头的文件， // 即该文件夹下递归的所有的文件和子文件夹。如果再把delimiter设置为"/"时，返回值就只罗列该文件夹下的文件，该文件夹下的子文件名 // 返回在CommonPrefixes部分，子文件夹下递归的文件和文件夹不被显示。例如一个bucket存在三个object，fun/test.jpg、 // fun/movie/001.avi、fun/movie/007.avi。若设定prefix为"fun/"，则返回三个object；如果增加设定 // delimiter为"/"，则返回文件"fun/test.jpg"和前缀"fun/movie/"，即实现了文件夹的逻辑。 // // 常用场景，请参数示例sample/list_object.go。 // // ListObjectsResponse 操作成功后的返回值，成员Objects为bucket中对象列表。error为nil时该返回值有效。 // func (bucket Bucket) ListObjects(options ...Option) (ListObjectsResult, error) { var out ListObjectsResult options = append(options, EncodingType("url")) params, err := handleParams(options) if err != nil { return out, err } resp, err := bucket.do("GET", "", params, "", nil, nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) if err != nil { return out, err } err = decodeListObjectsResult(&out) return out, err } // // SetObjectMeta 设置Object的Meta。 // // objectKey object // options 指定对象的属性，有以下可选项CacheControl、ContentDisposition、ContentEncoding、Expires、 // ServerSideEncryption、Meta。 // // error 操作无错误时error为nil，非nil为错误信息。 // func (bucket Bucket) SetObjectMeta(objectKey string, options ...Option) error { options = append(options, MetadataDirective(MetaReplace)) _, err := bucket.CopyObject(objectKey, objectKey, options...) return err } // // GetObjectDetailedMeta 查询Object的头信息。 // // objectKey object名称。 // objectPropertyConstraints 对象的属性限制项，满足时正常返回，不满足时返回错误。现在项有IfModifiedSince、IfUnmodifiedSince、 // IfMatch、IfNoneMatch。具体含义请参看 https://help.aliyun.com/document_detail/oss/api-reference/object/HeadObject.html // // http.Header 对象的meta，error为nil时有效。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) GetObjectDetailedMeta(objectKey string, options ...Option) (http.Header, error) { resp, err := bucket.do("HEAD", objectKey, "", "", options, nil, nil) if err != nil { return nil, err } defer resp.Body.Close() return resp.Headers, nil } // // GetObjectMeta 查询Object的头信息。 // // GetObjectMeta相比GetObjectDetailedMeta更轻量，仅返回指定Object的少量基本meta信息， // 包括该Object的ETag、Size（对象大小）、LastModified，其中Size由响应头Content-Length的数值表示。 // // objectKey object名称。 // // http.Header 对象的meta，error为nil时有效。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) GetObjectMeta(objectKey string) (http.Header, error) { resp, err := bucket.do("GET", objectKey, "?objectMeta", "", nil, nil, nil) if err != nil { return nil, err } defer resp.Body.Close() return resp.Headers, nil } // // SetObjectACL 修改Object的ACL权限。 // // 只有Bucket Owner才有权限调用PutObjectACL来修改Object的ACL。Object ACL优先级高于Bucket ACL。 // 例如Bucket ACL是private的，而Object ACL是public-read-write的，则访问这个Object时， // 先判断Object的ACL，所以所有用户都拥有这个Object的访问权限，即使这个Bucket是private bucket。 // 如果某个Object从来没设置过ACL，则访问权限遵循Bucket ACL。 // // Object的读操作包括GetObject，HeadObject，CopyObject和UploadPartCopy中的对source object的读； // Object的写操作包括：PutObject，PostObject，AppendObject，DeleteObject， // DeleteMultipleObjects，CompleteMultipartUpload以及CopyObject对新的Object的写。 // // objectKey 设置权限的object。 // objectAcl 对象权限。可选值PrivateACL(私有读写)、PublicReadACL(公共读私有写)、PublicReadWriteACL(公共读写)。 // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) SetObjectACL(objectKey string, objectACL ACLType) error { options := []Option{ObjectACL(objectACL)} resp, err := bucket.do("PUT", objectKey, "acl", "acl", options, nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // GetObjectACL 获取对象的ACL权限。 // // objectKey 获取权限的object。 // // GetObjectAclResponse 获取权限操作返回值，error为nil时有效。GetObjectAclResponse.Acl为对象的权限。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) GetObjectACL(objectKey string) (GetObjectACLResult, error) { var out GetObjectACLResult resp, err := bucket.do("GET", objectKey, "acl", "acl", nil, nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // PutSymlink 创建符号链接。 // // 符号链接的目标文件类型不能为符号链接。 // 创建符号链接时: 不检查目标文件是否存在, 不检查目标文件类型是否合法, 不检查目标文件是否有权限访问。 // 以上检查，都推迟到GetObject等需要访问目标文件的API。 // 如果试图添加的文件已经存在，并且有访问权限。新添加的文件将覆盖原来的文件。 // 如果在PutSymlink的时候，携带以x-oss-meta-为前缀的参数，则视为user meta。 // // symObjectKey 要创建的符号链接文件。 // targetObjectKey 目标文件。 // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) PutSymlink(symObjectKey string, targetObjectKey string, options ...Option) error { options = append(options, symlinkTarget(url.QueryEscape(targetObjectKey))) resp, err := bucket.do("PUT", symObjectKey, "symlink", "symlink", options, nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // GetSymlink 获取符号链接的目标文件。 // 如果符号链接不存在返回404。 // // objectKey 获取目标文件的符号链接object。 // // error 操作无错误为nil，非nil为错误信息。当error为nil时，返回的string为目标文件，否则该值无效。 // func (bucket Bucket) GetSymlink(objectKey string) (http.Header, error) { resp, err := bucket.do("GET", objectKey, "symlink", "symlink", nil, nil, nil) if err != nil { return nil, err } defer resp.Body.Close() targetObjectKey := resp.Headers.Get(HTTPHeaderOssSymlinkTarget) targetObjectKey, err = url.QueryUnescape(targetObjectKey) if err != nil { return resp.Headers, err } resp.Headers.Set(HTTPHeaderOssSymlinkTarget, targetObjectKey) return resp.Headers, err } // // RestoreObject 恢复处于冷冻状态的归档类型Object进入读就绪状态。 // // 如果是针对该Object第一次调用restore接口，则返回成功。 // 如果已经成功调用过restore接口，且restore没有完全完成，再次调用时返回409，错误码：RestoreAlreadyInProgress。 // 如果已经成功调用过restore接口，且restore已经完成，再次调用时返回成功，且会将object的可下载时间延长一天，最多延长7天。 // // objectKey 需要恢复状态的object名称。 // // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) RestoreObject(objectKey string) error { resp, err := bucket.do("POST", objectKey, "restore", "restore", nil, nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK, http.StatusAccepted}) } // Private func (bucket Bucket) do(method, objectName, urlParams, subResource string, options []Option, data io.Reader, listener ProgressListener) (*Response, error) { headers := make(map[string]string) err := handleOptions(headers, options) if err != nil { return nil, err } return bucket.Client.Conn.Do(method, bucket.BucketName, objectName, urlParams, subResource, headers, data, 0, listener) } func (bucket Bucket) getConfig() *Config { return bucket.Client.Config } func addContentType(options []Option, keys ...string) []Option { typ := TypeByExtension("") for _, key := range keys { typ = TypeByExtension(key) if typ != "" { break } } if typ == "" { typ = "application/octet-stream" } opts := []Option{ContentType(typ)} opts = append(opts, options...) return opts } aliyun-oss-go-sdk-1.5.0/oss/bucket_test.go000066400000000000000000001453231311621456000204600ustar00rootroot00000000000000// bucket test package oss import ( "bytes" "encoding/json" "errors" "fmt" "io" "io/ioutil" "math/rand" "net/http" "os" "path/filepath" "strconv" "strings" "time" . "gopkg.in/check.v1" ) type OssBucketSuite struct { client *Client bucket *Bucket archiveBucket *Bucket } var _ = Suite(&OssBucketSuite{}) var ( pastDate = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) futureDate = time.Date(2049, time.January, 10, 23, 0, 0, 0, time.UTC) ) // Run once when the suite starts running func (s *OssBucketSuite) SetUpSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) s.client = client s.client.CreateBucket(bucketName) err = s.client.CreateBucket(archiveBucketName, StorageClass(StorageArchive)) c.Assert(err, IsNil) time.Sleep(5 * time.Second) bucket, err := s.client.Bucket(bucketName) c.Assert(err, IsNil) s.bucket = bucket archiveBucket, err := s.client.Bucket(archiveBucketName) c.Assert(err, IsNil) s.archiveBucket = archiveBucket testLogger.Println("test bucket started") } // Run before each test or benchmark starts running func (s *OssBucketSuite) TearDownSuite(c *C) { for _, bucket := range []*Bucket{s.bucket, s.archiveBucket} { // Delete Multipart lmu, err := bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmu.Uploads { imur := InitiateMultipartUploadResult{Bucket: bucketName, Key: upload.Key, UploadID: upload.UploadID} err = bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } // Delete Objects lor, err := bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } } testLogger.Println("test bucket completed") } // Run after each test or benchmark runs func (s *OssBucketSuite) SetUpTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // Run once after all tests or benchmarks have finished running func (s *OssBucketSuite) TearDownTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".txt") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".temp") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".txt1") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".txt2") c.Assert(err, IsNil) } // TestPutObject func (s *OssBucketSuite) TestPutObject(c *C) { objectName := objectNamePrefix + "tpo" objectValue := "大江东去，浪淘尽，千古风流人物。 故垒西边，人道是、三国周郎赤壁。 乱石穿空，惊涛拍岸，卷起千堆雪。 江山如画，一时多少豪杰。" + "遥想公谨当年，小乔初嫁了，雄姿英发。 羽扇纶巾，谈笑间、樯橹灰飞烟灭。故国神游，多情应笑我，早生华发，人生如梦，一尊还酹江月。" // string put err := s.bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Check body, err := s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) acl, err := s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) testLogger.Println("aclRes:", acl) c.Assert(acl.ACL, Equals, "default") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // bytes put err = s.bucket.PutObject(objectName, bytes.NewReader([]byte(objectValue))) c.Assert(err, IsNil) // Check body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // file put err = createFileAndWrite(objectName+".txt", []byte(objectValue)) c.Assert(err, IsNil) fd, err := os.Open(objectName + ".txt") c.Assert(err, IsNil) err = s.bucket.PutObject(objectName, fd) c.Assert(err, IsNil) os.Remove(objectName + ".txt") // Check body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // Put with properties objectName = objectNamePrefix + "tpox" options := []Option{ Expires(futureDate), ObjectACL(ACLPublicRead), Meta("myprop", "mypropval"), } err = s.bucket.PutObject(objectName, strings.NewReader(objectValue), options...) c.Assert(err, IsNil) // Check body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) acl, err = s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectACL:", acl) c.Assert(acl.ACL, Equals, string(ACLPublicRead)) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestPutObjectType func (s *OssBucketSuite) TestPutObjectType(c *C) { objectName := objectNamePrefix + "tptt" objectValue := "乱石穿空，惊涛拍岸，卷起千堆雪。 江山如画，一时多少豪杰。" // Put err := s.bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Check time.Sleep(time.Second) body, err := s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) c.Assert(meta.Get("Content-Type"), Equals, "application/octet-stream") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // Put err = s.bucket.PutObject(objectName+".txt", strings.NewReader(objectValue)) c.Assert(err, IsNil) meta, err = s.bucket.GetObjectDetailedMeta(objectName + ".txt") c.Assert(err, IsNil) c.Assert(meta.Get("Content-Type"), Equals, "text/plain; charset=utf-8") err = s.bucket.DeleteObject(objectName + ".txt") c.Assert(err, IsNil) // Put err = s.bucket.PutObject(objectName+".apk", strings.NewReader(objectValue)) c.Assert(err, IsNil) meta, err = s.bucket.GetObjectDetailedMeta(objectName + ".apk") c.Assert(err, IsNil) c.Assert(meta.Get("Content-Type"), Equals, "application/vnd.android.package-archive") err = s.bucket.DeleteObject(objectName + ".txt") c.Assert(err, IsNil) } // TestPutObject func (s *OssBucketSuite) TestPutObjectKeyChars(c *C) { objectName := objectNamePrefix + "tpokc" objectValue := "白日依山尽，黄河入海流。欲穷千里目，更上一层楼。" // Put objectKey := objectName + "十步杀一人，千里不留行。事了拂衣去，深藏身与名" err := s.bucket.PutObject(objectKey, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Check body, err := s.bucket.GetObject(objectKey) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectKey) c.Assert(err, IsNil) // Put objectKey = objectName + "ごきげん如何ですかおれの顔をよく拝んでおけ" err = s.bucket.PutObject(objectKey, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Check body, err = s.bucket.GetObject(objectKey) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectKey) c.Assert(err, IsNil) // Put objectKey = objectName + "~!@#$%^&*()_-+=|\\[]{}<>,./?" err = s.bucket.PutObject(objectKey, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Check body, err = s.bucket.GetObject(objectKey) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectKey) c.Assert(err, IsNil) // Put objectKey = "go/中国 日本 +-#&=*" err = s.bucket.PutObject(objectKey, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Check body, err = s.bucket.GetObject(objectKey) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectKey) c.Assert(err, IsNil) } // TestPutObjectNegative func (s *OssBucketSuite) TestPutObjectNegative(c *C) { objectName := objectNamePrefix + "tpon" objectValue := "大江东去，浪淘尽，千古风流人物。 " // Put objectName = objectNamePrefix + "tpox" err := s.bucket.PutObject(objectName, strings.NewReader(objectValue), Meta("meta-my", "myprop")) c.Assert(err, IsNil) // Check meta body, err := s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) c.Assert(meta.Get("X-Oss-Meta-My"), Not(Equals), "myprop") c.Assert(meta.Get("X-Oss-Meta-My"), Equals, "") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // invalid option err = s.bucket.PutObject(objectName, strings.NewReader(objectValue), IfModifiedSince(pastDate)) c.Assert(err, NotNil) err = s.bucket.PutObjectFromFile(objectName, "bucket.go", IfModifiedSince(pastDate)) c.Assert(err, NotNil) err = s.bucket.PutObjectFromFile(objectName, "/tmp/xxx") c.Assert(err, NotNil) } // TestPutObjectFromFile func (s *OssBucketSuite) TestPutObjectFromFile(c *C) { objectName := objectNamePrefix + "tpoff" localFile := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "newpic11.jpg" // Put err := s.bucket.PutObjectFromFile(objectName, localFile) c.Assert(err, IsNil) // Check err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(localFile, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) acl, err := s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) testLogger.Println("aclRes:", acl) c.Assert(acl.ACL, Equals, "default") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // Put with properties options := []Option{ Expires(futureDate), ObjectACL(ACLPublicRead), Meta("myprop", "mypropval"), } os.Remove(newFile) err = s.bucket.PutObjectFromFile(objectName, localFile, options...) c.Assert(err, IsNil) // Check err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(localFile, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) acl, err = s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectACL:", acl) c.Assert(acl.ACL, Equals, string(ACLPublicRead)) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) os.Remove(newFile) } // TestPutObjectFromFile func (s *OssBucketSuite) TestPutObjectFromFileType(c *C) { objectName := objectNamePrefix + "tpoffwm" localFile := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "newpic11.jpg" // Put err := s.bucket.PutObjectFromFile(objectName, localFile) c.Assert(err, IsNil) // Check err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(localFile, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) c.Assert(meta.Get("Content-Type"), Equals, "image/jpeg") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) os.Remove(newFile) } // TestGetObject func (s *OssBucketSuite) TestGetObject(c *C) { objectName := objectNamePrefix + "tgo" objectValue := "长忆观潮，满郭人争江上望。来疑沧海尽成空，万面鼓声中。弄潮儿向涛头立，手把红旗旗不湿。别来几向梦中看，梦觉尚心寒。" // Put err := s.bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Check body, err := s.bucket.GetObject(objectName) c.Assert(err, IsNil) data, err := ioutil.ReadAll(body) body.Close() str := string(data) c.Assert(str, Equals, objectValue) testLogger.Println("GetObjec:", str) // Range var subObjectValue = string(([]byte(objectValue))[15:36]) body, err = s.bucket.GetObject(objectName, Range(15, 35)) c.Assert(err, IsNil) data, err = ioutil.ReadAll(body) body.Close() str = string(data) c.Assert(str, Equals, subObjectValue) testLogger.Println("GetObject:", str, ",", subObjectValue) // If-Modified-Since _, err = s.bucket.GetObject(objectName, IfModifiedSince(futureDate)) c.Assert(err, NotNil) // If-Unmodified-Since body, err = s.bucket.GetObject(objectName, IfUnmodifiedSince(futureDate)) c.Assert(err, IsNil) data, err = ioutil.ReadAll(body) body.Close() c.Assert(string(data), Equals, objectValue) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) // If-Match body, err = s.bucket.GetObject(objectName, IfMatch(meta.Get("Etag"))) c.Assert(err, IsNil) data, err = ioutil.ReadAll(body) body.Close() c.Assert(string(data), Equals, objectValue) // If-None-Match _, err = s.bucket.GetObject(objectName, IfNoneMatch(meta.Get("Etag"))) c.Assert(err, NotNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestGetObjectNegative func (s *OssBucketSuite) TestGetObjectToWriterNegative(c *C) { objectName := objectNamePrefix + "tgotwn" objectValue := "长忆观潮，满郭人争江上望。" // object not exist _, err := s.bucket.GetObject("NotExist") c.Assert(err, NotNil) // constraint invalid err = s.bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // out of range _, err = s.bucket.GetObject(objectName, Range(15, 1000)) c.Assert(err, IsNil) // no exist err = s.bucket.GetObjectToFile(objectName, "/root/123abc9874") c.Assert(err, NotNil) // invalid option _, err = s.bucket.GetObject(objectName, ACL(ACLPublicRead)) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(objectName, "newpic15.jpg", ACL(ACLPublicRead)) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestGetObjectToFile func (s *OssBucketSuite) TestGetObjectToFile(c *C) { objectName := objectNamePrefix + "tgotf" objectValue := "江南好，风景旧曾谙；日出江花红胜火，春来江水绿如蓝。能不忆江南？江南忆，最忆是杭州；山寺月中寻桂子，郡亭枕上看潮头。何日更重游！" newFile := "newpic15.jpg" // Put var val = []byte(objectValue) err := s.bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Check err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err := compareFileData(newFile, val) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFile) // Range err = s.bucket.GetObjectToFile(objectName, newFile, Range(15, 35)) c.Assert(err, IsNil) eq, err = compareFileData(newFile, val[15:36]) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile, NormalizedRange("15-35")) c.Assert(err, IsNil) eq, err = compareFileData(newFile, val[15:36]) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile, NormalizedRange("15-")) c.Assert(err, IsNil) eq, err = compareFileData(newFile, val[15:]) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile, NormalizedRange("-10")) c.Assert(err, IsNil) eq, err = compareFileData(newFile, val[(len(val)-10):len(val)]) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFile) // If-Modified-Since err = s.bucket.GetObjectToFile(objectName, newFile, IfModifiedSince(futureDate)) c.Assert(err, NotNil) // If-Unmodified-Since err = s.bucket.GetObjectToFile(objectName, newFile, IfUnmodifiedSince(futureDate)) c.Assert(err, IsNil) eq, err = compareFileData(newFile, val) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFile) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) // If-Match err = s.bucket.GetObjectToFile(objectName, newFile, IfMatch(meta.Get("Etag"))) c.Assert(err, IsNil) eq, err = compareFileData(newFile, val) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFile) // If-None-Match err = s.bucket.GetObjectToFile(objectName, newFile, IfNoneMatch(meta.Get("Etag"))) c.Assert(err, NotNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestListObjects func (s *OssBucketSuite) TestListObjects(c *C) { objectName := objectNamePrefix + "tlo" // list empty bucket lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) left := len(lor.Objects) // Put three object err = s.bucket.PutObject(objectName+"1", strings.NewReader("")) c.Assert(err, IsNil) err = s.bucket.PutObject(objectName+"2", strings.NewReader("")) c.Assert(err, IsNil) err = s.bucket.PutObject(objectName+"3", strings.NewReader("")) c.Assert(err, IsNil) // list lor, err = s.bucket.ListObjects() c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, left+3) // list with prefix lor, err = s.bucket.ListObjects(Prefix(objectName + "2")) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 1) lor, err = s.bucket.ListObjects(Prefix(objectName + "22")) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 0) // list with max keys lor, err = s.bucket.ListObjects(Prefix(objectName), MaxKeys(2)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 2) // list with marker lor, err = s.bucket.ListObjects(Marker(objectName+"1"), MaxKeys(1)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 1) err = s.bucket.DeleteObject(objectName + "1") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName + "2") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName + "3") c.Assert(err, IsNil) } // TestListObjects func (s *OssBucketSuite) TestListObjectsEncodingType(c *C) { objectName := objectNamePrefix + "床前明月光，疑是地上霜。举头望明月，低头思故乡。" + "tloet" for i := 0; i < 10; i++ { err := s.bucket.PutObject(objectName+strconv.Itoa(i), strings.NewReader("")) c.Assert(err, IsNil) } lor, err := s.bucket.ListObjects(Prefix(objectNamePrefix + "床前明月光，")) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 10) lor, err = s.bucket.ListObjects(Prefix(objectNamePrefix + "床前明月光，")) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 10) lor, err = s.bucket.ListObjects(Marker(objectNamePrefix + "床前明月光，疑是地上霜。举头望明月，低头思故乡。")) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 10) lor, err = s.bucket.ListObjects(Prefix(objectNamePrefix + "床前明月光")) c.Assert(err, IsNil) for i, obj := range lor.Objects { c.Assert(obj.Key, Equals, objectNamePrefix+"床前明月光，疑是地上霜。举头望明月，低头思故乡。tloet"+strconv.Itoa(i)) } for i := 0; i < 10; i++ { err = s.bucket.DeleteObject(objectName + strconv.Itoa(i)) c.Assert(err, IsNil) } // 特殊字符 objectName = "go go ` ~ ! @ # $ % ^ & * () - _ + =[] {} \\ | < > , . ? / 0" err = s.bucket.PutObject(objectName, strings.NewReader("明月几时有，把酒问青天")) c.Assert(err, IsNil) lor, err = s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 1) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) objectName = "go/中国 日本 +-#&=*" err = s.bucket.PutObject(objectName, strings.NewReader("明月几时有，把酒问青天")) c.Assert(err, IsNil) lor, err = s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 1) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestIsBucketExist func (s *OssBucketSuite) TestIsObjectExist(c *C) { objectName := objectNamePrefix + "tibe" // Put three object err := s.bucket.PutObject(objectName+"1", strings.NewReader("")) c.Assert(err, IsNil) err = s.bucket.PutObject(objectName+"11", strings.NewReader("")) c.Assert(err, IsNil) err = s.bucket.PutObject(objectName+"111", strings.NewReader("")) c.Assert(err, IsNil) // exist exist, err := s.bucket.IsObjectExist(objectName + "11") c.Assert(err, IsNil) c.Assert(exist, Equals, true) exist, err = s.bucket.IsObjectExist(objectName + "1") c.Assert(err, IsNil) c.Assert(exist, Equals, true) exist, err = s.bucket.IsObjectExist(objectName + "111") c.Assert(err, IsNil) c.Assert(exist, Equals, true) // not exist exist, err = s.bucket.IsObjectExist(objectName + "1111") c.Assert(err, IsNil) c.Assert(exist, Equals, false) exist, err = s.bucket.IsObjectExist(objectName) c.Assert(err, IsNil) c.Assert(exist, Equals, false) err = s.bucket.DeleteObject(objectName + "1") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName + "11") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName + "111") c.Assert(err, IsNil) } // TestDeleteObject func (s *OssBucketSuite) TestDeleteObject(c *C) { objectName := objectNamePrefix + "tdo" err := s.bucket.PutObject(objectName, strings.NewReader("")) c.Assert(err, IsNil) lor, err := s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 1) // delete err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // duplicate delete err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) lor, err = s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 0) } // TestDeleteObjects func (s *OssBucketSuite) TestDeleteObjects(c *C) { objectName := objectNamePrefix + "tdos" // delete object err := s.bucket.PutObject(objectName, strings.NewReader("")) c.Assert(err, IsNil) res, err := s.bucket.DeleteObjects([]string{objectName}) c.Assert(err, IsNil) c.Assert(len(res.DeletedObjects), Equals, 1) lor, err := s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 0) // delete objects err = s.bucket.PutObject(objectName+"1", strings.NewReader("")) c.Assert(err, IsNil) err = s.bucket.PutObject(objectName+"2", strings.NewReader("")) c.Assert(err, IsNil) res, err = s.bucket.DeleteObjects([]string{objectName + "1", objectName + "2"}) c.Assert(err, IsNil) c.Assert(len(res.DeletedObjects), Equals, 2) lor, err = s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 0) // delete 0 _, err = s.bucket.DeleteObjects([]string{}) c.Assert(err, NotNil) // DeleteObjectsQuiet err = s.bucket.PutObject(objectName+"1", strings.NewReader("")) c.Assert(err, IsNil) err = s.bucket.PutObject(objectName+"2", strings.NewReader("")) c.Assert(err, IsNil) res, err = s.bucket.DeleteObjects([]string{objectName + "1", objectName + "2"}, DeleteObjectsQuiet(false)) c.Assert(err, IsNil) c.Assert(len(res.DeletedObjects), Equals, 2) lor, err = s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 0) // DeleteObjectsQuiet err = s.bucket.PutObject(objectName+"1", strings.NewReader("")) c.Assert(err, IsNil) err = s.bucket.PutObject(objectName+"2", strings.NewReader("")) c.Assert(err, IsNil) res, err = s.bucket.DeleteObjects([]string{objectName + "1", objectName + "2"}, DeleteObjectsQuiet(true)) c.Assert(err, IsNil) c.Assert(len(res.DeletedObjects), Equals, 0) lor, err = s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, 0) // EncodingType err = s.bucket.PutObject("中国人", strings.NewReader("")) c.Assert(err, IsNil) res, err = s.bucket.DeleteObjects([]string{"中国人"}) c.Assert(err, IsNil) c.Assert(len(res.DeletedObjects), Equals, 1) c.Assert(res.DeletedObjects[0], Equals, "中国人") // EncodingType err = s.bucket.PutObject("中国人", strings.NewReader("")) c.Assert(err, IsNil) res, err = s.bucket.DeleteObjects([]string{"中国人"}, DeleteObjectsQuiet(false)) c.Assert(err, IsNil) c.Assert(len(res.DeletedObjects), Equals, 1) c.Assert(res.DeletedObjects[0], Equals, "中国人") // EncodingType err = s.bucket.PutObject("中国人", strings.NewReader("")) c.Assert(err, IsNil) res, err = s.bucket.DeleteObjects([]string{"中国人"}, DeleteObjectsQuiet(true)) c.Assert(err, IsNil) c.Assert(len(res.DeletedObjects), Equals, 0) // 特殊字符 key := "A ' < > \" & ~ ` ! @ # $ % ^ & * ( ) [] {} - _ + = / | \\ ? . , : ; A" err = s.bucket.PutObject(key, strings.NewReader("value")) c.Assert(err, IsNil) _, err = s.bucket.DeleteObjects([]string{key}) c.Assert(err, IsNil) ress, err := s.bucket.ListObjects(Prefix(key)) c.Assert(err, IsNil) c.Assert(len(ress.Objects), Equals, 0) // not exist _, err = s.bucket.DeleteObjects([]string{"NotExistObject"}) c.Assert(err, IsNil) } // TestSetObjectMeta func (s *OssBucketSuite) TestSetObjectMeta(c *C) { objectName := objectNamePrefix + "tsom" err := s.bucket.PutObject(objectName, strings.NewReader("")) c.Assert(err, IsNil) err = s.bucket.SetObjectMeta(objectName, Expires(futureDate), Meta("myprop", "mypropval")) c.Assert(err, IsNil) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("Meta:", meta) c.Assert(meta.Get("Expires"), Equals, futureDate.Format(http.TimeFormat)) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") acl, err := s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) c.Assert(acl.ACL, Equals, "default") // invalid option err = s.bucket.SetObjectMeta(objectName, AcceptEncoding("url")) c.Assert(err, IsNil) // invalid option value err = s.bucket.SetObjectMeta(objectName, ServerSideEncryption("invalid")) c.Assert(err, NotNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // no exist err = s.bucket.SetObjectMeta(objectName, Expires(futureDate)) c.Assert(err, NotNil) } // TestGetObjectMeta func (s *OssBucketSuite) TestGetObjectMeta(c *C) { objectName := objectNamePrefix + "tgom" // Put err := s.bucket.PutObject(objectName, strings.NewReader("")) c.Assert(err, IsNil) meta, err := s.bucket.GetObjectMeta(objectName) c.Assert(err, IsNil) c.Assert(len(meta) > 0, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) _, err = s.bucket.GetObjectMeta("NotExistObject") c.Assert(err, NotNil) } // TestGetObjectDetailedMeta func (s *OssBucketSuite) TestGetObjectDetailedMeta(c *C) { objectName := objectNamePrefix + "tgodm" // Put err := s.bucket.PutObject(objectName, strings.NewReader(""), Expires(futureDate), Meta("myprop", "mypropval")) c.Assert(err, IsNil) // Check meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) c.Assert(meta.Get("Expires"), Equals, futureDate.Format(http.TimeFormat)) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") c.Assert(meta.Get("Content-Length"), Equals, "0") c.Assert(len(meta.Get("Date")) > 0, Equals, true) c.Assert(len(meta.Get("X-Oss-Request-Id")) > 0, Equals, true) c.Assert(len(meta.Get("Last-Modified")) > 0, Equals, true) // IfModifiedSince/IfModifiedSince _, err = s.bucket.GetObjectDetailedMeta(objectName, IfModifiedSince(futureDate)) c.Assert(err, NotNil) meta, err = s.bucket.GetObjectDetailedMeta(objectName, IfUnmodifiedSince(futureDate)) c.Assert(err, IsNil) c.Assert(meta.Get("Expires"), Equals, futureDate.Format(http.TimeFormat)) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") // IfMatch/IfNoneMatch _, err = s.bucket.GetObjectDetailedMeta(objectName, IfNoneMatch(meta.Get("Etag"))) c.Assert(err, NotNil) meta, err = s.bucket.GetObjectDetailedMeta(objectName, IfMatch(meta.Get("Etag"))) c.Assert(err, IsNil) c.Assert(meta.Get("Expires"), Equals, futureDate.Format(http.TimeFormat)) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) _, err = s.bucket.GetObjectDetailedMeta("NotExistObject") c.Assert(err, NotNil) } // TestSetAndGetObjectAcl func (s *OssBucketSuite) TestSetAndGetObjectAcl(c *C) { objectName := objectNamePrefix + "tsgba" err := s.bucket.PutObject(objectName, strings.NewReader("")) c.Assert(err, IsNil) // default acl, err := s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) c.Assert(acl.ACL, Equals, "default") // set ACL_PUBLIC_RW err = s.bucket.SetObjectACL(objectName, ACLPublicReadWrite) c.Assert(err, IsNil) acl, err = s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) c.Assert(acl.ACL, Equals, string(ACLPublicReadWrite)) // set ACL_PRIVATE err = s.bucket.SetObjectACL(objectName, ACLPrivate) c.Assert(err, IsNil) acl, err = s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) c.Assert(acl.ACL, Equals, string(ACLPrivate)) // set ACL_PUBLIC_R err = s.bucket.SetObjectACL(objectName, ACLPublicRead) c.Assert(err, IsNil) acl, err = s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) c.Assert(acl.ACL, Equals, string(ACLPublicRead)) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestSetAndGetObjectAclNegative func (s *OssBucketSuite) TestSetAndGetObjectAclNegative(c *C) { objectName := objectNamePrefix + "tsgban" // object not exist err := s.bucket.SetObjectACL(objectName, ACLPublicRead) c.Assert(err, NotNil) } // TestCopyObject func (s *OssBucketSuite) TestCopyObject(c *C) { objectName := objectNamePrefix + "tco" objectValue := "男儿何不带吴钩，收取关山五十州。请君暂上凌烟阁，若个书生万户侯？" err := s.bucket.PutObject(objectName, strings.NewReader(objectValue), ACL(ACLPublicRead), Meta("my", "myprop")) c.Assert(err, IsNil) // copy var objectNameDest = objectName + "dest" _, err = s.bucket.CopyObject(objectName, objectNameDest) c.Assert(err, IsNil) // check lor, err := s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) testLogger.Println("objects:", lor.Objects) c.Assert(len(lor.Objects), Equals, 2) body, err := s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectNameDest) c.Assert(err, IsNil) // copy with constraints x-oss-copy-source-if-modified-since _, err = s.bucket.CopyObject(objectName, objectNameDest, CopySourceIfModifiedSince(futureDate)) c.Assert(err, NotNil) testLogger.Println("CopyObject:", err) // copy with constraints x-oss-copy-source-if-unmodified-since _, err = s.bucket.CopyObject(objectName, objectNameDest, CopySourceIfUnmodifiedSince(futureDate)) c.Assert(err, IsNil) // check lor, err = s.bucket.ListObjects(Prefix(objectName)) c.Assert(err, IsNil) testLogger.Println("objects:", lor.Objects) c.Assert(len(lor.Objects), Equals, 2) body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectNameDest) c.Assert(err, IsNil) // copy with constraints x-oss-copy-source-if-match meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) _, err = s.bucket.CopyObject(objectName, objectNameDest, CopySourceIfMatch(meta.Get("Etag"))) c.Assert(err, IsNil) // check body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectNameDest) c.Assert(err, IsNil) // copy with constraints x-oss-copy-source-if-none-match _, err = s.bucket.CopyObject(objectName, objectNameDest, CopySourceIfNoneMatch(meta.Get("Etag"))) c.Assert(err, NotNil) // copy with constraints x-oss-metadata-directive _, err = s.bucket.CopyObject(objectName, objectNameDest, Meta("my", "mydestprop"), MetadataDirective(MetaCopy)) c.Assert(err, IsNil) // check body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) destMeta, err := s.bucket.GetObjectDetailedMeta(objectNameDest) c.Assert(err, IsNil) c.Assert(meta.Get("X-Oss-Meta-My"), Equals, "myprop") acl, err := s.bucket.GetObjectACL(objectNameDest) c.Assert(err, IsNil) c.Assert(acl.ACL, Equals, "default") err = s.bucket.DeleteObject(objectNameDest) c.Assert(err, IsNil) // copy with constraints x-oss-metadata-directive and self defined dest object meta options := []Option{ ObjectACL(ACLPublicReadWrite), Meta("my", "mydestprop"), MetadataDirective(MetaReplace), } _, err = s.bucket.CopyObject(objectName, objectNameDest, options...) c.Assert(err, IsNil) // check body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) destMeta, err = s.bucket.GetObjectDetailedMeta(objectNameDest) c.Assert(err, IsNil) c.Assert(destMeta.Get("X-Oss-Meta-My"), Equals, "mydestprop") acl, err = s.bucket.GetObjectACL(objectNameDest) c.Assert(err, IsNil) c.Assert(acl.ACL, Equals, string(ACLPublicReadWrite)) err = s.bucket.DeleteObject(objectNameDest) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestCopyObjectToOrFrom func (s *OssBucketSuite) TestCopyObjectToOrFrom(c *C) { objectName := objectNamePrefix + "tcotof" objectValue := "男儿何不带吴钩，收取关山五十州。请君暂上凌烟阁，若个书生万户侯？" destBucket := bucketName + "-dest" objectNameDest := objectName + "dest" s.client.CreateBucket(destBucket) destBuck, err := s.client.Bucket(destBucket) c.Assert(err, IsNil) err = s.bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // copy from _, err = destBuck.CopyObjectFrom(bucketName, objectName, objectNameDest) c.Assert(err, IsNil) // check body, err := destBuck.GetObject(objectNameDest) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // copy to _, err = destBuck.CopyObjectTo(bucketName, objectName, objectNameDest) c.Assert(err, IsNil) // check body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) // clean err = destBuck.DeleteObject(objectNameDest) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) err = s.client.DeleteBucket(destBucket) c.Assert(err, IsNil) } // TestCopyObjectToOrFromNegative func (s *OssBucketSuite) TestCopyObjectToOrFromNegative(c *C) { objectName := objectNamePrefix + "tcotofn" destBucket := bucketName + "-destn" objectNameDest := objectName + "destn" // object no exist _, err := s.bucket.CopyObjectTo(bucketName, objectName, objectNameDest) c.Assert(err, NotNil) // bucket no exist _, err = s.bucket.CopyObjectFrom(destBucket, objectNameDest, objectName) c.Assert(err, NotNil) } // TestAppendObject func (s *OssBucketSuite) TestAppendObject(c *C) { objectName := objectNamePrefix + "tao" objectValue := "昨夜雨疏风骤，浓睡不消残酒。试问卷帘人，却道海棠依旧。知否？知否？应是绿肥红瘦。" var val = []byte(objectValue) var localFile = "testx.txt" var nextPos int64 var midPos = 1 + rand.Intn(len(val)-1) var err = createFileAndWrite(localFile+"1", val[0:midPos]) c.Assert(err, IsNil) err = createFileAndWrite(localFile+"2", val[midPos:]) c.Assert(err, IsNil) // string append nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader("昨夜雨疏风骤，浓睡不消残酒。试问卷帘人，"), nextPos) c.Assert(err, IsNil) nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader("却道海棠依旧。知否？知否？应是绿肥红瘦。"), nextPos) c.Assert(err, IsNil) body, err := s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // byte append nextPos = 0 nextPos, err = s.bucket.AppendObject(objectName, bytes.NewReader(val[0:midPos]), nextPos) c.Assert(err, IsNil) nextPos, err = s.bucket.AppendObject(objectName, bytes.NewReader(val[midPos:]), nextPos) c.Assert(err, IsNil) body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // file append options := []Option{ ObjectACL(ACLPublicReadWrite), Meta("my", "myprop"), } fd, err := os.Open(localFile + "1") c.Assert(err, IsNil) defer fd.Close() nextPos = 0 nextPos, err = s.bucket.AppendObject(objectName, fd, nextPos, options...) c.Assert(err, IsNil) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta, ",", nextPos) c.Assert(meta.Get("X-Oss-Object-Type"), Equals, "Appendable") c.Assert(meta.Get("X-Oss-Meta-My"), Equals, "myprop") c.Assert(meta.Get("x-oss-Meta-Mine"), Equals, "") c.Assert(meta.Get("X-Oss-Next-Append-Position"), Equals, strconv.FormatInt(nextPos, 10)) acl, err := s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectACL:", acl) c.Assert(acl.ACL, Equals, string(ACLPublicReadWrite)) // second append options = []Option{ ObjectACL(ACLPublicRead), Meta("my", "myproptwo"), Meta("mine", "mypropmine"), } fd, err = os.Open(localFile + "2") c.Assert(err, IsNil) defer fd.Close() nextPos, err = s.bucket.AppendObject(objectName, fd, nextPos, options...) c.Assert(err, IsNil) body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) meta, err = s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta xxx:", meta) c.Assert(meta.Get("X-Oss-Object-Type"), Equals, "Appendable") c.Assert(meta.Get("X-Oss-Meta-My"), Equals, "myprop") c.Assert(meta.Get("x-Oss-Meta-Mine"), Equals, "") c.Assert(meta.Get("X-Oss-Next-Append-Position"), Equals, strconv.FormatInt(nextPos, 10)) acl, err = s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) c.Assert(acl.ACL, Equals, string(ACLPublicRead)) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestContentType func (s *OssBucketSuite) TestAddContentType(c *C) { opts := addContentType(nil, "abc.txt") typ, err := findOption(opts, HTTPHeaderContentType, "") c.Assert(err, IsNil) c.Assert(typ, Equals, "text/plain; charset=utf-8") opts = addContentType(nil) typ, err = findOption(opts, HTTPHeaderContentType, "") c.Assert(err, IsNil) c.Assert(len(opts), Equals, 1) c.Assert(typ, Equals, "application/octet-stream") opts = addContentType(nil, "abc.txt", "abc.pdf") typ, err = findOption(opts, HTTPHeaderContentType, "") c.Assert(err, IsNil) c.Assert(typ, Equals, "text/plain; charset=utf-8") opts = addContentType(nil, "abc", "abc.txt", "abc.pdf") typ, err = findOption(opts, HTTPHeaderContentType, "") c.Assert(err, IsNil) c.Assert(typ, Equals, "text/plain; charset=utf-8") opts = addContentType(nil, "abc", "abc", "edf") typ, err = findOption(opts, HTTPHeaderContentType, "") c.Assert(err, IsNil) c.Assert(typ, Equals, "application/octet-stream") opts = addContentType([]Option{Meta("meta", "my")}, "abc", "abc.txt", "abc.pdf") typ, err = findOption(opts, HTTPHeaderContentType, "") c.Assert(err, IsNil) c.Assert(len(opts), Equals, 2) c.Assert(typ, Equals, "text/plain; charset=utf-8") } func (s *OssBucketSuite) TestGetConfig(c *C) { client, err := New(endpoint, accessID, accessKey, UseCname(true), Timeout(11, 12), SecurityToken("token"), EnableMD5(false)) c.Assert(err, IsNil) bucket, err := client.Bucket(bucketName) c.Assert(err, IsNil) c.Assert(bucket.getConfig().HTTPTimeout.ConnectTimeout, Equals, time.Second*11) c.Assert(bucket.getConfig().HTTPTimeout.ReadWriteTimeout, Equals, time.Second*12) c.Assert(bucket.getConfig().HTTPTimeout.HeaderTimeout, Equals, time.Second*12) c.Assert(bucket.getConfig().HTTPTimeout.LongTimeout, Equals, time.Second*12*10) c.Assert(bucket.getConfig().SecurityToken, Equals, "token") c.Assert(bucket.getConfig().IsCname, Equals, true) c.Assert(bucket.getConfig().IsEnableMD5, Equals, false) } // TestSTSTonek func (s *OssBucketSuite) _TestSTSTonek(c *C) { objectName := objectNamePrefix + "tst" objectValue := "红藕香残玉簟秋。轻解罗裳，独上兰舟。云中谁寄锦书来？雁字回时，月满西楼。" stsServer := "" stsEndpoint := "" stsBucketName := "" stsRes, err := getSTSToken(stsServer) c.Assert(err, IsNil) testLogger.Println("sts:", stsRes) client, err := New(stsEndpoint, stsRes.AccessID, stsRes.AccessKey, SecurityToken(stsRes.SecurityToken)) c.Assert(err, IsNil) bucket, err := client.Bucket(stsBucketName) c.Assert(err, IsNil) // Put err = bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Get body, err := s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) // List lor, err := bucket.ListObjects() c.Assert(err, IsNil) testLogger.Println("Objects:", lor.Objects) // Delete err = bucket.DeleteObject(objectName) c.Assert(err, IsNil) } func (s *OssBucketSuite) TestSTSTonekNegative(c *C) { objectName := objectNamePrefix + "tstg" localFile := objectName + ".jpg" client, err := New(endpoint, accessID, accessKey, SecurityToken("Invalid")) c.Assert(err, IsNil) _, err = client.ListBuckets() c.Assert(err, NotNil) bucket, err := client.Bucket(bucketName) c.Assert(err, IsNil) err = bucket.PutObject(objectName, strings.NewReader("")) c.Assert(err, NotNil) err = bucket.PutObjectFromFile(objectName, "") c.Assert(err, NotNil) _, err = bucket.GetObject(objectName) c.Assert(err, NotNil) err = bucket.GetObjectToFile(objectName, "") c.Assert(err, NotNil) _, err = bucket.ListObjects() c.Assert(err, NotNil) err = bucket.SetObjectACL(objectName, ACLPublicRead) c.Assert(err, NotNil) _, err = bucket.GetObjectACL(objectName) c.Assert(err, NotNil) err = bucket.UploadFile(objectName, localFile, MinPartSize) c.Assert(err, NotNil) err = bucket.DownloadFile(objectName, localFile, MinPartSize) c.Assert(err, NotNil) _, err = bucket.IsObjectExist(objectName) c.Assert(err, NotNil) _, err = bucket.ListMultipartUploads() c.Assert(err, NotNil) err = bucket.DeleteObject(objectName) c.Assert(err, NotNil) _, err = bucket.DeleteObjects([]string{objectName}) c.Assert(err, NotNil) err = client.DeleteBucket(bucketName) c.Assert(err, NotNil) _, err = getSTSToken("") c.Assert(err, NotNil) _, err = getSTSToken("http://me.php") c.Assert(err, NotNil) } func (s *OssBucketSuite) TestUploadBigFile(c *C) { objectName := objectNamePrefix + "tubf" bigFile := "D:\\tmp\\bigfile.zip" newFile := "D:\\tmp\\newbigfile.zip" exist, err := isFileExist(bigFile) c.Assert(err, IsNil) if !exist { return } // Put start := GetNowSec() err = s.bucket.PutObjectFromFile(objectName, bigFile) c.Assert(err, IsNil) end := GetNowSec() testLogger.Println("Put big file:", bigFile, "use sec:", end-start) // Check start = GetNowSec() err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) end = GetNowSec() testLogger.Println("Get big file:", bigFile, "use sec:", end-start) start = GetNowSec() eq, err := compareFiles(bigFile, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) end = GetNowSec() testLogger.Println("Compare big file:", bigFile, "use sec:", end-start) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } func (s *OssBucketSuite) TestSymlink(c *C) { objectName := objectNamePrefix + "符号链接" targetObjectName := objectNamePrefix + "符号链接目标文件" err := s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) err = s.bucket.DeleteObject(targetObjectName) c.Assert(err, IsNil) meta, err := s.bucket.GetSymlink(objectName) c.Assert(err, NotNil) // Put symlink err = s.bucket.PutSymlink(objectName, targetObjectName) c.Assert(err, IsNil) err = s.bucket.PutObject(targetObjectName, strings.NewReader("target")) c.Assert(err, IsNil) err = s.bucket.PutSymlink(objectName, targetObjectName) c.Assert(err, IsNil) meta, err = s.bucket.GetSymlink(objectName) c.Assert(err, IsNil) c.Assert(meta.Get(HTTPHeaderOssSymlinkTarget), Equals, targetObjectName) // List object lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) exist, v := s.getObject(lor.Objects, objectName) c.Assert(exist, Equals, true) c.Assert(v.Type, Equals, "Symlink") body, err := s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err := readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, "target") meta, err = s.bucket.GetSymlink(targetObjectName) c.Assert(err, NotNil) err = s.bucket.PutObject(objectName, strings.NewReader("src")) c.Assert(err, IsNil) body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, "src") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) err = s.bucket.DeleteObject(targetObjectName) c.Assert(err, IsNil) // put symlink again objectName = objectNamePrefix + "symlink" targetObjectName = objectNamePrefix + "symlink-target" err = s.bucket.PutSymlink(objectName, targetObjectName) c.Assert(err, IsNil) err = s.bucket.PutObject(targetObjectName, strings.NewReader("target1")) c.Assert(err, IsNil) meta, err = s.bucket.GetSymlink(objectName) c.Assert(err, IsNil) c.Assert(meta.Get(HTTPHeaderOssSymlinkTarget), Equals, targetObjectName) body, err = s.bucket.GetObject(objectName) c.Assert(err, IsNil) str, err = readBody(body) c.Assert(err, IsNil) c.Assert(str, Equals, "target1") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) err = s.bucket.DeleteObject(targetObjectName) c.Assert(err, IsNil) } // TestRestoreObject func (s *OssBucketSuite) TestRestoreObject(c *C) { objectName := objectNamePrefix + "restore" // List Object lor, err := s.archiveBucket.ListObjects() c.Assert(err, IsNil) left := len(lor.Objects) // Put three object err = s.archiveBucket.PutObject(objectName, strings.NewReader("")) c.Assert(err, IsNil) // List lor, err = s.archiveBucket.ListObjects() c.Assert(err, IsNil) c.Assert(len(lor.Objects), Equals, left+1) for _, object := range lor.Objects { c.Assert(object.StorageClass, Equals, string(StorageArchive)) c.Assert(object.Type, Equals, "Normal") } // Head Object meta, err := s.archiveBucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) _, ok := meta["X-Oss-Restore"] c.Assert(ok, Equals, false) c.Assert(meta.Get("X-Oss-Storage-Class"), Equals, "Archive") // Error Restore err = s.archiveBucket.RestoreObject("notexistobject") c.Assert(err, NotNil) // Restore Object err = s.archiveBucket.RestoreObject(objectName) c.Assert(err, IsNil) // Head Object meta, err = s.archiveBucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) c.Assert(meta.Get("X-Oss-Restore"), Equals, "ongoing-request=\"true\"") c.Assert(meta.Get("X-Oss-Storage-Class"), Equals, "Archive") } // private func createFileAndWrite(fileName string, data []byte) error { os.Remove(fileName) fo, err := os.Create(fileName) if err != nil { return err } defer fo.Close() bytes, err := fo.Write(data) if err != nil { return err } if bytes != len(data) { return fmt.Errorf(fmt.Sprintf("write %d bytes not equal data length %d", bytes, len(data))) } return nil } // compare the content between fileL and fileR func compareFiles(fileL string, fileR string) (bool, error) { finL, err := os.Open(fileL) if err != nil { return false, err } defer finL.Close() finR, err := os.Open(fileR) if err != nil { return false, err } defer finR.Close() statL, err := finL.Stat() if err != nil { return false, err } statR, err := finR.Stat() if err != nil { return false, err } if statL.Size() != statR.Size() { return false, nil } size := statL.Size() if size > 102400 { size = 102400 } bufL := make([]byte, size) bufR := make([]byte, size) for { n, _ := finL.Read(bufL) if 0 == n { break } n, _ = finR.Read(bufR) if 0 == n { break } if !bytes.Equal(bufL, bufR) { return false, nil } } return true, nil } // compare the content of file and data func compareFileData(file string, data []byte) (bool, error) { fin, err := os.Open(file) if err != nil { return false, err } defer fin.Close() stat, err := fin.Stat() if err != nil { return false, err } if stat.Size() != (int64)(len(data)) { return false, nil } buf := make([]byte, stat.Size()) n, err := fin.Read(buf) if err != nil { return false, err } if stat.Size() != (int64)(n) { return false, errors.New("read error") } if !bytes.Equal(buf, data) { return false, nil } return true, nil } func walkDir(dirPth, suffix string) ([]string, error) { var files = []string{} suffix = strings.ToUpper(suffix) err := filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error { if err != nil { return err } if fi.IsDir() { return nil } if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) { files = append(files, filename) } return nil }) return files, err } func removeTempFiles(path string, prefix string) error { files, err := walkDir(path, prefix) if err != nil { return nil } for _, file := range files { os.Remove(file) } return nil } func isFileExist(filename string) (bool, error) { _, err := os.Stat(filename) if err != nil && os.IsNotExist(err) { return false, nil } else if err != nil { return false, err } else { return true, nil } } // STS Server的GET请求返回的数据 type getSTSResult struct { Status int `json:"status"` // 返回状态码， 200表示获取成功，非200表示失败 AccessID string `json:"accessId"` //STS AccessId AccessKey string `json:"accessKey"` // STS AccessKey Expiration string `json:"expiration"` // STS Token SecurityToken string `json:"securityToken"` // Token失效的时间， GMT时间 Bucket string `json:"bucket"` // 可以使用的bucket Endpoint string `json:"bucket"` // 要访问的endpoint } // 从STS Server获取STS信息。返回值中当error为nil时，GetSTSResult有效。 func getSTSToken(STSServer string) (getSTSResult, error) { result := getSTSResult{} resp, err := http.Get(STSServer) if err != nil { return result, err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { return result, err } err = json.Unmarshal(body, &result) if err != nil { return result, err } if result.Status != 200 { return result, errors.New("Server Return Status:" + strconv.Itoa(result.Status)) } return result, nil } func readBody(body io.ReadCloser) (string, error) { data, err := ioutil.ReadAll(body) body.Close() if err != nil { return "", err } return string(data), nil } func (s *OssBucketSuite) getObject(objects []ObjectProperties, object string) (bool, ObjectProperties) { for _, v := range objects { if v.Key == object { return true, v } } return false, ObjectProperties{} } aliyun-oss-go-sdk-1.5.0/oss/client.go000066400000000000000000000561271311621456000174250ustar00rootroot00000000000000// Package oss implements functions for access oss service. // It has two main struct Client and Bucket. package oss import ( "bytes" "encoding/xml" "io" "net/http" "strings" "time" ) // // Client Sdk的入口，Client的方法可以完成bucket的各种操作，如create/delete bucket， // set/get acl/lifecycle/referer/logging/website等。文件(object)的上传下载通过Bucket完成。 // 用户用oss.New创建Client。 // type ( // Client oss client Client struct { Config *Config // Oss Client configure Conn *Conn // Send http request } // ClientOption client option such as UseCname, Timeout, SecurityToken. ClientOption func(*Client) ) // // New 生成一个新的Client。 // // endpoint 用户Bucket所在数据中心的访问域名，如http://oss-cn-hangzhou.aliyuncs.com。 // accessKeyId 用户标识。 // accessKeySecret 用户密钥。 // // Client 生成的新Client。error为nil时有效。 // error 操作无错误时为nil，非nil时表示操作出错。 // func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption) (*Client, error) { // configuration config := getDefaultOssConfig() config.Endpoint = endpoint config.AccessKeyID = accessKeyID config.AccessKeySecret = accessKeySecret // url parse url := &urlMaker{} url.Init(config.Endpoint, config.IsCname, config.IsUseProxy) // http connect conn := &Conn{config: config, url: url} // oss client client := &Client{ config, conn, } // client options parse for _, option := range options { option(client) } // create http connect err := conn.init(config, url) return client, err } // // Bucket 取存储空间（Bucket）的对象实例。 // // bucketName 存储空间名称。 // Bucket 新的Bucket。error为nil时有效。 // // error 操作无错误时返回nil，非nil为错误信息。 // func (client Client) Bucket(bucketName string) (*Bucket, error) { return &Bucket{ client, bucketName, }, nil } // // CreateBucket 创建Bucket。 // // bucketName bucket名称，在整个OSS中具有全局唯一性，且不能修改。bucket名称的只能包括小写字母，数字和短横线-， // 必须以小写字母或者数字开头，长度必须在3-255字节之间。 // options 创建bucket的选项。您可以使用选项ACL，指定bucket的访问权限。Bucket有以下三种访问权限，私有读写（ACLPrivate）、 // 公共读私有写（ACLPublicRead），公共读公共写(ACLPublicReadWrite)，默认访问权限是私有读写。可以使用StorageClass选项设置bucket的存储方式。 // // error 操作无错误时返回nil，非nil为错误信息。 // func (client Client) CreateBucket(bucketName string, options ...Option) error { headers := make(map[string]string) handleOptions(headers, options) buffer := new(bytes.Buffer) isOptSet, val, _ := isOptionSet(options, storageClass) if isOptSet { cbConfig := createBucketConfiguration{StorageClass: val.(StorageClassType)} bs, err := xml.Marshal(cbConfig) if err != nil { return err } buffer.Write(bs) contentType := http.DetectContentType(buffer.Bytes()) headers[HTTPHeaderContentType] = contentType } resp, err := client.do("PUT", bucketName, "", "", headers, buffer) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // ListBuckets 获取当前用户下的bucket。 // // options 指定ListBuckets的筛选行为，Prefix、Marker、MaxKeys三个选项。Prefix限定前缀。 // Marker设定从Marker之后的第一个开始返回。MaxKeys限定此次返回的最大数目，默认为100。 // 常用使用场景的实现，参数示例程序list_bucket.go。 // ListBucketsResponse 操作成功后的返回值，error为nil时该返回值有效。 // // error 操作无错误时返回nil，非nil为错误信息。 // func (client Client) ListBuckets(options ...Option) (ListBucketsResult, error) { var out ListBucketsResult params, err := handleParams(options) if err != nil { return out, err } resp, err := client.do("GET", "", params, "", nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // IsBucketExist Bucket是否存在。 // // bucketName 存储空间名称。 // // bool 存储空间是否存在。error为nil时有效。 // error 操作无错误时返回nil，非nil为错误信息。 // func (client Client) IsBucketExist(bucketName string) (bool, error) { listRes, err := client.ListBuckets(Prefix(bucketName), MaxKeys(1)) if err != nil { return false, err } if len(listRes.Buckets) == 1 && listRes.Buckets[0].Name == bucketName { return true, nil } return false, nil } // // DeleteBucket 删除空存储空间。非空时请先清理Object、Upload。 // // bucketName 存储空间名称。 // // error 操作无错误时返回nil，非nil为错误信息。 // func (client Client) DeleteBucket(bucketName string) error { resp, err := client.do("DELETE", bucketName, "", "", nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusNoContent}) } // // GetBucketLocation 查看Bucket所属数据中心位置的信息。 // // 如果您想了解"访问域名和数据中心"详细信息，请参看 // https://help.aliyun.com/document_detail/oss/user_guide/oss_concept/endpoint.html // // bucketName 存储空间名称。 // // string Bucket所属的数据中心位置信息。 // error 操作无错误时返回nil，非nil为错误信息。 // func (client Client) GetBucketLocation(bucketName string) (string, error) { resp, err := client.do("GET", bucketName, "location", "location", nil, nil) if err != nil { return "", err } defer resp.Body.Close() var LocationConstraint string err = xmlUnmarshal(resp.Body, &LocationConstraint) return LocationConstraint, err } // // SetBucketACL 修改Bucket的访问权限。 // // bucketName 存储空间名称。 // bucketAcl bucket的访问权限。Bucket有以下三种访问权限，Bucket有以下三种访问权限，私有读写（ACLPrivate）、 // 公共读私有写（ACLPublicRead），公共读公共写(ACLPublicReadWrite)。 // // error 操作无错误时返回nil，非nil为错误信息。 // func (client Client) SetBucketACL(bucketName string, bucketACL ACLType) error { headers := map[string]string{HTTPHeaderOssACL: string(bucketACL)} resp, err := client.do("PUT", bucketName, "", "", headers, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // GetBucketACL 获得Bucket的访问权限。 // // bucketName 存储空间名称。 // // GetBucketAclResponse 操作成功后的返回值，error为nil时该返回值有效。 // error 操作无错误时返回nil，非nil为错误信息。 // func (client Client) GetBucketACL(bucketName string) (GetBucketACLResult, error) { var out GetBucketACLResult resp, err := client.do("GET", bucketName, "acl", "acl", nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // SetBucketLifecycle 修改Bucket的生命周期设置。 // // OSS提供Object生命周期管理来为用户管理对象。用户可以为某个Bucket定义生命周期配置，来为该Bucket的Object定义各种规则。 // Bucket的拥有者可以通过SetBucketLifecycle来设置Bucket的Lifecycle配置。Lifecycle开启后，OSS将按照配置， // 定期自动删除与Lifecycle规则相匹配的Object。如果您想了解更多的生命周期的信息，请参看 // https://help.aliyun.com/document_detail/oss/user_guide/manage_object/object_lifecycle.html // // bucketName 存储空间名称。 // rules 生命周期规则列表。生命周期规则有两种格式，指定绝对和相对过期时间，分布由days和year/month/day控制。 // 具体用法请参考示例程序sample/bucket_lifecycle.go。 // // error 操作无错误时返回error为nil，非nil为错误信息。 // func (client Client) SetBucketLifecycle(bucketName string, rules []LifecycleRule) error { lxml := lifecycleXML{Rules: convLifecycleRule(rules)} bs, err := xml.Marshal(lxml) if err != nil { return err } buffer := new(bytes.Buffer) buffer.Write(bs) contentType := http.DetectContentType(buffer.Bytes()) headers := map[string]string{} headers[HTTPHeaderContentType] = contentType resp, err := client.do("PUT", bucketName, "lifecycle", "lifecycle", headers, buffer) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // DeleteBucketLifecycle 删除Bucket的生命周期设置。 // // // bucketName 存储空间名称。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) DeleteBucketLifecycle(bucketName string) error { resp, err := client.do("DELETE", bucketName, "lifecycle", "lifecycle", nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusNoContent}) } // // GetBucketLifecycle 查看Bucket的生命周期设置。 // // bucketName 存储空间名称。 // // GetBucketLifecycleResponse 操作成功的返回值，error为nil时该返回值有效。Rules为该bucket上的规则列表。 // error 操作无错误时为nil，非nil为错误信息。 // func (client Client) GetBucketLifecycle(bucketName string) (GetBucketLifecycleResult, error) { var out GetBucketLifecycleResult resp, err := client.do("GET", bucketName, "lifecycle", "lifecycle", nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // SetBucketReferer 设置bucket的referer访问白名单和是否允许referer字段为空的请求访问。 // // 防止用户在OSS上的数据被其他人盗用，OSS支持基于HTTP header中表头字段referer的防盗链方法。可以通过OSS控制台或者API的方式对 // 一个bucket设置referer字段的白名单和是否允许referer字段为空的请求访问。例如，对于一个名为oss-example的bucket， // 设置其referer白名单为http://www.aliyun.com。则所有referer为http://www.aliyun.com的请求才能访问oss-example // 这个bucket中的object。如果您还需要了解更多信息，请参看 // https://help.aliyun.com/document_detail/oss/user_guide/security_management/referer.html // // bucketName 存储空间名称。 // referers 访问白名单列表。一个bucket可以支持多个referer参数。referer参数支持通配符"*"和"?"。 // 用法请参看示例sample/bucket_referer.go // allowEmptyReferer 指定是否允许referer字段为空的请求访问。 默认为true。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) SetBucketReferer(bucketName string, referers []string, allowEmptyReferer bool) error { rxml := RefererXML{} rxml.AllowEmptyReferer = allowEmptyReferer if referers == nil { rxml.RefererList = append(rxml.RefererList, "") } else { for _, referer := range referers { rxml.RefererList = append(rxml.RefererList, referer) } } bs, err := xml.Marshal(rxml) if err != nil { return err } buffer := new(bytes.Buffer) buffer.Write(bs) contentType := http.DetectContentType(buffer.Bytes()) headers := map[string]string{} headers[HTTPHeaderContentType] = contentType resp, err := client.do("PUT", bucketName, "referer", "referer", headers, buffer) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // GetBucketReferer 获得Bucket的白名单地址。 // // bucketName 存储空间名称。 // // GetBucketRefererResponse 操作成功的返回值，error为nil时该返回值有效。 // error 操作无错误时为nil，非nil为错误信息。 // func (client Client) GetBucketReferer(bucketName string) (GetBucketRefererResult, error) { var out GetBucketRefererResult resp, err := client.do("GET", bucketName, "referer", "referer", nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // SetBucketLogging 修改Bucket的日志设置。 // // OSS为您提供自动保存访问日志记录功能。Bucket的拥有者可以开启访问日志记录功能。当一个bucket开启访问日志记录功能后， // OSS自动将访问这个bucket的请求日志，以小时为单位，按照固定的命名规则，生成一个Object写入用户指定的bucket中。 // 如果您需要更多，请参看 https://help.aliyun.com/document_detail/oss/user_guide/security_management/logging.html // // bucketName 需要记录访问日志的Bucket。 // targetBucket 访问日志记录到的Bucket。 // targetPrefix bucketName中需要存储访问日志记录的object前缀。为空记录所有object的访问日志。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) SetBucketLogging(bucketName, targetBucket, targetPrefix string, isEnable bool) error { var err error var bs []byte if isEnable { lxml := LoggingXML{} lxml.LoggingEnabled.TargetBucket = targetBucket lxml.LoggingEnabled.TargetPrefix = targetPrefix bs, err = xml.Marshal(lxml) } else { lxml := loggingXMLEmpty{} bs, err = xml.Marshal(lxml) } if err != nil { return err } buffer := new(bytes.Buffer) buffer.Write(bs) contentType := http.DetectContentType(buffer.Bytes()) headers := map[string]string{} headers[HTTPHeaderContentType] = contentType resp, err := client.do("PUT", bucketName, "logging", "logging", headers, buffer) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // DeleteBucketLogging 删除Bucket的日志设置。 // // bucketName 需要删除访问日志的Bucket。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) DeleteBucketLogging(bucketName string) error { resp, err := client.do("DELETE", bucketName, "logging", "logging", nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusNoContent}) } // // GetBucketLogging 获得Bucket的日志设置。 // // bucketName 需要删除访问日志的Bucket。 // GetBucketLoggingResponse 操作成功的返回值，error为nil时该返回值有效。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) GetBucketLogging(bucketName string) (GetBucketLoggingResult, error) { var out GetBucketLoggingResult resp, err := client.do("GET", bucketName, "logging", "logging", nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // SetBucketWebsite 设置/修改Bucket的默认首页以及错误页。 // // OSS支持静态网站托管，Website操作可以将一个bucket设置成静态网站托管模式 。您可以将自己的Bucket配置成静态网站托管模式。 // 如果您需要更多，请参看 https://help.aliyun.com/document_detail/oss/user_guide/static_host_website.html // // bucketName 需要设置Website的Bucket。 // indexDocument 索引文档。 // errorDocument 错误文档。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) SetBucketWebsite(bucketName, indexDocument, errorDocument string) error { wxml := WebsiteXML{} wxml.IndexDocument.Suffix = indexDocument wxml.ErrorDocument.Key = errorDocument bs, err := xml.Marshal(wxml) if err != nil { return err } buffer := new(bytes.Buffer) buffer.Write(bs) contentType := http.DetectContentType(buffer.Bytes()) headers := make(map[string]string) headers[HTTPHeaderContentType] = contentType resp, err := client.do("PUT", bucketName, "website", "website", headers, buffer) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // DeleteBucketWebsite 删除Bucket的Website设置。 // // bucketName 需要删除website设置的Bucket。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) DeleteBucketWebsite(bucketName string) error { resp, err := client.do("DELETE", bucketName, "website", "website", nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusNoContent}) } // // GetBucketWebsite 获得Bucket的默认首页以及错误页。 // // bucketName 存储空间名称。 // // GetBucketWebsiteResponse 操作成功的返回值，error为nil时该返回值有效。 // error 操作无错误为nil，非nil为错误信息。 // func (client Client) GetBucketWebsite(bucketName string) (GetBucketWebsiteResult, error) { var out GetBucketWebsiteResult resp, err := client.do("GET", bucketName, "website", "website", nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // SetBucketCORS 设置Bucket的跨域访问(CORS)规则。 // // 跨域访问的更多信息，请参看 https://help.aliyun.com/document_detail/oss/user_guide/security_management/cors.html // // bucketName 需要设置Website的Bucket。 // corsRules 待设置的CORS规则。用法请参看示例代码sample/bucket_cors.go。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) SetBucketCORS(bucketName string, corsRules []CORSRule) error { corsxml := CORSXML{} for _, v := range corsRules { cr := CORSRule{} cr.AllowedMethod = v.AllowedMethod cr.AllowedOrigin = v.AllowedOrigin cr.AllowedHeader = v.AllowedHeader cr.ExposeHeader = v.ExposeHeader cr.MaxAgeSeconds = v.MaxAgeSeconds corsxml.CORSRules = append(corsxml.CORSRules, cr) } bs, err := xml.Marshal(corsxml) if err != nil { return err } buffer := new(bytes.Buffer) buffer.Write(bs) contentType := http.DetectContentType(buffer.Bytes()) headers := map[string]string{} headers[HTTPHeaderContentType] = contentType resp, err := client.do("PUT", bucketName, "cors", "cors", headers, buffer) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusOK}) } // // DeleteBucketCORS 删除Bucket的Website设置。 // // bucketName 需要删除cors设置的Bucket。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) DeleteBucketCORS(bucketName string) error { resp, err := client.do("DELETE", bucketName, "cors", "cors", nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusNoContent}) } // // GetBucketCORS 获得Bucket的CORS设置。 // // // bucketName 存储空间名称。 // GetBucketCORSResult 操作成功的返回值，error为nil时该返回值有效。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) GetBucketCORS(bucketName string) (GetBucketCORSResult, error) { var out GetBucketCORSResult resp, err := client.do("GET", bucketName, "cors", "cors", nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // GetBucketInfo 获得Bucket的信息。 // // bucketName 存储空间名称。 // GetBucketInfoResult 操作成功的返回值，error为nil时该返回值有效。 // // error 操作无错误为nil，非nil为错误信息。 // func (client Client) GetBucketInfo(bucketName string) (GetBucketInfoResult, error) { var out GetBucketInfoResult resp, err := client.do("GET", bucketName, "bucketInfo", "bucketInfo", nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // UseCname 设置是否使用CNAME，默认不使用。 // // isUseCname true设置endpoint格式是cname格式，false为非cname格式，默认false // func UseCname(isUseCname bool) ClientOption { return func(client *Client) { client.Config.IsCname = isUseCname client.Conn.url.Init(client.Config.Endpoint, client.Config.IsCname, client.Config.IsUseProxy) } } // // Timeout 设置HTTP超时时间。 // // connectTimeoutSec HTTP链接超时时间，单位是秒，默认10秒。0表示永不超时。 // readWriteTimeout HTTP发送接受数据超时时间，单位是秒，默认20秒。0表示永不超时。 // func Timeout(connectTimeoutSec, readWriteTimeout int64) ClientOption { return func(client *Client) { client.Config.HTTPTimeout.ConnectTimeout = time.Second * time.Duration(connectTimeoutSec) client.Config.HTTPTimeout.ReadWriteTimeout = time.Second * time.Duration(readWriteTimeout) client.Config.HTTPTimeout.HeaderTimeout = time.Second * time.Duration(readWriteTimeout) client.Config.HTTPTimeout.LongTimeout = time.Second * time.Duration(readWriteTimeout*10) } } // // SecurityToken 临时用户设置SecurityToken。 // // token STS token // func SecurityToken(token string) ClientOption { return func(client *Client) { client.Config.SecurityToken = strings.TrimSpace(token) } } // // EnableMD5 是否启用MD5校验，默认启用。 // // isEnableMD5 true启用MD5校验，false不启用MD5校验 // func EnableMD5(isEnableMD5 bool) ClientOption { return func(client *Client) { client.Config.IsEnableMD5 = isEnableMD5 } } // // MD5ThresholdCalcInMemory 使用内存计算MD5值的上限，默认16MB。 // // threshold 单位Byte。上传内容小于threshold在MD5在内存中计算，大于使用临时文件计算MD5 // func MD5ThresholdCalcInMemory(threshold int64) ClientOption { return func(client *Client) { client.Config.MD5Threshold = threshold } } // // EnableCRC 上传是否启用CRC校验，默认启用。 // // isEnableCRC true启用CRC校验，false不启用CRC校验 // func EnableCRC(isEnableCRC bool) ClientOption { return func(client *Client) { client.Config.IsEnableCRC = isEnableCRC } } // // UserAgent 指定UserAgent，默认如下aliyun-sdk-go/1.2.0 (windows/-/amd64;go1.5.2)。 // // userAgent user agent字符串。 // func UserAgent(userAgent string) ClientOption { return func(client *Client) { client.Config.UserAgent = userAgent } } // // Proxy 设置代理服务器，默认不使用代理。 // // proxyHost 代理服务器地址，格式是host或host:port // func Proxy(proxyHost string) ClientOption { return func(client *Client) { client.Config.IsUseProxy = true client.Config.ProxyHost = proxyHost client.Conn.url.Init(client.Config.Endpoint, client.Config.IsCname, client.Config.IsUseProxy) } } // // AuthProxy 设置需要认证的代理服务器，默认不使用代理。 // // proxyHost 代理服务器地址，格式是host或host:port // proxyUser 代理服务器认证的用户名 // proxyPassword 代理服务器认证的用户密码 // func AuthProxy(proxyHost, proxyUser, proxyPassword string) ClientOption { return func(client *Client) { client.Config.IsUseProxy = true client.Config.ProxyHost = proxyHost client.Config.IsAuthProxy = true client.Config.ProxyUser = proxyUser client.Config.ProxyPassword = proxyPassword client.Conn.url.Init(client.Config.Endpoint, client.Config.IsCname, client.Config.IsUseProxy) } } // Private func (client Client) do(method, bucketName, urlParams, subResource string, headers map[string]string, data io.Reader) (*Response, error) { return client.Conn.Do(method, bucketName, "", urlParams, subResource, headers, data, 0, nil) } aliyun-oss-go-sdk-1.5.0/oss/client_test.go000066400000000000000000001166401311621456000204610ustar00rootroot00000000000000// client test // use gocheck, install gocheck to execute "go get gopkg.in/check.v1", // see https://labix.org/gocheck package oss import ( "log" "math/rand" "os" "strings" "testing" "time" . "gopkg.in/check.v1" ) // Hook up gocheck into the "go test" runner. func Test(t *testing.T) { TestingT(t) } type OssClientSuite struct{} var _ = Suite(&OssClientSuite{}) var ( // endpoint/id/key endpoint = os.Getenv("OSS_TEST_ENDPOINT") accessID = os.Getenv("OSS_TEST_ACCESS_KEY_ID") accessKey = os.Getenv("OSS_TEST_ACCESS_KEY_SECRET") // proxy proxyHost = os.Getenv("OSS_TEST_PROXY_HOST") proxyUser = os.Getenv("OSS_TEST_PROXY_USER") proxyPasswd = os.Getenv("OSS_TEST_PROXY_PASSWORD") // sts stsaccessID = os.Getenv("OSS_TEST_STS_ID") stsaccessKey = os.Getenv("OSS_TEST_STS_KEY") stsARN = os.Getenv("OSS_TEST_STS_ARN") ) const ( // prefix of bucket name for bucket ops test bucketNamePrefix = "go-sdk-test-bucket-xyz-" // bucket name for object ops test bucketName = "go-sdk-test-bucket-xyz-for-object" archiveBucketName = "go-sdk-test-bucket-xyz-for-archive" // object name for object ops test objectNamePrefix = "go-sdk-test-object-xyz-" // sts region is one and only hangzhou stsRegion = "cn-hangzhou" ) var ( logPath = "go_sdk_test_" + time.Now().Format("20060102_150405") + ".log" testLogFile, _ = os.OpenFile(logPath, os.O_RDWR|os.O_CREATE, 0664) testLogger = log.New(testLogFile, "", log.Ldate|log.Ltime|log.Lshortfile) letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") ) func randStr(n int) string { b := make([]rune, n) r := rand.New(rand.NewSource(time.Now().UnixNano())) for i := range b { b[i] = letters[r.Intn(len(letters))] } return string(b) } func randLowStr(n int) string { return strings.ToLower(randStr(n)) } // Run once when the suite starts running func (s *OssClientSuite) SetUpSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) lbr, err := client.ListBuckets(Prefix(bucketNamePrefix), MaxKeys(1000)) c.Assert(err, IsNil) for _, bucket := range lbr.Buckets { s.deleteBucket(client, bucket.Name, c) } testLogger.Println("test client started") } // Run before each test or benchmark starts running func (s *OssClientSuite) TearDownSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) lbr, err := client.ListBuckets(Prefix(bucketNamePrefix), MaxKeys(1000)) c.Assert(err, IsNil) for _, bucket := range lbr.Buckets { s.deleteBucket(client, bucket.Name, c) } testLogger.Println("test client completed") } func (s *OssClientSuite) deleteBucket(client *Client, bucketName string, c *C) { bucket, err := client.Bucket(bucketName) c.Assert(err, IsNil) // Delete Object lor, err := bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } // Delete Part lmur, err := bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmur.Uploads { var imur = InitiateMultipartUploadResult{Bucket: bucketName, Key: upload.Key, UploadID: upload.UploadID} err = bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } // Delete Bucket err = client.DeleteBucket(bucketName) c.Assert(err, IsNil) } // Run after each test or benchmark runs func (s *OssClientSuite) SetUpTest(c *C) { } // Run once after all tests or benchmarks have finished running func (s *OssClientSuite) TearDownTest(c *C) { } // TestCreateBucket func (s *OssClientSuite) TestCreateBucket(c *C) { var bucketNameTest = bucketNamePrefix + "tcb" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // Create client.DeleteBucket(bucketNameTest) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // Check lbr, err := client.ListBuckets() c.Assert(err, IsNil) found := s.checkBucket(lbr.Buckets, bucketNameTest) c.Assert(found, Equals, true) time.Sleep(5 * time.Second) res, err := client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) // Create with ACLPublicRead err = client.CreateBucket(bucketNameTest, ACL(ACLPublicRead)) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPublicRead)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) // ACLPublicReadWrite err = client.CreateBucket(bucketNameTest, ACL(ACLPublicReadWrite)) c.Assert(err, IsNil) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPublicReadWrite)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) // ACLPrivate err = client.CreateBucket(bucketNameTest, ACL(ACLPrivate)) c.Assert(err, IsNil) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) // Delete err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) // create bucket with config and test get bucket info for _, storage := range []StorageClassType{StorageStandard, StorageIA, StorageArchive} { bucketNameTest := bucketNamePrefix + randLowStr(5) err = client.CreateBucket(bucketNameTest, StorageClass(storage), ACL(ACLPublicRead)) c.Assert(err, IsNil) res, err := client.GetBucketInfo(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.BucketInfo.Name, Equals, bucketNameTest) c.Assert(res.BucketInfo.StorageClass, Equals, string(storage)) c.Assert(res.BucketInfo.ACL, Equals, string(ACLPublicRead)) // Delete err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // error put bucket with config err = client.CreateBucket("ERRORBUCKETNAME", StorageClass(StorageArchive)) c.Assert(err, NotNil) // create bucket with config and test list bucket for _, storage := range []StorageClassType{StorageStandard, StorageIA, StorageArchive} { bucketNameTest := bucketNamePrefix + randLowStr(5) err = client.CreateBucket(bucketNameTest, StorageClass(storage)) c.Assert(err, IsNil) res, err := client.ListBuckets() c.Assert(err, IsNil) exist, b := s.getBucket(res.Buckets, bucketNameTest) c.Assert(exist, Equals, true) c.Assert(b.Name, Equals, bucketNameTest) c.Assert(b.StorageClass, Equals, string(storage)) // Delete err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } } // TestCreateBucketNegative func (s *OssClientSuite) TestCreateBucketNegative(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // BucketName invalid err = client.CreateBucket("xx") c.Assert(err, NotNil) err = client.CreateBucket("XXXX") c.Assert(err, NotNil) testLogger.Println(err) err = client.CreateBucket("_bucket") c.Assert(err, NotNil) testLogger.Println(err) // Acl invalid err = client.CreateBucket(bucketNamePrefix+"tcbn", ACL("InvaldAcl")) c.Assert(err, NotNil) testLogger.Println(err) } // TestDeleteBucket func (s *OssClientSuite) TestDeleteBucket(c *C) { var bucketNameTest = bucketNamePrefix + "tdb" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // Create err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // Check lbr, err := client.ListBuckets() c.Assert(err, IsNil) found := s.checkBucket(lbr.Buckets, bucketNameTest) c.Assert(found, Equals, true) // Delete err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(time.Second * 1) // Check lbr, err = client.ListBuckets() c.Assert(err, IsNil) // Sometimes failed because of cache found = s.checkBucket(lbr.Buckets, bucketNameTest) // c.Assert(found, Equals, false) err = client.DeleteBucket(bucketNameTest) // c.Assert(err, IsNil) } // TestDeleteBucketNegative func (s *OssClientSuite) TestDeleteBucketNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tdbn" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // BucketName invalid err = client.DeleteBucket("xx") c.Assert(err, NotNil) err = client.DeleteBucket("XXXX") c.Assert(err, NotNil) err = client.DeleteBucket("_bucket") c.Assert(err, NotNil) // Delete no exist err = client.DeleteBucket("notexist") c.Assert(err, NotNil) // No permission to delete, this ak/sk for js sdk err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) accessID := "" accessKey := "" clientOtherUser, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = clientOtherUser.DeleteBucket(bucketNameTest) c.Assert(err, NotNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestListBucket func (s *OssClientSuite) TestListBucket(c *C) { var bucketNameLbOne = bucketNamePrefix + "tlb1" var bucketNameLbTwo = bucketNamePrefix + "tlb2" var bucketNameLbThree = bucketNamePrefix + "tlb3" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // CreateBucket err = client.CreateBucket(bucketNameLbOne) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameLbTwo) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameLbThree) c.Assert(err, IsNil) // ListBuckets, specified prefix lbr, err := client.ListBuckets(Prefix(bucketNamePrefix), MaxKeys(2)) c.Assert(err, IsNil) c.Assert(len(lbr.Buckets), Equals, 2) // ListBuckets, specified max keys lbr, err = client.ListBuckets(MaxKeys(2)) c.Assert(err, IsNil) c.Assert(len(lbr.Buckets), Equals, 2) // ListBuckets, specified max keys lbr, err = client.ListBuckets(Marker(bucketNameLbOne), MaxKeys(1)) c.Assert(err, IsNil) c.Assert(len(lbr.Buckets), Equals, 1) // ListBuckets, specified max keys lbr, err = client.ListBuckets(Marker(bucketNameLbOne)) c.Assert(err, IsNil) c.Assert(len(lbr.Buckets) >= 2, Equals, true) // DeleteBucket err = client.DeleteBucket(bucketNameLbOne) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameLbTwo) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameLbThree) c.Assert(err, IsNil) } // TestListBucket func (s *OssClientSuite) TestIsBucketExist(c *C) { var bucketNameLbOne = bucketNamePrefix + "tibe1" var bucketNameLbTwo = bucketNamePrefix + "tibe11" var bucketNameLbThree = bucketNamePrefix + "tibe111" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // CreateBucket err = client.CreateBucket(bucketNameLbOne) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameLbTwo) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameLbThree) c.Assert(err, IsNil) // exist exist, err := client.IsBucketExist(bucketNameLbTwo) c.Assert(err, IsNil) c.Assert(exist, Equals, true) exist, err = client.IsBucketExist(bucketNameLbThree) c.Assert(err, IsNil) c.Assert(exist, Equals, true) exist, err = client.IsBucketExist(bucketNameLbOne) c.Assert(err, IsNil) c.Assert(exist, Equals, true) // not exist exist, err = client.IsBucketExist(bucketNamePrefix + "tibe") c.Assert(err, IsNil) c.Assert(exist, Equals, false) exist, err = client.IsBucketExist(bucketNamePrefix + "tibe1111") c.Assert(err, IsNil) c.Assert(exist, Equals, false) // negative exist, err = client.IsBucketExist("BucketNameInvalid") c.Assert(err, NotNil) // DeleteBucket err = client.DeleteBucket(bucketNameLbOne) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameLbTwo) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameLbThree) c.Assert(err, IsNil) } // TestSetBucketAcl func (s *OssClientSuite) TestSetBucketAcl(c *C) { var bucketNameTest = bucketNamePrefix + "tsba" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // Private err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) res, err := client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) // set ACL_PUBLIC_R err = client.SetBucketACL(bucketNameTest, ACLPublicRead) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPublicRead)) // set ACL_PUBLIC_RW err = client.SetBucketACL(bucketNameTest, ACLPublicReadWrite) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPublicReadWrite)) // set ACL_PUBLIC_RW err = client.SetBucketACL(bucketNameTest, ACLPrivate) c.Assert(err, IsNil) err = client.SetBucketACL(bucketNameTest, ACLPrivate) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestSetBucketAclNegative func (s *OssClientSuite) TestBucketAclNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tsban" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) err = client.SetBucketACL(bucketNameTest, "InvalidACL") c.Assert(err, NotNil) testLogger.Println(err) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestGetBucketAcl func (s *OssClientSuite) TestGetBucketAcl(c *C) { var bucketNameTest = bucketNamePrefix + "tgba" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // Private err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) res, err := client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) // PublicRead err = client.CreateBucket(bucketNameTest, ACL(ACLPublicRead)) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPublicRead)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) // PublicReadWrite err = client.CreateBucket(bucketNameTest, ACL(ACLPublicReadWrite)) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPublicReadWrite)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestGetBucketAcl func (s *OssClientSuite) TestGetBucketLocation(c *C) { var bucketNameTest = bucketNamePrefix + "tgbl" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // Private err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) loc, err := client.GetBucketLocation(bucketNameTest) c.Assert(strings.HasPrefix(loc, "oss-"), Equals, true) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestGetBucketLocationNegative func (s *OssClientSuite) TestGetBucketLocationNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tgblg" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // not exist _, err = client.GetBucketLocation(bucketNameTest) c.Assert(err, NotNil) // not exist _, err = client.GetBucketLocation("InvalidBucketName_") c.Assert(err, NotNil) } // TestSetBucketLifecycle func (s *OssClientSuite) TestSetBucketLifecycle(c *C) { var bucketNameTest = bucketNamePrefix + "tsbl" var rule1 = BuildLifecycleRuleByDate("idone", "one", true, 2015, 11, 11) var rule2 = BuildLifecycleRuleByDays("idtwo", "two", true, 3) client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // set single rule var rules = []LifecycleRule{rule1} err = client.SetBucketLifecycle(bucketNameTest, rules) c.Assert(err, IsNil) // double set rule err = client.SetBucketLifecycle(bucketNameTest, rules) c.Assert(err, IsNil) res, err := client.GetBucketLifecycle(bucketNameTest) c.Assert(err, IsNil) c.Assert(len(res.Rules), Equals, 1) c.Assert(res.Rules[0].ID, Equals, "idone") err = client.DeleteBucketLifecycle(bucketNameTest) c.Assert(err, IsNil) // set two rules rules = []LifecycleRule{rule1, rule2} err = client.SetBucketLifecycle(bucketNameTest, rules) c.Assert(err, IsNil) // eliminate effect of cache time.Sleep(5 * time.Second) res, err = client.GetBucketLifecycle(bucketNameTest) c.Assert(err, IsNil) c.Assert(len(res.Rules), Equals, 2) c.Assert(res.Rules[0].ID, Equals, "idone") c.Assert(res.Rules[1].ID, Equals, "idtwo") err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestDeleteBucketLifecycle func (s *OssClientSuite) TestDeleteBucketLifecycle(c *C) { var bucketNameTest = bucketNamePrefix + "tdbl" var rule1 = BuildLifecycleRuleByDate("idone", "one", true, 2015, 11, 11) var rule2 = BuildLifecycleRuleByDays("idtwo", "two", true, 3) var rules = []LifecycleRule{rule1, rule2} client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucketLifecycle(bucketNameTest) c.Assert(err, IsNil) err = client.SetBucketLifecycle(bucketNameTest, rules) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err := client.GetBucketLifecycle(bucketNameTest) c.Assert(err, IsNil) c.Assert(len(res.Rules), Equals, 2) // delete err = client.DeleteBucketLifecycle(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketLifecycle(bucketNameTest) c.Assert(err, NotNil) // eliminate effect of cache time.Sleep(time.Second * 3) // delete when not set err = client.DeleteBucketLifecycle(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestSetBucketLifecycleNegative func (s *OssClientSuite) TestBucketLifecycleNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tsbln" var rules = []LifecycleRule{} client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // set with no rule err = client.SetBucketLifecycle(bucketNameTest, rules) c.Assert(err, NotNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) // not exist err = client.SetBucketLifecycle(bucketNameTest, rules) c.Assert(err, NotNil) // not exist _, err = client.GetBucketLifecycle(bucketNameTest) c.Assert(err, NotNil) // not exist err = client.DeleteBucketLifecycle(bucketNameTest) c.Assert(err, NotNil) } // TestSetBucketReferer func (s *OssClientSuite) TestSetBucketReferer(c *C) { var bucketNameTest = bucketNamePrefix + "tsbr" var referers = []string{"http://www.aliyun.com", "https://www.aliyun.com"} client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) res, err := client.GetBucketReferer(bucketNameTest) c.Assert(res.AllowEmptyReferer, Equals, true) c.Assert(len(res.RefererList), Equals, 0) // set referers err = client.SetBucketReferer(bucketNameTest, referers, false) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketReferer(bucketNameTest) c.Assert(res.AllowEmptyReferer, Equals, false) c.Assert(len(res.RefererList), Equals, 2) c.Assert(res.RefererList[0], Equals, "http://www.aliyun.com") c.Assert(res.RefererList[1], Equals, "https://www.aliyun.com") // reset referer, referers empty referers = []string{""} err = client.SetBucketReferer(bucketNameTest, referers, true) c.Assert(err, IsNil) referers = []string{} err = client.SetBucketReferer(bucketNameTest, referers, true) c.Assert(err, IsNil) res, err = client.GetBucketReferer(bucketNameTest) c.Assert(res.AllowEmptyReferer, Equals, true) c.Assert(len(res.RefererList), Equals, 0) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestSetBucketRefererNegative func (s *OssClientSuite) TestBucketRefererNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tsbrn" var referers = []string{""} client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // not exist _, err = client.GetBucketReferer(bucketNameTest) c.Assert(err, NotNil) testLogger.Println(err) // not exist err = client.SetBucketReferer(bucketNameTest, referers, true) c.Assert(err, NotNil) testLogger.Println(err) } // TestSetBucketLogging func (s *OssClientSuite) TestSetBucketLogging(c *C) { var bucketNameTest = bucketNamePrefix + "tsbll" var bucketNameTarget = bucketNamePrefix + "tsbllt" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTarget) c.Assert(err, IsNil) time.Sleep(5 * time.Second) // set logging err = client.SetBucketLogging(bucketNameTest, bucketNameTarget, "prefix", true) c.Assert(err, IsNil) // reset err = client.SetBucketLogging(bucketNameTest, bucketNameTarget, "prefix", false) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err := client.GetBucketLogging(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.LoggingEnabled.TargetBucket, Equals, "") c.Assert(res.LoggingEnabled.TargetPrefix, Equals, "") err = client.DeleteBucketLogging(bucketNameTest) c.Assert(err, IsNil) // set to self err = client.SetBucketLogging(bucketNameTest, bucketNameTest, "prefix", true) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTarget) c.Assert(err, IsNil) } // TestDeleteBucketLogging func (s *OssClientSuite) TestDeleteBucketLogging(c *C) { var bucketNameTest = bucketNamePrefix + "tdbl" var bucketNameTarget = bucketNamePrefix + "tdblt" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTarget) c.Assert(err, IsNil) // get when not set res, err := client.GetBucketLogging(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.LoggingEnabled.TargetBucket, Equals, "") c.Assert(res.LoggingEnabled.TargetPrefix, Equals, "") // set err = client.SetBucketLogging(bucketNameTest, bucketNameTarget, "prefix", true) c.Assert(err, IsNil) // get time.Sleep(5 * time.Second) res, err = client.GetBucketLogging(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.LoggingEnabled.TargetBucket, Equals, bucketNameTarget) c.Assert(res.LoggingEnabled.TargetPrefix, Equals, "prefix") // set err = client.SetBucketLogging(bucketNameTest, bucketNameTarget, "prefix", false) c.Assert(err, IsNil) // get time.Sleep(5 * time.Second) res, err = client.GetBucketLogging(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.LoggingEnabled.TargetBucket, Equals, "") c.Assert(res.LoggingEnabled.TargetPrefix, Equals, "") // delete err = client.DeleteBucketLogging(bucketNameTest) c.Assert(err, IsNil) // get after delete time.Sleep(5 * time.Second) res, err = client.GetBucketLogging(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.LoggingEnabled.TargetBucket, Equals, "") c.Assert(res.LoggingEnabled.TargetPrefix, Equals, "") err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTarget) c.Assert(err, IsNil) } // TestSetBucketLoggingNegative func (s *OssClientSuite) TestSetBucketLoggingNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tsblnn" var bucketNameTarget = bucketNamePrefix + "tsblnnt" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // not exist _, err = client.GetBucketLogging(bucketNameTest) c.Assert(err, NotNil) // not exist err = client.SetBucketLogging(bucketNameTest, "targetbucket", "prefix", true) c.Assert(err, NotNil) // not exist err = client.DeleteBucketLogging(bucketNameTest) c.Assert(err, NotNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) // target bucket not exist err = client.SetBucketLogging(bucketNameTest, bucketNameTarget, "prefix", true) c.Assert(err, NotNil) // parameter invalid err = client.SetBucketLogging(bucketNameTest, "XXXX", "prefix", true) c.Assert(err, NotNil) err = client.SetBucketLogging(bucketNameTest, "xx", "prefix", true) c.Assert(err, NotNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestSetBucketWebsite func (s *OssClientSuite) TestSetBucketWebsite(c *C) { var bucketNameTest = bucketNamePrefix + "tsbw" var indexWebsite = "myindex.html" var errorWebsite = "myerror.html" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // set err = client.SetBucketWebsite(bucketNameTest, indexWebsite, errorWebsite) c.Assert(err, IsNil) // double set err = client.SetBucketWebsite(bucketNameTest, indexWebsite, errorWebsite) c.Assert(err, IsNil) res, err := client.GetBucketWebsite(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.IndexDocument.Suffix, Equals, indexWebsite) c.Assert(res.ErrorDocument.Key, Equals, errorWebsite) // reset err = client.SetBucketWebsite(bucketNameTest, "your"+indexWebsite, "your"+errorWebsite) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketWebsite(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.IndexDocument.Suffix, Equals, "your"+indexWebsite) c.Assert(res.ErrorDocument.Key, Equals, "your"+errorWebsite) err = client.DeleteBucketWebsite(bucketNameTest) c.Assert(err, IsNil) // set after delete err = client.SetBucketWebsite(bucketNameTest, indexWebsite, errorWebsite) c.Assert(err, IsNil) // eliminate effect of cache time.Sleep(5 * time.Second) res, err = client.GetBucketWebsite(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.IndexDocument.Suffix, Equals, indexWebsite) c.Assert(res.ErrorDocument.Key, Equals, errorWebsite) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestDeleteBucketWebsite func (s *OssClientSuite) TestDeleteBucketWebsite(c *C) { var bucketNameTest = bucketNamePrefix + "tdbw" var indexWebsite = "myindex.html" var errorWebsite = "myerror.html" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // get res, err := client.GetBucketWebsite(bucketNameTest) c.Assert(err, NotNil) // detele without set err = client.DeleteBucketWebsite(bucketNameTest) c.Assert(err, IsNil) // set err = client.SetBucketWebsite(bucketNameTest, indexWebsite, errorWebsite) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketWebsite(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.IndexDocument.Suffix, Equals, indexWebsite) c.Assert(res.ErrorDocument.Key, Equals, errorWebsite) // detele time.Sleep(5 * time.Second) err = client.DeleteBucketWebsite(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketWebsite(bucketNameTest) c.Assert(err, NotNil) // detele after delete err = client.DeleteBucketWebsite(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestSetBucketWebsiteNegative func (s *OssClientSuite) TestSetBucketWebsiteNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tdbw" var indexWebsite = "myindex.html" var errorWebsite = "myerror.html" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) // not exist _, err = client.GetBucketWebsite(bucketNameTest) c.Assert(err, NotNil) err = client.DeleteBucketWebsite(bucketNameTest) c.Assert(err, NotNil) err = client.SetBucketWebsite(bucketNameTest, indexWebsite, errorWebsite) c.Assert(err, NotNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // set time.Sleep(5 * time.Second) err = client.SetBucketWebsite(bucketNameTest, "myindex", "myerror") c.Assert(err, IsNil) res, err := client.GetBucketWebsite(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.IndexDocument.Suffix, Equals, "myindex") c.Assert(res.ErrorDocument.Key, Equals, "myerror") // detele err = client.DeleteBucketWebsite(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) _, err = client.GetBucketWebsite(bucketNameTest) c.Assert(err, NotNil) // detele after delete err = client.DeleteBucketWebsite(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestSetBucketWebsite func (s *OssClientSuite) TestSetBucketCORS(c *C) { var bucketNameTest = bucketNamePrefix + "tsbc" var rule1 = CORSRule{ AllowedOrigin: []string{"*"}, AllowedMethod: []string{"PUT", "GET", "POST"}, AllowedHeader: []string{}, ExposeHeader: []string{}, MaxAgeSeconds: 100, } var rule2 = CORSRule{ AllowedOrigin: []string{"http://www.a.com", "http://www.b.com"}, AllowedMethod: []string{"GET"}, AllowedHeader: []string{"Authorization"}, ExposeHeader: []string{"x-oss-test", "x-oss-test1"}, MaxAgeSeconds: 200, } client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) // set err = client.SetBucketCORS(bucketNameTest, []CORSRule{rule1}) c.Assert(err, IsNil) gbcr, err := client.GetBucketCORS(bucketNameTest) c.Assert(err, IsNil) c.Assert(len(gbcr.CORSRules), Equals, 1) c.Assert(len(gbcr.CORSRules[0].AllowedOrigin), Equals, 1) c.Assert(len(gbcr.CORSRules[0].AllowedMethod), Equals, 3) c.Assert(len(gbcr.CORSRules[0].AllowedHeader), Equals, 0) c.Assert(len(gbcr.CORSRules[0].ExposeHeader), Equals, 0) c.Assert(gbcr.CORSRules[0].MaxAgeSeconds, Equals, 100) // double set err = client.SetBucketCORS(bucketNameTest, []CORSRule{rule1}) c.Assert(err, IsNil) gbcr, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, IsNil) c.Assert(len(gbcr.CORSRules), Equals, 1) c.Assert(len(gbcr.CORSRules[0].AllowedOrigin), Equals, 1) c.Assert(len(gbcr.CORSRules[0].AllowedMethod), Equals, 3) c.Assert(len(gbcr.CORSRules[0].AllowedHeader), Equals, 0) c.Assert(len(gbcr.CORSRules[0].ExposeHeader), Equals, 0) c.Assert(gbcr.CORSRules[0].MaxAgeSeconds, Equals, 100) // set rule2 err = client.SetBucketCORS(bucketNameTest, []CORSRule{rule2}) c.Assert(err, IsNil) time.Sleep(5 * time.Second) gbcr, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, IsNil) c.Assert(len(gbcr.CORSRules), Equals, 1) c.Assert(len(gbcr.CORSRules[0].AllowedOrigin), Equals, 2) c.Assert(len(gbcr.CORSRules[0].AllowedMethod), Equals, 1) c.Assert(len(gbcr.CORSRules[0].AllowedHeader), Equals, 1) c.Assert(len(gbcr.CORSRules[0].ExposeHeader), Equals, 2) c.Assert(gbcr.CORSRules[0].MaxAgeSeconds, Equals, 200) // reset err = client.SetBucketCORS(bucketNameTest, []CORSRule{rule1, rule2}) c.Assert(err, IsNil) time.Sleep(5 * time.Second) gbcr, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, IsNil) c.Assert(len(gbcr.CORSRules), Equals, 2) // set after delete err = client.DeleteBucketCORS(bucketNameTest) c.Assert(err, IsNil) err = client.SetBucketCORS(bucketNameTest, []CORSRule{rule1, rule2}) c.Assert(err, IsNil) time.Sleep(5 * time.Second) gbcr, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, IsNil) c.Assert(len(gbcr.CORSRules), Equals, 2) err = client.DeleteBucketCORS(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestDeleteBucketCORS func (s *OssClientSuite) TestDeleteBucketCORS(c *C) { var bucketNameTest = bucketNamePrefix + "tdbc" var rule = CORSRule{ AllowedOrigin: []string{"*"}, AllowedMethod: []string{"PUT", "GET", "POST"}, AllowedHeader: []string{}, ExposeHeader: []string{}, MaxAgeSeconds: 100, } client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // delete not set err = client.DeleteBucketCORS(bucketNameTest) c.Assert(err, IsNil) // set err = client.SetBucketCORS(bucketNameTest, []CORSRule{rule}) c.Assert(err, IsNil) time.Sleep(5 * time.Second) _, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, IsNil) // detele err = client.DeleteBucketCORS(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) _, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, NotNil) // detele after delete err = client.DeleteBucketCORS(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestSetBucketCORSNegative func (s *OssClientSuite) TestSetBucketCORSNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tsbcn" var rule = CORSRule{ AllowedOrigin: []string{"*"}, AllowedMethod: []string{"PUT", "GET", "POST"}, AllowedHeader: []string{}, ExposeHeader: []string{}, MaxAgeSeconds: 100, } client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) // not exist _, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, NotNil) err = client.DeleteBucketCORS(bucketNameTest) c.Assert(err, NotNil) err = client.SetBucketCORS(bucketNameTest, []CORSRule{rule}) c.Assert(err, NotNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) _, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, NotNil) // set err = client.SetBucketCORS(bucketNameTest, []CORSRule{rule}) c.Assert(err, IsNil) time.Sleep(5 * time.Second) _, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, IsNil) // detele err = client.DeleteBucketCORS(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) _, err = client.GetBucketCORS(bucketNameTest) c.Assert(err, NotNil) // detele after delete err = client.DeleteBucketCORS(bucketNameTest) c.Assert(err, IsNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestGetBucketInfo func (s *OssClientSuite) TestGetBucketInfo(c *C) { var bucketNameTest = bucketNamePrefix + "tgbi" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) res, err := client.GetBucketInfo(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.BucketInfo.Name, Equals, bucketNameTest) c.Assert(strings.HasPrefix(res.BucketInfo.Location, "oss-"), Equals, true) c.Assert(res.BucketInfo.ACL, Equals, "private") c.Assert(strings.HasSuffix(res.BucketInfo.ExtranetEndpoint, ".com"), Equals, true) c.Assert(strings.HasSuffix(res.BucketInfo.IntranetEndpoint, ".com"), Equals, true) c.Assert(res.BucketInfo.CreationDate, NotNil) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestGetBucketInfoNegative func (s *OssClientSuite) TestGetBucketInfoNegative(c *C) { var bucketNameTest = bucketNamePrefix + "tgbig" client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) // not exist _, err = client.GetBucketInfo(bucketNameTest) c.Assert(err, NotNil) // bucket name invalid _, err = client.GetBucketInfo("InvalidBucketName_") c.Assert(err, NotNil) } // TestEndpointFormat func (s *OssClientSuite) TestEndpointFormat(c *C) { var bucketNameTest = bucketNamePrefix + "tef" // http://host client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) res, err := client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) // http://host:port client, err = New(endpoint+":80", accessID, accessKey) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) time.Sleep(5 * time.Second) res, err = client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestCname func (s *OssClientSuite) _TestCname(c *C) { var bucketNameTest = "" client, err := New("", "", "", UseCname(true)) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) _, err = client.ListBuckets() c.Assert(err, NotNil) res, err := client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) } // TestCnameNegative func (s *OssClientSuite) _TestCnameNegative(c *C) { var bucketNameTest = "" client, err := New("", "", "", UseCname(true)) c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, NotNil) _, err = client.ListBuckets() c.Assert(err, NotNil) _, err = client.GetBucketACL(bucketNameTest) c.Assert(err, NotNil) } // _TestHTTPS func (s *OssClientSuite) _TestHTTPS(c *C) { var bucketNameTest = "" client, err := New("", "", "") c.Assert(err, IsNil) err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) res, err := client.GetBucketACL(bucketNameTest) c.Assert(err, IsNil) c.Assert(res.ACL, Equals, string(ACLPrivate)) err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // TestClientOption func (s *OssClientSuite) TestClientOption(c *C) { var bucketNameTest = bucketNamePrefix + "tco" client, err := New(endpoint, accessID, accessKey, UseCname(true), Timeout(11, 12), SecurityToken("token"), Proxy(proxyHost)) c.Assert(err, IsNil) // Create Bucket timeout err = client.CreateBucket(bucketNameTest) c.Assert(err, NotNil) c.Assert(client.Conn.config.HTTPTimeout.ConnectTimeout, Equals, time.Second*11) c.Assert(client.Conn.config.HTTPTimeout.ReadWriteTimeout, Equals, time.Second*12) c.Assert(client.Conn.config.HTTPTimeout.HeaderTimeout, Equals, time.Second*12) c.Assert(client.Conn.config.HTTPTimeout.LongTimeout, Equals, time.Second*12*10) c.Assert(client.Conn.config.SecurityToken, Equals, "token") c.Assert(client.Conn.config.IsCname, Equals, true) c.Assert(client.Conn.config.IsUseProxy, Equals, true) c.Assert(client.Config.ProxyHost, Equals, proxyHost) client, err = New(endpoint, accessID, accessKey, AuthProxy(proxyHost, proxyUser, proxyPasswd)) c.Assert(client.Conn.config.IsUseProxy, Equals, true) c.Assert(client.Config.ProxyHost, Equals, proxyHost) c.Assert(client.Conn.config.IsAuthProxy, Equals, true) c.Assert(client.Conn.config.ProxyUser, Equals, proxyUser) c.Assert(client.Conn.config.ProxyPassword, Equals, proxyPasswd) client, err = New(endpoint, accessID, accessKey, UserAgent("go sdk user agent")) c.Assert(client.Conn.config.UserAgent, Equals, "go sdk user agent") } // TestProxy func (s *OssClientSuite) TestProxy(c *C) { bucketNameTest := bucketNamePrefix + "tp" objectName := "体育/奥运/首金" objectValue := "大江东去，浪淘尽，千古风流人物。 故垒西边，人道是、三国周郎赤壁。" client, err := New(endpoint, accessID, accessKey, AuthProxy(proxyHost, proxyUser, proxyPasswd)) // Create bucket err = client.CreateBucket(bucketNameTest) c.Assert(err, IsNil) // Get bucket info _, err = client.GetBucketInfo(bucketNameTest) c.Assert(err, IsNil) bucket, err := client.Bucket(bucketNameTest) // Put object err = bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // Get Object _, err = bucket.GetObject(objectName) c.Assert(err, IsNil) // List Objects _, err = bucket.ListObjects() c.Assert(err, IsNil) // Delete Object err = bucket.DeleteObject(objectName) c.Assert(err, IsNil) // Delete bucket err = client.DeleteBucket(bucketNameTest) c.Assert(err, IsNil) } // private func (s *OssClientSuite) checkBucket(buckets []BucketProperties, bucket string) bool { for _, v := range buckets { if v.Name == bucket { return true } } return false } func (s *OssClientSuite) getBucket(buckets []BucketProperties, bucket string) (bool, BucketProperties) { for _, v := range buckets { if v.Name == bucket { return true, v } } return false, BucketProperties{} } aliyun-oss-go-sdk-1.5.0/oss/conf.go000066400000000000000000000041731311621456000170660ustar00rootroot00000000000000package oss import ( "time" ) // HTTPTimeout http timeout type HTTPTimeout struct { ConnectTimeout time.Duration ReadWriteTimeout time.Duration HeaderTimeout time.Duration LongTimeout time.Duration } // Config oss configure type Config struct { Endpoint string // oss地址 AccessKeyID string // accessId AccessKeySecret string // accessKey RetryTimes uint // 失败重试次数，默认5 UserAgent string // SDK名称/版本/系统信息 IsDebug bool // 是否开启调试模式，默认false Timeout uint // 超时时间，默认60s SecurityToken string // STS Token IsCname bool // Endpoint是否是CNAME HTTPTimeout HTTPTimeout // HTTP的超时时间设置 IsUseProxy bool // 是否使用代理 ProxyHost string // 代理服务器地址 IsAuthProxy bool // 代理服务器是否使用用户认证 ProxyUser string // 代理服务器认证用户名 ProxyPassword string // 代理服务器认证密码 IsEnableMD5 bool // 上传数据时是否启用MD5校验 MD5Threshold int64 // 内存中计算MD5的上线大小，大于该值启用临时文件，单位Byte IsEnableCRC bool // 上传数据时是否启用CRC64校验 } // 获取默认配置 func getDefaultOssConfig() *Config { config := Config{} config.Endpoint = "" config.AccessKeyID = "" config.AccessKeySecret = "" config.RetryTimes = 5 config.IsDebug = false config.UserAgent = userAgent config.Timeout = 60 // seconds config.SecurityToken = "" config.IsCname = false config.HTTPTimeout.ConnectTimeout = time.Second * 30 // 30s config.HTTPTimeout.ReadWriteTimeout = time.Second * 60 // 60s config.HTTPTimeout.HeaderTimeout = time.Second * 60 // 60s config.HTTPTimeout.LongTimeout = time.Second * 300 // 300s config.IsUseProxy = false config.ProxyHost = "" config.IsAuthProxy = false config.ProxyUser = "" config.ProxyPassword = "" config.MD5Threshold = 16 * 1024 * 1024 // 16MB config.IsEnableMD5 = false config.IsEnableCRC = true return &config } aliyun-oss-go-sdk-1.5.0/oss/conn.go000066400000000000000000000251261311621456000170770ustar00rootroot00000000000000package oss import ( "bytes" "crypto/md5" "encoding/base64" "encoding/xml" "fmt" "hash" "io" "io/ioutil" "net" "net/http" "net/url" "os" "strconv" "strings" "time" ) // Conn oss conn type Conn struct { config *Config url *urlMaker client *http.Client } // init 初始化Conn func (conn *Conn) init(config *Config, urlMaker *urlMaker) error { httpTimeOut := conn.config.HTTPTimeout // new Transport transport := &http.Transport{ Dial: func(netw, addr string) (net.Conn, error) { conn, err := net.DialTimeout(netw, addr, httpTimeOut.ConnectTimeout) if err != nil { return nil, err } return newTimeoutConn(conn, httpTimeOut.ReadWriteTimeout, httpTimeOut.LongTimeout), nil }, ResponseHeaderTimeout: httpTimeOut.HeaderTimeout, } // Proxy if conn.config.IsUseProxy { proxyURL, err := url.Parse(config.ProxyHost) if err != nil { return err } transport.Proxy = http.ProxyURL(proxyURL) } conn.config = config conn.url = urlMaker conn.client = &http.Client{Transport: transport} return nil } // Do 处理请求，返回响应结果。 func (conn Conn) Do(method, bucketName, objectName, urlParams, subResource string, headers map[string]string, data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) { uri := conn.url.getURL(bucketName, objectName, urlParams) resource := conn.url.getResource(bucketName, objectName, subResource) return conn.doRequest(method, uri, resource, headers, data, initCRC, listener) } func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource string, headers map[string]string, data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) { method = strings.ToUpper(method) if !conn.config.IsUseProxy { uri.Opaque = uri.Path } req := &http.Request{ Method: method, URL: uri, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: make(http.Header), Host: uri.Host, } tracker := &readerTracker{completedBytes: 0} fd, crc := conn.handleBody(req, data, initCRC, listener, tracker) if fd != nil { defer func() { fd.Close() os.Remove(fd.Name()) }() } if conn.config.IsAuthProxy { auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) req.Header.Set("Proxy-Authorization", basic) } date := time.Now().UTC().Format(http.TimeFormat) req.Header.Set(HTTPHeaderDate, date) req.Header.Set(HTTPHeaderHost, conn.config.Endpoint) req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent) if conn.config.SecurityToken != "" { req.Header.Set(HTTPHeaderOssSecurityToken, conn.config.SecurityToken) } if headers != nil { for k, v := range headers { req.Header.Set(k, v) } } conn.signHeader(req, canonicalizedResource) // transfer started event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength) publishProgress(listener, event) resp, err := conn.client.Do(req) if err != nil { // transfer failed event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength) publishProgress(listener, event) return nil, err } // transfer completed event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength) publishProgress(listener, event) return conn.handleResponse(resp, crc) } // handle request body func (conn Conn) handleBody(req *http.Request, body io.Reader, initCRC uint64, listener ProgressListener, tracker *readerTracker) (*os.File, hash.Hash64) { var file *os.File var crc hash.Hash64 reader := body // length switch v := body.(type) { case *bytes.Buffer: req.ContentLength = int64(v.Len()) case *bytes.Reader: req.ContentLength = int64(v.Len()) case *strings.Reader: req.ContentLength = int64(v.Len()) case *os.File: req.ContentLength = tryGetFileSize(v) case *io.LimitedReader: req.ContentLength = int64(v.N) } req.Header.Set(HTTPHeaderContentLength, strconv.FormatInt(req.ContentLength, 10)) // md5 if body != nil && conn.config.IsEnableMD5 && req.Header.Get(HTTPHeaderContentMD5) == "" { md5 := "" reader, md5, file, _ = calcMD5(body, req.ContentLength, conn.config.MD5Threshold) req.Header.Set(HTTPHeaderContentMD5, md5) } // crc if reader != nil && conn.config.IsEnableCRC { crc = NewCRC(crcTable(), initCRC) reader = TeeReader(reader, crc, req.ContentLength, listener, tracker) } // http body rc, ok := reader.(io.ReadCloser) if !ok && reader != nil { rc = ioutil.NopCloser(reader) } req.Body = rc return file, crc } func tryGetFileSize(f *os.File) int64 { fInfo, _ := f.Stat() return fInfo.Size() } // handle response func (conn Conn) handleResponse(resp *http.Response, crc hash.Hash64) (*Response, error) { var cliCRC uint64 var srvCRC uint64 statusCode := resp.StatusCode if statusCode >= 400 && statusCode <= 505 { // 4xx and 5xx indicate that the operation has error occurred var respBody []byte respBody, err := readResponseBody(resp) if err != nil { return nil, err } if len(respBody) == 0 { // no error in response body err = fmt.Errorf("oss: service returned without a response body (%s)", resp.Status) } else { // response contains storage service error object, unmarshal srvErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, resp.Header.Get(HTTPHeaderOssRequestID)) if err != nil { // error unmarshaling the error response err = errIn } err = srvErr } return &Response{ StatusCode: resp.StatusCode, Headers: resp.Header, Body: ioutil.NopCloser(bytes.NewReader(respBody)), // restore the body }, err } else if statusCode >= 300 && statusCode <= 307 { // oss use 3xx, but response has no body err := fmt.Errorf("oss: service returned %d,%s", resp.StatusCode, resp.Status) return &Response{ StatusCode: resp.StatusCode, Headers: resp.Header, Body: resp.Body, }, err } if conn.config.IsEnableCRC && crc != nil { cliCRC = crc.Sum64() } srvCRC, _ = strconv.ParseUint(resp.Header.Get(HTTPHeaderOssCRC64), 10, 64) // 2xx, successful return &Response{ StatusCode: resp.StatusCode, Headers: resp.Header, Body: resp.Body, ClientCRC: cliCRC, ServerCRC: srvCRC, }, nil } func calcMD5(body io.Reader, contentLen, md5Threshold int64) (reader io.Reader, b64 string, tempFile *os.File, err error) { if contentLen == 0 || contentLen > md5Threshold { // huge body, use temporary file tempFile, err = ioutil.TempFile(os.TempDir(), TempFilePrefix) if tempFile != nil { io.Copy(tempFile, body) tempFile.Seek(0, os.SEEK_SET) md5 := md5.New() io.Copy(md5, tempFile) sum := md5.Sum(nil) b64 = base64.StdEncoding.EncodeToString(sum[:]) tempFile.Seek(0, os.SEEK_SET) reader = tempFile } } else { // small body, use memory buf, _ := ioutil.ReadAll(body) sum := md5.Sum(buf) b64 = base64.StdEncoding.EncodeToString(sum[:]) reader = bytes.NewReader(buf) } return } func readResponseBody(resp *http.Response) ([]byte, error) { defer resp.Body.Close() out, err := ioutil.ReadAll(resp.Body) if err == io.EOF { err = nil } return out, err } func serviceErrFromXML(body []byte, statusCode int, requestID string) (ServiceError, error) { var storageErr ServiceError if err := xml.Unmarshal(body, &storageErr); err != nil { return storageErr, err } storageErr.StatusCode = statusCode storageErr.RequestID = requestID storageErr.RawMessage = string(body) return storageErr, nil } func xmlUnmarshal(body io.Reader, v interface{}) error { data, err := ioutil.ReadAll(body) if err != nil { return err } return xml.Unmarshal(data, v) } // Handle http timeout type timeoutConn struct { conn net.Conn timeout time.Duration longTimeout time.Duration } func newTimeoutConn(conn net.Conn, timeout time.Duration, longTimeout time.Duration) *timeoutConn { conn.SetReadDeadline(time.Now().Add(longTimeout)) return &timeoutConn{ conn: conn, timeout: timeout, longTimeout: longTimeout, } } func (c *timeoutConn) Read(b []byte) (n int, err error) { c.SetReadDeadline(time.Now().Add(c.timeout)) n, err = c.conn.Read(b) c.SetReadDeadline(time.Now().Add(c.longTimeout)) return n, err } func (c *timeoutConn) Write(b []byte) (n int, err error) { c.SetWriteDeadline(time.Now().Add(c.timeout)) n, err = c.conn.Write(b) c.SetReadDeadline(time.Now().Add(c.longTimeout)) return n, err } func (c *timeoutConn) Close() error { return c.conn.Close() } func (c *timeoutConn) LocalAddr() net.Addr { return c.conn.LocalAddr() } func (c *timeoutConn) RemoteAddr() net.Addr { return c.conn.RemoteAddr() } func (c *timeoutConn) SetDeadline(t time.Time) error { return c.conn.SetDeadline(t) } func (c *timeoutConn) SetReadDeadline(t time.Time) error { return c.conn.SetReadDeadline(t) } func (c *timeoutConn) SetWriteDeadline(t time.Time) error { return c.conn.SetWriteDeadline(t) } // UrlMaker - build url and resource const ( urlTypeCname = 1 urlTypeIP = 2 urlTypeAliyun = 3 ) type urlMaker struct { Scheme string // http or https NetLoc string // host or ip Type int // 1 CNAME 2 IP 3 ALIYUN IsProxy bool // proxy } // Parse endpoint func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) { if strings.HasPrefix(endpoint, "http://") { um.Scheme = "http" um.NetLoc = endpoint[len("http://"):] } else if strings.HasPrefix(endpoint, "https://") { um.Scheme = "https" um.NetLoc = endpoint[len("https://"):] } else { um.Scheme = "http" um.NetLoc = endpoint } host, _, err := net.SplitHostPort(um.NetLoc) if err != nil { host = um.NetLoc } ip := net.ParseIP(host) if ip != nil { um.Type = urlTypeIP } else if isCname { um.Type = urlTypeCname } else { um.Type = urlTypeAliyun } um.IsProxy = isProxy } // Build URL func (um urlMaker) getURL(bucket, object, params string) *url.URL { var host = "" var path = "" if !um.IsProxy { object = url.QueryEscape(object) } if um.Type == urlTypeCname { host = um.NetLoc path = "/" + object } else if um.Type == urlTypeIP { if bucket == "" { host = um.NetLoc path = "/" } else { host = um.NetLoc path = fmt.Sprintf("/%s/%s", bucket, object) } } else { if bucket == "" { host = um.NetLoc path = "/" } else { host = bucket + "." + um.NetLoc path = "/" + object } } uri := &url.URL{ Scheme: um.Scheme, Host: host, Path: path, RawQuery: params, } return uri } // Canonicalized Resource func (um urlMaker) getResource(bucketName, objectName, subResource string) string { if subResource != "" { subResource = "?" + subResource } if bucketName == "" { return fmt.Sprintf("/%s%s", bucketName, subResource) } return fmt.Sprintf("/%s/%s%s", bucketName, objectName, subResource) } aliyun-oss-go-sdk-1.5.0/oss/conn_test.go000066400000000000000000000102731311621456000201330ustar00rootroot00000000000000package oss import ( "net/http" . "gopkg.in/check.v1" ) type OssConnSuite struct{} var _ = Suite(&OssConnSuite{}) func (s *OssConnSuite) TestURLMarker(c *C) { um := urlMaker{} um.Init("docs.github.com", true, false) c.Assert(um.Type, Equals, urlTypeCname) c.Assert(um.Scheme, Equals, "http") c.Assert(um.NetLoc, Equals, "docs.github.com") c.Assert(um.getURL("bucket", "object", "params").String(), Equals, "http://docs.github.com/object?params") c.Assert(um.getURL("bucket", "object", "").String(), Equals, "http://docs.github.com/object") c.Assert(um.getURL("", "object", "").String(), Equals, "http://docs.github.com/object") c.Assert(um.getResource("bucket", "object", "subres"), Equals, "/bucket/object?subres") c.Assert(um.getResource("bucket", "object", ""), Equals, "/bucket/object") c.Assert(um.getResource("", "object", ""), Equals, "/") um.Init("https://docs.github.com", true, false) c.Assert(um.Type, Equals, urlTypeCname) c.Assert(um.Scheme, Equals, "https") c.Assert(um.NetLoc, Equals, "docs.github.com") um.Init("http://docs.github.com", true, false) c.Assert(um.Type, Equals, urlTypeCname) c.Assert(um.Scheme, Equals, "http") c.Assert(um.NetLoc, Equals, "docs.github.com") um.Init("docs.github.com:8080", false, true) c.Assert(um.Type, Equals, urlTypeAliyun) c.Assert(um.Scheme, Equals, "http") c.Assert(um.NetLoc, Equals, "docs.github.com:8080") c.Assert(um.getURL("bucket", "object", "params").String(), Equals, "http://bucket.docs.github.com:8080/object?params") c.Assert(um.getURL("bucket", "object", "").String(), Equals, "http://bucket.docs.github.com:8080/object") c.Assert(um.getURL("", "object", "").String(), Equals, "http://docs.github.com:8080/") c.Assert(um.getResource("bucket", "object", "subres"), Equals, "/bucket/object?subres") c.Assert(um.getResource("bucket", "object", ""), Equals, "/bucket/object") c.Assert(um.getResource("", "object", ""), Equals, "/") um.Init("https://docs.github.com:8080", false, true) c.Assert(um.Type, Equals, urlTypeAliyun) c.Assert(um.Scheme, Equals, "https") c.Assert(um.NetLoc, Equals, "docs.github.com:8080") um.Init("127.0.0.1", false, true) c.Assert(um.Type, Equals, urlTypeIP) c.Assert(um.Scheme, Equals, "http") c.Assert(um.NetLoc, Equals, "127.0.0.1") um.Init("http://127.0.0.1", false, false) c.Assert(um.Type, Equals, urlTypeIP) c.Assert(um.Scheme, Equals, "http") c.Assert(um.NetLoc, Equals, "127.0.0.1") um.Init("https://127.0.0.1:8080", false, false) c.Assert(um.Type, Equals, urlTypeIP) c.Assert(um.Scheme, Equals, "https") c.Assert(um.NetLoc, Equals, "127.0.0.1:8080") } func (s *OssConnSuite) TestAuth(c *C) { endpoint := "https://github.com/" cfg := getDefaultOssConfig() um := urlMaker{} um.Init(endpoint, false, false) conn := Conn{cfg, &um, nil} uri := um.getURL("bucket", "object", "") req := &http.Request{ Method: "PUT", URL: uri, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: make(http.Header), Host: uri.Host, } req.Header.Set("Content-Type", "text/html") req.Header.Set("Date", "Thu, 17 Nov 2005 18:49:58 GMT") req.Header.Set("Host", endpoint) req.Header.Set("X-OSS-Meta-Your", "your") req.Header.Set("X-OSS-Meta-Author", "foo@bar.com") req.Header.Set("X-OSS-Magic", "abracadabra") req.Header.Set("Content-Md5", "ODBGOERFMDMzQTczRUY3NUE3NzA5QzdFNUYzMDQxNEM=") conn.signHeader(req, um.getResource("bucket", "object", "")) testLogger.Println("AUTHORIZATION:", req.Header.Get(HTTPHeaderAuthorization)) } func (s *OssConnSuite) TestConnToolFunc(c *C) { err := checkRespCode(202, []int{}) c.Assert(err, NotNil) err = checkRespCode(202, []int{404}) c.Assert(err, NotNil) err = checkRespCode(202, []int{202, 404}) c.Assert(err, IsNil) srvErr, err := serviceErrFromXML([]byte(""), 312, "") c.Assert(err, NotNil) c.Assert(srvErr.StatusCode, Equals, 0) srvErr, err = serviceErrFromXML([]byte("ABC"), 312, "") c.Assert(err, NotNil) c.Assert(srvErr.StatusCode, Equals, 0) srvErr, err = serviceErrFromXML([]byte(""), 312, "") c.Assert(err, IsNil) c.Assert(srvErr.StatusCode, Equals, 312) unexpect := UnexpectedStatusCodeError{[]int{200}, 202} c.Assert(len(unexpect.Error()) > 0, Equals, true) c.Assert(unexpect.Got(), Equals, 202) } aliyun-oss-go-sdk-1.5.0/oss/const.go000066400000000000000000000073631311621456000172730ustar00rootroot00000000000000package oss import "os" // ACLType Bucket/Object的访问控制 type ACLType string const ( // ACLPrivate 私有读写 ACLPrivate ACLType = "private" // ACLPublicRead 公共读私有写 ACLPublicRead ACLType = "public-read" // ACLPublicReadWrite 公共读写 ACLPublicReadWrite ACLType = "public-read-write" // ACLDefault Object默认权限，Bucket无此权限 ACLDefault ACLType = "default" ) // MetadataDirectiveType 对象COPY时新对象是否使用原对象的Meta type MetadataDirectiveType string const ( // MetaCopy 目标对象使用源对象的META MetaCopy MetadataDirectiveType = "COPY" // MetaReplace 目标对象使用自定义的META MetaReplace MetadataDirectiveType = "REPLACE" ) // StorageClassType Bucket的存储类型 type StorageClassType string const ( // StorageStandard 标准存储模式 StorageStandard StorageClassType = "Standard" // StorageIA IA存储模式 StorageIA StorageClassType = "IA" // StorageArchive Archive存储模式 StorageArchive StorageClassType = "Archive" ) // Http头标签 const ( HTTPHeaderAcceptEncoding string = "Accept-Encoding" HTTPHeaderAuthorization = "Authorization" HTTPHeaderCacheControl = "Cache-Control" HTTPHeaderContentDisposition = "Content-Disposition" HTTPHeaderContentEncoding = "Content-Encoding" HTTPHeaderContentLength = "Content-Length" HTTPHeaderContentMD5 = "Content-MD5" HTTPHeaderContentType = "Content-Type" HTTPHeaderContentLanguage = "Content-Language" HTTPHeaderDate = "Date" HTTPHeaderEtag = "ETag" HTTPHeaderExpires = "Expires" HTTPHeaderHost = "Host" HTTPHeaderLastModified = "Last-Modified" HTTPHeaderRange = "Range" HTTPHeaderLocation = "Location" HTTPHeaderOrigin = "Origin" HTTPHeaderServer = "Server" HTTPHeaderUserAgent = "User-Agent" HTTPHeaderIfModifiedSince = "If-Modified-Since" HTTPHeaderIfUnmodifiedSince = "If-Unmodified-Since" HTTPHeaderIfMatch = "If-Match" HTTPHeaderIfNoneMatch = "If-None-Match" HTTPHeaderOssACL = "X-Oss-Acl" HTTPHeaderOssMetaPrefix = "X-Oss-Meta-" HTTPHeaderOssObjectACL = "X-Oss-Object-Acl" HTTPHeaderOssSecurityToken = "X-Oss-Security-Token" HTTPHeaderOssServerSideEncryption = "X-Oss-Server-Side-Encryption" HTTPHeaderOssCopySource = "X-Oss-Copy-Source" HTTPHeaderOssCopySourceRange = "X-Oss-Copy-Source-Range" HTTPHeaderOssCopySourceIfMatch = "X-Oss-Copy-Source-If-Match" HTTPHeaderOssCopySourceIfNoneMatch = "X-Oss-Copy-Source-If-None-Match" HTTPHeaderOssCopySourceIfModifiedSince = "X-Oss-Copy-Source-If-Modified-Since" HTTPHeaderOssCopySourceIfUnmodifiedSince = "X-Oss-Copy-Source-If-Unmodified-Since" HTTPHeaderOssMetadataDirective = "X-Oss-Metadata-Directive" HTTPHeaderOssNextAppendPosition = "X-Oss-Next-Append-Position" HTTPHeaderOssRequestID = "X-Oss-Request-Id" HTTPHeaderOssCRC64 = "X-Oss-Hash-Crc64ecma" HTTPHeaderOssSymlinkTarget = "X-Oss-Symlink-Target" ) // 其它常量 const ( MaxPartSize = 5 * 1024 * 1024 * 1024 // 文件片最大值，5GB MinPartSize = 100 * 1024 // 文件片最小值，100KBß FilePermMode = os.FileMode(0664) // 新建文件默认权限 TempFilePrefix = "oss-go-temp-" // 临时文件前缀 TempFileSuffix = ".temp" // 临时文件后缀 CheckpointFileSuffix = ".cp" // Checkpoint文件后缀 Version = "1.4.0" // Go sdk版本 ) aliyun-oss-go-sdk-1.5.0/oss/crc.go000066400000000000000000000024521311621456000167060ustar00rootroot00000000000000package oss import ( "hash" "hash/crc64" ) // digest represents the partial evaluation of a checksum. type digest struct { crc uint64 tab *crc64.Table } // NewCRC creates a new hash.Hash64 computing the CRC-64 checksum // using the polynomial represented by the Table. func NewCRC(tab *crc64.Table, init uint64) hash.Hash64 { return &digest{init, tab} } // Size returns the number of bytes Sum will return. func (d *digest) Size() int { return crc64.Size } // BlockSize returns the hash's underlying block size. // The Write method must be able to accept any amount // of data, but it may operate more efficiently if all writes // are a multiple of the block size. func (d *digest) BlockSize() int { return 1 } // Reset resets the Hash to its initial state. func (d *digest) Reset() { d.crc = 0 } // Write (via the embedded io.Writer interface) adds more data to the running hash. // It never returns an error. func (d *digest) Write(p []byte) (n int, err error) { d.crc = crc64.Update(d.crc, d.tab, p) return len(p), nil } // Sum64 returns crc64 value. func (d *digest) Sum64() uint64 { return d.crc } // Sum returns hash value. func (d *digest) Sum(in []byte) []byte { s := d.Sum64() return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) } aliyun-oss-go-sdk-1.5.0/oss/crc_test.go000066400000000000000000000311061311621456000177430ustar00rootroot00000000000000package oss import ( "crypto/md5" "encoding/base64" "hash/crc64" "io" "io/ioutil" "os" "strings" "time" . "gopkg.in/check.v1" ) type OssCrcSuite struct { client *Client bucket *Bucket } var _ = Suite(&OssCrcSuite{}) // Run once when the suite starts running func (s *OssCrcSuite) SetUpSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) s.client = client s.client.CreateBucket(bucketName) time.Sleep(5 * time.Second) bucket, err := s.client.Bucket(bucketName) c.Assert(err, IsNil) s.bucket = bucket testLogger.Println("test crc started") } // Run before each test or benchmark starts running func (s *OssCrcSuite) TearDownSuite(c *C) { // Delete Part lmur, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmur.Uploads { var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName, Key: upload.Key, UploadID: upload.UploadID} err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } // Delete Objects lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = s.bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } testLogger.Println("test crc completed") } // Run after each test or benchmark runs func (s *OssCrcSuite) SetUpTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // Run once after all tests or benchmarks have finished running func (s *OssCrcSuite) TearDownTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // TestCRCGolden 测试OSS实现的CRC64 func (s *OssCrcSuite) TestCRCGolden(c *C) { type crcTest struct { out uint64 in string } var crcGolden = []crcTest{ {0x0, ""}, {0x3420000000000000, "a"}, {0x36c4200000000000, "ab"}, {0x3776c42000000000, "abc"}, {0x336776c420000000, "abcd"}, {0x32d36776c4200000, "abcde"}, {0x3002d36776c42000, "abcdef"}, {0x31b002d36776c420, "abcdefg"}, {0xe21b002d36776c4, "abcdefgh"}, {0x8b6e21b002d36776, "abcdefghi"}, {0x7f5b6e21b002d367, "abcdefghij"}, {0x8ec0e7c835bf9cdf, "Discard medicine more than two years old."}, {0xc7db1759e2be5ab4, "He who has a shady past knows that nice guys finish last."}, {0xfbf9d9603a6fa020, "I wouldn't marry him with a ten foot pole."}, {0xeafc4211a6daa0ef, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, {0x3e05b21c7a4dc4da, "The days of the digital watch are numbered. -Tom Stoppard"}, {0x5255866ad6ef28a6, "Nepal premier won't resign."}, {0x8a79895be1e9c361, "For every action there is an equal and opposite government program."}, {0x8878963a649d4916, "His money is twice tainted: 'taint yours and 'taint mine."}, {0xa7b9d53ea87eb82f, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, {0xdb6805c0966a2f9c, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, {0xf3553c65dacdadd2, "size: a.out: bad magic"}, {0x9d5e034087a676b9, "The major problem is with sendmail. -Mark Horton"}, {0xa6db2d7f8da96417, "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, {0x325e00cd2fe819f9, "If the enemy is within range, then so are you."}, {0x88c6600ce58ae4c6, "It's well we cannot hear the screams/That we create in others' dreams."}, {0x28c4a3f3b769e078, "You remind me of a TV show, but that's all right: I watch it anyway."}, {0xa698a34c9d9f1dca, "C is as portable as Stonehedge!!"}, {0xf6c1e2a8c26c5cfc, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, {0xd402559dfe9b70c, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, {0xdb6efff26aa94946, "How can you write a big system without C++? -Paul Glick"}, } var tab = crc64.MakeTable(crc64.ISO) for i := 0; i < len(crcGolden); i++ { golden := crcGolden[i] crc := NewCRC(tab, 0) io.WriteString(crc, golden.in) sum := crc.Sum64() c.Assert(sum, Equals, golden.out) } } // TestEnableCRCAndMD5 开启MD5和CRC校验 func (s *OssCrcSuite) TestEnableCRCAndMD5(c *C) { objectName := objectNamePrefix + "tecam" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFileName := "BingWallpaper-2015-11-07-2.jpg" objectValue := "空山新雨后，天气晚来秋。明月松间照，清泉石上流。竹喧归浣女，莲动下渔舟。随意春芳歇，王孙自可留。" client, err := New(endpoint, accessID, accessKey, EnableCRC(true), EnableMD5(true), MD5ThresholdCalcInMemory(200*1024)) c.Assert(err, IsNil) bucket, err := client.Bucket(bucketName) c.Assert(err, IsNil) // PutObject err = bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // GetObject body, err := bucket.GetObject(objectName) c.Assert(err, IsNil) _, err = ioutil.ReadAll(body) c.Assert(err, IsNil) body.Close() // GetObjectWithCRC getResult, err := bucket.DoGetObject(&GetObjectRequest{objectName}, nil) c.Assert(err, IsNil) str, err := readBody(getResult.Response.Body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) c.Assert(getResult.ClientCRC.Sum64(), Equals, getResult.ServerCRC) // PutObjectFromFile err = bucket.PutObjectFromFile(objectName, fileName) c.Assert(err, IsNil) // GetObjectToFile err = bucket.GetObjectToFile(objectName, newFileName) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFileName) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // DeleteObject err = bucket.DeleteObject(objectName) c.Assert(err, IsNil) // AppendObject var nextPos int64 nextPos, err = bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos) c.Assert(err, IsNil) nextPos, err = bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos) c.Assert(err, IsNil) err = bucket.DeleteObject(objectName) c.Assert(err, IsNil) request := &AppendObjectRequest{ ObjectKey: objectName, Reader: strings.NewReader(objectValue), Position: 0, } appendResult, err := bucket.DoAppendObject(request, []Option{InitCRC(0)}) c.Assert(err, IsNil) request.Position = appendResult.NextPosition appendResult, err = bucket.DoAppendObject(request, []Option{InitCRC(appendResult.CRC)}) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // MultipartUpload chunks, err := SplitFileByPartSize(fileName, 100*1024) imurUpload, err := bucket.InitiateMultipartUpload(objectName) c.Assert(err, IsNil) var partsUpload []UploadPart for _, chunk := range chunks { part, err := bucket.UploadPartFromFile(imurUpload, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) partsUpload = append(partsUpload, part) } _, err = bucket.CompleteMultipartUpload(imurUpload, partsUpload) c.Assert(err, IsNil) // Check MultipartUpload err = bucket.GetObjectToFile(objectName, newFileName) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFileName) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // DeleteObjects _, err = bucket.DeleteObjects([]string{objectName}) c.Assert(err, IsNil) } // TestDisableCRCAndMD5 关闭MD5和CRC校验 func (s *OssCrcSuite) TestDisableCRCAndMD5(c *C) { objectName := objectNamePrefix + "tdcam" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFileName := "BingWallpaper-2015-11-07-3.jpg" objectValue := "中岁颇好道，晚家南山陲。兴来每独往，胜事空自知。行到水穷处，坐看云起时。偶然值林叟，谈笑无还期。" client, err := New(endpoint, accessID, accessKey, EnableCRC(false), EnableMD5(false)) c.Assert(err, IsNil) bucket, err := client.Bucket(bucketName) c.Assert(err, IsNil) // PutObject err = bucket.PutObject(objectName, strings.NewReader(objectValue)) c.Assert(err, IsNil) // GetObject body, err := bucket.GetObject(objectName) c.Assert(err, IsNil) _, err = ioutil.ReadAll(body) c.Assert(err, IsNil) body.Close() // GetObjectWithCRC getResult, err := bucket.DoGetObject(&GetObjectRequest{objectName}, nil) c.Assert(err, IsNil) str, err := readBody(getResult.Response.Body) c.Assert(err, IsNil) c.Assert(str, Equals, objectValue) // PutObjectFromFile err = bucket.PutObjectFromFile(objectName, fileName) c.Assert(err, IsNil) // GetObjectToFile err = bucket.GetObjectToFile(objectName, newFileName) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFileName) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // DeleteObject err = bucket.DeleteObject(objectName) c.Assert(err, IsNil) // AppendObject var nextPos int64 nextPos, err = bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos) c.Assert(err, IsNil) nextPos, err = bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos) c.Assert(err, IsNil) err = bucket.DeleteObject(objectName) c.Assert(err, IsNil) request := &AppendObjectRequest{ ObjectKey: objectName, Reader: strings.NewReader(objectValue), Position: 0, } appendResult, err := bucket.DoAppendObject(request, []Option{InitCRC(0)}) c.Assert(err, IsNil) request.Position = appendResult.NextPosition appendResult, err = bucket.DoAppendObject(request, []Option{InitCRC(appendResult.CRC)}) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // MultipartUpload chunks, err := SplitFileByPartSize(fileName, 100*1024) imurUpload, err := bucket.InitiateMultipartUpload(objectName) c.Assert(err, IsNil) var partsUpload []UploadPart for _, chunk := range chunks { part, err := bucket.UploadPartFromFile(imurUpload, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) partsUpload = append(partsUpload, part) } _, err = bucket.CompleteMultipartUpload(imurUpload, partsUpload) c.Assert(err, IsNil) // Check MultipartUpload err = bucket.GetObjectToFile(objectName, newFileName) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFileName) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // DeleteObjects _, err = bucket.DeleteObjects([]string{objectName}) c.Assert(err, IsNil) } // TestSpecifyContentMD5 指定MD5 func (s *OssCrcSuite) TestSpecifyContentMD5(c *C) { objectName := objectNamePrefix + "tdcam" fileName := "../sample/BingWallpaper-2015-11-07.jpg" objectValue := "积雨空林烟火迟，蒸藜炊黍饷东菑。漠漠水田飞白鹭，阴阴夏木啭黄鹂。山中习静观朝槿，松下清斋折露葵。野老与人争席罢，海鸥何事更相疑。" mh := md5.Sum([]byte(objectValue)) md5B64 := base64.StdEncoding.EncodeToString(mh[:]) // PutObject err := s.bucket.PutObject(objectName, strings.NewReader(objectValue), ContentMD5(md5B64)) c.Assert(err, IsNil) // PutObjectFromFile file, err := os.Open(fileName) md5 := md5.New() io.Copy(md5, file) mdHex := base64.StdEncoding.EncodeToString(md5.Sum(nil)[:]) err = s.bucket.PutObjectFromFile(objectName, fileName, ContentMD5(mdHex)) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // AppendObject var nextPos int64 nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos) c.Assert(err, IsNil) nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) request := &AppendObjectRequest{ ObjectKey: objectName, Reader: strings.NewReader(objectValue), Position: 0, } appendResult, err := s.bucket.DoAppendObject(request, []Option{InitCRC(0)}) c.Assert(err, IsNil) request.Position = appendResult.NextPosition appendResult, err = s.bucket.DoAppendObject(request, []Option{InitCRC(appendResult.CRC)}) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // MultipartUpload imurUpload, err := s.bucket.InitiateMultipartUpload(objectName) c.Assert(err, IsNil) var partsUpload []UploadPart part, err := s.bucket.UploadPart(imurUpload, strings.NewReader(objectValue), (int64)(len([]byte(objectValue))), 1) c.Assert(err, IsNil) partsUpload = append(partsUpload, part) _, err = s.bucket.CompleteMultipartUpload(imurUpload, partsUpload) c.Assert(err, IsNil) // DeleteObject err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestCopyObjectToOrFromNegative func (s *OssCrcSuite) TestAppendObjectNegative(c *C) { objectName := objectNamePrefix + "taoncrc" objectValue := "空山不见人，但闻人语响。返影入深林，复照青苔上。" nextPos, err := s.bucket.AppendObject(objectName, strings.NewReader(objectValue), 0, InitCRC(0)) c.Assert(err, IsNil) nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos, InitCRC(0)) c.Assert(err, NotNil) c.Assert(strings.HasPrefix(err.Error(), "oss: the crc"), Equals, true) } aliyun-oss-go-sdk-1.5.0/oss/download.go000066400000000000000000000274421311621456000177540ustar00rootroot00000000000000package oss import ( "crypto/md5" "encoding/base64" "encoding/json" "errors" "io" "io/ioutil" "os" "strconv" ) // // DownloadFile 分片下载文件 // // objectKey object key。 // filePath 本地文件。objectKey下载到文件。 // partSize 本次上传文件片的大小，字节数。比如100 * 1024为每片100KB。 // options Object的属性限制项。详见GetObject。 // // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) DownloadFile(objectKey, filePath string, partSize int64, options ...Option) error { if partSize < 1 || partSize > MaxPartSize { return errors.New("oss: part size invalid range (1, 5GB]") } cpConf, err := getCpConfig(options, filePath) if err != nil { return err } uRange, err := getRangeConfig(options) if err != nil { return err } routines := getRoutines(options) if cpConf.IsEnable { return bucket.downloadFileWithCp(objectKey, filePath, partSize, options, cpConf.FilePath, routines, uRange) } return bucket.downloadFile(objectKey, filePath, partSize, options, routines, uRange) } // 获取下载范围 func getRangeConfig(options []Option) (*unpackedRange, error) { rangeOpt, err := findOption(options, HTTPHeaderRange, nil) if err != nil || rangeOpt == nil { return nil, err } return parseRange(rangeOpt.(string)) } // ----- 并发无断点的下载 ----- // 工作协程参数 type downloadWorkerArg struct { bucket *Bucket key string filePath string options []Option hook downloadPartHook } // Hook用于测试 type downloadPartHook func(part downloadPart) error var downloadPartHooker downloadPartHook = defaultDownloadPartHook func defaultDownloadPartHook(part downloadPart) error { return nil } // 默认ProgressListener，屏蔽GetObject的Options中ProgressListener type defaultDownloadProgressListener struct { } // ProgressChanged 静默处理 func (listener *defaultDownloadProgressListener) ProgressChanged(event *ProgressEvent) { } // 工作协程 func downloadWorker(id int, arg downloadWorkerArg, jobs <-chan downloadPart, results chan<- downloadPart, failed chan<- error, die <-chan bool) { for part := range jobs { if err := arg.hook(part); err != nil { failed <- err break } // resolve options r := Range(part.Start, part.End) p := Progress(&defaultDownloadProgressListener{}) opts := make([]Option, len(arg.options)+2) // append orderly, can not be reversed! opts = append(opts, arg.options...) opts = append(opts, r, p) rd, err := arg.bucket.GetObject(arg.key, opts...) if err != nil { failed <- err break } defer rd.Close() select { case <-die: return default: } fd, err := os.OpenFile(arg.filePath, os.O_WRONLY, FilePermMode) if err != nil { failed <- err break } defer fd.Close() _, err = fd.Seek(part.Start-part.Offset, os.SEEK_SET) if err != nil { failed <- err break } _, err = io.Copy(fd, rd) if err != nil { failed <- err break } results <- part } } // 调度协程 func downloadScheduler(jobs chan downloadPart, parts []downloadPart) { for _, part := range parts { jobs <- part } close(jobs) } // 下载片 type downloadPart struct { Index int // 片序号，从0开始编号 Start int64 // 片起始位置 End int64 // 片结束位置 Offset int64 // 偏移位置 } // 文件分片 func getDownloadParts(bucket *Bucket, objectKey string, partSize int64, uRange *unpackedRange) ([]downloadPart, error) { meta, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { return nil, err } parts := []downloadPart{} objectSize, err := strconv.ParseInt(meta.Get(HTTPHeaderContentLength), 10, 0) if err != nil { return nil, err } part := downloadPart{} i := 0 start, end := adjustRange(uRange, objectSize) for offset := start; offset < end; offset += partSize { part.Index = i part.Start = offset part.End = GetPartEnd(offset, end, partSize) part.Offset = start parts = append(parts, part) i++ } return parts, nil } // 文件大小 func getObjectBytes(parts []downloadPart) int64 { var ob int64 for _, part := range parts { ob += (part.End - part.Start + 1) } return ob } // 并发无断点续传的下载 func (bucket Bucket) downloadFile(objectKey, filePath string, partSize int64, options []Option, routines int, uRange *unpackedRange) error { tempFilePath := filePath + TempFileSuffix listener := getProgressListener(options) // 如果文件不存在则创建，存在不清空，下载分片会重写文件内容 fd, err := os.OpenFile(tempFilePath, os.O_WRONLY|os.O_CREATE, FilePermMode) if err != nil { return err } fd.Close() // 分割文件 parts, err := getDownloadParts(&bucket, objectKey, partSize, uRange) if err != nil { return err } jobs := make(chan downloadPart, len(parts)) results := make(chan downloadPart, len(parts)) failed := make(chan error) die := make(chan bool) var completedBytes int64 totalBytes := getObjectBytes(parts) event := newProgressEvent(TransferStartedEvent, 0, totalBytes) publishProgress(listener, event) // 启动工作协程 arg := downloadWorkerArg{&bucket, objectKey, tempFilePath, options, downloadPartHooker} for w := 1; w <= routines; w++ { go downloadWorker(w, arg, jobs, results, failed, die) } // 并发上传分片 go downloadScheduler(jobs, parts) // 等待分片下载完成 completed := 0 ps := make([]downloadPart, len(parts)) for completed < len(parts) { select { case part := <-results: completed++ ps[part.Index] = part completedBytes += (part.End - part.Start + 1) event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes) publishProgress(listener, event) case err := <-failed: close(die) event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes) publishProgress(listener, event) return err } if completed >= len(parts) { break } } event = newProgressEvent(TransferCompletedEvent, completedBytes, totalBytes) publishProgress(listener, event) return os.Rename(tempFilePath, filePath) } // ----- 并发有断点的下载 ----- const downloadCpMagic = "92611BED-89E2-46B6-89E5-72F273D4B0A3" type downloadCheckpoint struct { Magic string // magic MD5 string // cp内容的MD5 FilePath string // 本地文件 Object string // key ObjStat objectStat // 文件状态 Parts []downloadPart // 全部分片 PartStat []bool // 分片下载是否完成 Start int64 // 起点 End int64 // 终点 } type objectStat struct { Size int64 // 大小 LastModified string // 最后修改时间 Etag string // etag } // CP数据是否有效，CP有效且Object没有更新时有效 func (cp downloadCheckpoint) isValid(bucket *Bucket, objectKey string, uRange *unpackedRange) (bool, error) { // 比较CP的Magic及MD5 cpb := cp cpb.MD5 = "" js, _ := json.Marshal(cpb) sum := md5.Sum(js) b64 := base64.StdEncoding.EncodeToString(sum[:]) if cp.Magic != downloadCpMagic || b64 != cp.MD5 { return false, nil } // 确认object没有更新 meta, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { return false, err } objectSize, err := strconv.ParseInt(meta.Get(HTTPHeaderContentLength), 10, 0) if err != nil { return false, err } // 比较Object的大小/最后修改时间/etag if cp.ObjStat.Size != objectSize || cp.ObjStat.LastModified != meta.Get(HTTPHeaderLastModified) || cp.ObjStat.Etag != meta.Get(HTTPHeaderEtag) { return false, nil } // 确认下载范围是否变化 if uRange != nil { start, end := adjustRange(uRange, objectSize) if start != cp.Start || end != cp.End { return false, nil } } return true, nil } // 从文件中load func (cp *downloadCheckpoint) load(filePath string) error { contents, err := ioutil.ReadFile(filePath) if err != nil { return err } err = json.Unmarshal(contents, cp) return err } // dump到文件 func (cp *downloadCheckpoint) dump(filePath string) error { bcp := *cp // 计算MD5 bcp.MD5 = "" js, err := json.Marshal(bcp) if err != nil { return err } sum := md5.Sum(js) b64 := base64.StdEncoding.EncodeToString(sum[:]) bcp.MD5 = b64 // 序列化 js, err = json.Marshal(bcp) if err != nil { return err } // dump return ioutil.WriteFile(filePath, js, FilePermMode) } // 未完成的分片 func (cp downloadCheckpoint) todoParts() []downloadPart { dps := []downloadPart{} for i, ps := range cp.PartStat { if !ps { dps = append(dps, cp.Parts[i]) } } return dps } // 完成的字节数 func (cp downloadCheckpoint) getCompletedBytes() int64 { var completedBytes int64 for i, part := range cp.Parts { if cp.PartStat[i] { completedBytes += (part.End - part.Start + 1) } } return completedBytes } // 初始化下载任务 func (cp *downloadCheckpoint) prepare(bucket *Bucket, objectKey, filePath string, partSize int64, uRange *unpackedRange) error { // cp cp.Magic = downloadCpMagic cp.FilePath = filePath cp.Object = objectKey // object meta, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { return err } objectSize, err := strconv.ParseInt(meta.Get(HTTPHeaderContentLength), 10, 0) if err != nil { return err } cp.ObjStat.Size = objectSize cp.ObjStat.LastModified = meta.Get(HTTPHeaderLastModified) cp.ObjStat.Etag = meta.Get(HTTPHeaderEtag) // parts cp.Parts, err = getDownloadParts(bucket, objectKey, partSize, uRange) if err != nil { return err } cp.PartStat = make([]bool, len(cp.Parts)) for i := range cp.PartStat { cp.PartStat[i] = false } return nil } func (cp *downloadCheckpoint) complete(cpFilePath, downFilepath string) error { os.Remove(cpFilePath) return os.Rename(downFilepath, cp.FilePath) } // 并发带断点的下载 func (bucket Bucket) downloadFileWithCp(objectKey, filePath string, partSize int64, options []Option, cpFilePath string, routines int, uRange *unpackedRange) error { tempFilePath := filePath + TempFileSuffix listener := getProgressListener(options) // LOAD CP数据 dcp := downloadCheckpoint{} err := dcp.load(cpFilePath) if err != nil { os.Remove(cpFilePath) } // LOAD出错或数据无效重新初始化下载 valid, err := dcp.isValid(&bucket, objectKey, uRange) if err != nil || !valid { if err = dcp.prepare(&bucket, objectKey, filePath, partSize, uRange); err != nil { return err } os.Remove(cpFilePath) } // 如果文件不存在则创建，存在不清空，下载分片会重写文件内容 fd, err := os.OpenFile(tempFilePath, os.O_WRONLY|os.O_CREATE, FilePermMode) if err != nil { return err } fd.Close() // 未完成的分片 parts := dcp.todoParts() jobs := make(chan downloadPart, len(parts)) results := make(chan downloadPart, len(parts)) failed := make(chan error) die := make(chan bool) completedBytes := dcp.getCompletedBytes() event := newProgressEvent(TransferStartedEvent, completedBytes, dcp.ObjStat.Size) publishProgress(listener, event) // 启动工作协程 arg := downloadWorkerArg{&bucket, objectKey, tempFilePath, options, downloadPartHooker} for w := 1; w <= routines; w++ { go downloadWorker(w, arg, jobs, results, failed, die) } // 并发下载分片 go downloadScheduler(jobs, parts) // 等待分片下载完成 completed := 0 for completed < len(parts) { select { case part := <-results: completed++ dcp.PartStat[part.Index] = true dcp.dump(cpFilePath) completedBytes += (part.End - part.Start + 1) event = newProgressEvent(TransferDataEvent, completedBytes, dcp.ObjStat.Size) publishProgress(listener, event) case err := <-failed: close(die) event = newProgressEvent(TransferFailedEvent, completedBytes, dcp.ObjStat.Size) publishProgress(listener, event) return err } if completed >= len(parts) { break } } event = newProgressEvent(TransferCompletedEvent, completedBytes, dcp.ObjStat.Size) publishProgress(listener, event) return dcp.complete(cpFilePath, tempFilePath) } aliyun-oss-go-sdk-1.5.0/oss/download_test.go000066400000000000000000000422611311621456000210070ustar00rootroot00000000000000package oss import ( "bytes" "fmt" "os" "time" . "gopkg.in/check.v1" ) type OssDownloadSuite struct { client *Client bucket *Bucket } var _ = Suite(&OssDownloadSuite{}) // Run once when the suite starts running func (s *OssDownloadSuite) SetUpSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) s.client = client s.client.CreateBucket(bucketName) time.Sleep(5 * time.Second) bucket, err := s.client.Bucket(bucketName) c.Assert(err, IsNil) s.bucket = bucket testLogger.Println("test download started") } // Run before each test or benchmark starts running func (s *OssDownloadSuite) TearDownSuite(c *C) { // Delete Part lmur, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmur.Uploads { var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName, Key: upload.Key, UploadID: upload.UploadID} err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } // Delete Objects lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = s.bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } testLogger.Println("test download completed") } // Run after each test or benchmark runs func (s *OssDownloadSuite) SetUpTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // Run once after all tests or benchmarks have finished running func (s *OssDownloadSuite) TearDownTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".temp") c.Assert(err, IsNil) } // TestUploadRoutineWithoutRecovery 多线程无断点恢复的下载 func (s *OssDownloadSuite) TestDownloadRoutineWithoutRecovery(c *C) { objectName := objectNamePrefix + "tdrwr" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "down-new-file.jpg" // 上传文件 err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) // 使用默认值下载 err = s.bucket.DownloadFile(objectName, newFile, 100*1024) c.Assert(err, IsNil) // check eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 使用2个协程下载，小于总分片数5 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2)) c.Assert(err, IsNil) // check eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 使用5个协程下载，等于总分片数5 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(5)) c.Assert(err, IsNil) // check eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 使用10个协程下载，大于总分片数5 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10)) c.Assert(err, IsNil) // check eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // ErrorHooker DownloadPart请求Hook func DownErrorHooker(part downloadPart) error { if part.Index == 4 { time.Sleep(time.Second) return fmt.Errorf("ErrorHooker") } return nil } // TestDownloadRoutineWithRecovery 多线程有断点恢复的下载 func (s *OssDownloadSuite) TestDownloadRoutineWithRecovery(c *C) { objectName := objectNamePrefix + "tdrtr" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "down-new-file-2.jpg" // 上传文件 err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) // 下载，CP使用默认值 downloadPartHooker = DownErrorHooker err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, "")) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") downloadPartHooker = defaultDownloadPartHook // check dcp := downloadCheckpoint{} err = dcp.load(newFile + ".cp") c.Assert(err, IsNil) c.Assert(dcp.Magic, Equals, downloadCpMagic) c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ==")) c.Assert(dcp.FilePath, Equals, newFile) c.Assert(dcp.ObjStat.Size, Equals, int64(482048)) c.Assert(len(dcp.ObjStat.LastModified), Equals, len("2015-12-17 18:43:03 +0800 CST")) c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"") c.Assert(dcp.Object, Equals, objectName) c.Assert(len(dcp.Parts), Equals, 5) c.Assert(len(dcp.todoParts()), Equals, 1) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, "")) c.Assert(err, IsNil) err = dcp.load(newFile + ".cp") c.Assert(err, NotNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 下载，指定CP os.Remove(newFile) downloadPartHooker = DownErrorHooker err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, objectName+".cp")) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") downloadPartHooker = defaultDownloadPartHook // check dcp = downloadCheckpoint{} err = dcp.load(objectName + ".cp") c.Assert(err, IsNil) c.Assert(dcp.Magic, Equals, downloadCpMagic) c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ==")) c.Assert(dcp.FilePath, Equals, newFile) c.Assert(dcp.ObjStat.Size, Equals, int64(482048)) c.Assert(len(dcp.ObjStat.LastModified), Equals, len("2015-12-17 18:43:03 +0800 CST")) c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"") c.Assert(dcp.Object, Equals, objectName) c.Assert(len(dcp.Parts), Equals, 5) c.Assert(len(dcp.todoParts()), Equals, 1) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, objectName+".cp")) c.Assert(err, IsNil) err = dcp.load(objectName + ".cp") c.Assert(err, NotNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 一次完成下载，中间没有错误 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, "")) c.Assert(err, IsNil) err = dcp.load(newFile + ".cp") c.Assert(err, NotNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 一次完成下载，中间没有错误 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10), Checkpoint(true, "")) c.Assert(err, IsNil) err = dcp.load(newFile + ".cp") c.Assert(err, NotNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestDownloadOption 选项 func (s *OssDownloadSuite) TestDownloadOption(c *C) { objectName := objectNamePrefix + "tdmo" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "down-new-file-3.jpg" // 上传文件 err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) // IfMatch os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag"))) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // IfNoneMatch os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag"))) c.Assert(err, NotNil) // IfMatch err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Checkpoint(true, ""), IfMatch(meta.Get("Etag"))) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // IfNoneMatch err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Checkpoint(true, ""), IfNoneMatch(meta.Get("Etag"))) c.Assert(err, NotNil) } // TestDownloadObjectChange 上传过程中文件修改了 func (s *OssDownloadSuite) TestDownloadObjectChange(c *C) { objectName := objectNamePrefix + "tdloc" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "down-new-file-4.jpg" // 上传文件 err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) // 下载，CP使用默认值 downloadPartHooker = DownErrorHooker err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, "")) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") downloadPartHooker = defaultDownloadPartHook err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, "")) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) } // TestDownloadNegative Download Negative func (s *OssDownloadSuite) TestDownloadNegative(c *C) { objectName := objectNamePrefix + "tdn" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "down-new-file-3.jpg" // 上传文件 err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) // worker线程错误 downloadPartHooker = DownErrorHooker err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2)) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") downloadPartHooker = defaultDownloadPartHook // 本地文件不存在 err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2)) c.Assert(err, NotNil) // 指定的分片大小无效 err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2)) c.Assert(err, NotNil) err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2)) c.Assert(err, NotNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 本地文件不存在 err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2), Checkpoint(true, "")) c.Assert(err, NotNil) // 指定的分片大小无效 err = s.bucket.DownloadFile(objectName, newFile, -1, Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2), Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2), Checkpoint(true, "")) c.Assert(err, NotNil) } // TestDownloadWithRange 带范围的并发下载、断点下载测试 func (s *OssDownloadSuite) TestDownloadWithRange(c *C) { objectName := objectNamePrefix + "tdwr" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "down-new-file-tdwr.jpg" newFileGet := "down-new-file-tdwr-2.jpg" // 上传文件 err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) fileSize, err := getFileSize(fileName) c.Assert(err, IsNil) // 范围下载，从1024到4096 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Range(1024, 4095)) c.Assert(err, IsNil) // check eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFileGet) err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095)) c.Assert(err, IsNil) // compare get and download eq, err = compareFiles(newFile, newFileGet) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 范围下载，从1024到4096 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("1024-4095")) c.Assert(err, IsNil) // check eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFileGet) err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095")) c.Assert(err, IsNil) // compare get and download eq, err = compareFiles(newFile, newFileGet) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 范围下载，从2048到结束 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), NormalizedRange("2048-")) c.Assert(err, IsNil) // check eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFileGet) err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-")) c.Assert(err, IsNil) // compare get and download eq, err = compareFiles(newFile, newFileGet) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 范围下载，最后4096个字节 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("-4096")) c.Assert(err, IsNil) // check eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFileGet) err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096")) c.Assert(err, IsNil) // compare get and download eq, err = compareFiles(newFile, newFileGet) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestDownloadWithCheckoutAndRange 带范围的并发下载、断点下载测试 func (s *OssDownloadSuite) TestDownloadWithCheckoutAndRange(c *C) { objectName := objectNamePrefix + "tdwcr" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "down-new-file-tdwcr.jpg" newFileGet := "down-new-file-tdwcr-2.jpg" // 上传文件 err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) fileSize, err := getFileSize(fileName) c.Assert(err, IsNil) // 范围下载，从1024到4096 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Checkpoint(true, ""), Range(1024, 4095)) c.Assert(err, IsNil) // check eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFileGet) err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095)) c.Assert(err, IsNil) // compare get and download eq, err = compareFiles(newFile, newFileGet) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 范围下载，从1024到4096 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, ""), NormalizedRange("1024-4095")) c.Assert(err, IsNil) // check eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFileGet) err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095")) c.Assert(err, IsNil) // compare get and download eq, err = compareFiles(newFile, newFileGet) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 范围下载，从2048到结束 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), Checkpoint(true, ""), NormalizedRange("2048-")) c.Assert(err, IsNil) // check eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFileGet) err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-")) c.Assert(err, IsNil) // compare get and download eq, err = compareFiles(newFile, newFileGet) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 范围下载，最后4096个字节 os.Remove(newFile) err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, ""), NormalizedRange("-4096")) c.Assert(err, IsNil) // check eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096) c.Assert(err, IsNil) c.Assert(eq, Equals, true) os.Remove(newFileGet) err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096")) c.Assert(err, IsNil) // compare get and download eq, err = compareFiles(newFile, newFileGet) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } func getFileSize(fileName string) (int64, error) { file, err := os.Open(fileName) if err != nil { return 0, err } defer file.Close() stat, err := file.Stat() if err != nil { return 0, err } return stat.Size(), nil } // compare the content between fileL and fileR with specified range func compareFilesWithRange(fileL string, offsetL int64, fileR string, offsetR int64, size int64) (bool, error) { finL, err := os.Open(fileL) if err != nil { return false, err } defer finL.Close() finL.Seek(offsetL, os.SEEK_SET) finR, err := os.Open(fileR) if err != nil { return false, err } defer finR.Close() finR.Seek(offsetR, os.SEEK_SET) statL, err := finL.Stat() if err != nil { return false, err } statR, err := finR.Stat() if err != nil { return false, err } if (offsetL+size > statL.Size()) || (offsetR+size > statR.Size()) { return false, nil } part := statL.Size() - offsetL if part > 16*1024 { part = 16 * 1024 } bufL := make([]byte, part) bufR := make([]byte, part) for readN := int64(0); readN < size; { n, _ := finL.Read(bufL) if 0 == n { break } n, _ = finR.Read(bufR) if 0 == n { break } tailer := part if tailer > size-readN { tailer = size - readN } readN += tailer if !bytes.Equal(bufL[0:tailer], bufR[0:tailer]) { return false, nil } } return true, nil } aliyun-oss-go-sdk-1.5.0/oss/error.go000066400000000000000000000053351311621456000172730ustar00rootroot00000000000000package oss import ( "encoding/xml" "fmt" "net/http" "strings" ) // ServiceError contains fields of the error response from Oss Service REST API. type ServiceError struct { XMLName xml.Name `xml:"Error"` Code string `xml:"Code"` // OSS返回给用户的错误码 Message string `xml:"Message"` // OSS给出的详细错误信息 RequestID string `xml:"RequestId"` // 用于唯一标识该次请求的UUID HostID string `xml:"HostId"` // 用于标识访问的OSS集群 RawMessage string // OSS返回的原始消息内容 StatusCode int // HTTP状态码 } // Implement interface error func (e ServiceError) Error() string { return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s", e.StatusCode, e.Code, e.Message, e.RequestID) } // UnexpectedStatusCodeError is returned when a storage service responds with neither an error // nor with an HTTP status code indicating success. type UnexpectedStatusCodeError struct { allowed []int // 预期OSS返回HTTP状态码 got int // OSS实际返回HTTP状态码 } // Implement interface error func (e UnexpectedStatusCodeError) Error() string { s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) } got := s(e.got) expected := []string{} for _, v := range e.allowed { expected = append(expected, s(v)) } return fmt.Sprintf("oss: status code from service response is %s; was expecting %s", got, strings.Join(expected, " or ")) } // Got is the actual status code returned by oss. func (e UnexpectedStatusCodeError) Got() int { return e.got } // checkRespCode returns UnexpectedStatusError if the given response code is not // one of the allowed status codes; otherwise nil. func checkRespCode(respCode int, allowed []int) error { for _, v := range allowed { if respCode == v { return nil } } return UnexpectedStatusCodeError{allowed, respCode} } // CRCCheckError is returned when crc check is inconsistent between client and server type CRCCheckError struct { clientCRC uint64 // 客户端计算的CRC64值 serverCRC uint64 // 服务端计算的CRC64值 operation string // 上传操作，如PutObject/AppendObject/UploadPart等 requestID string // 本次操作的RequestID } // Implement interface error func (e CRCCheckError) Error() string { return fmt.Sprintf("oss: the crc of %s is inconsistent, client %d but server %d; request id is %s", e.operation, e.clientCRC, e.serverCRC, e.requestID) } func checkCRC(resp *Response, operation string) error { if resp.Headers.Get(HTTPHeaderOssCRC64) == "" || resp.ClientCRC == resp.ServerCRC { return nil } return CRCCheckError{resp.ClientCRC, resp.ServerCRC, operation, resp.Headers.Get(HTTPHeaderOssRequestID)} } aliyun-oss-go-sdk-1.5.0/oss/error_test.go000066400000000000000000000025411311621456000203260ustar00rootroot00000000000000package oss import ( "math" "net/http" . "gopkg.in/check.v1" ) type OssErrorSuite struct{} var _ = Suite(&OssErrorSuite{}) func (s *OssErrorSuite) TestCheckCRCHasCRCInResp(c *C) { headers := http.Header{ "Expires": {"-1"}, "Content-Length": {"0"}, "Content-Encoding": {"gzip"}, "X-Oss-Hash-Crc64ecma": {"0"}, } resp := &Response{ StatusCode: 200, Headers: headers, Body: nil, ClientCRC: math.MaxUint64, ServerCRC: math.MaxUint64, } err := checkCRC(resp, "test") c.Assert(err, IsNil) } func (s *OssErrorSuite) TestCheckCRCNotHasCRCInResp(c *C) { headers := http.Header{ "Expires": {"-1"}, "Content-Length": {"0"}, "Content-Encoding": {"gzip"}, } resp := &Response{ StatusCode: 200, Headers: headers, Body: nil, ClientCRC: math.MaxUint64, ServerCRC: math.MaxUint32, } err := checkCRC(resp, "test") c.Assert(err, IsNil) } func (s *OssErrorSuite) TestCheckCRCCNegative(c *C) { headers := http.Header{ "Expires": {"-1"}, "Content-Length": {"0"}, "Content-Encoding": {"gzip"}, "X-Oss-Hash-Crc64ecma": {"0"}, } resp := &Response{ StatusCode: 200, Headers: headers, Body: nil, ClientCRC: 0, ServerCRC: math.MaxUint64, } err := checkCRC(resp, "test") c.Assert(err, NotNil) testLogger.Println("error:", err) } aliyun-oss-go-sdk-1.5.0/oss/mime.go000066400000000000000000000216431311621456000170710ustar00rootroot00000000000000package oss import ( "mime" "path" "strings" ) var extToMimeType = map[string]string{ ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ".xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", ".potx": "application/vnd.openxmlformats-officedocument.presentationml.template", ".ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", ".sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", ".xlam": "application/vnd.ms-excel.addin.macroEnabled.12", ".xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12", ".apk": "application/vnd.android.package-archive", ".hqx": "application/mac-binhex40", ".cpt": "application/mac-compactpro", ".doc": "application/msword", ".ogg": "application/ogg", ".pdf": "application/pdf", ".rtf": "text/rtf", ".mif": "application/vnd.mif", ".xls": "application/vnd.ms-excel", ".ppt": "application/vnd.ms-powerpoint", ".odc": "application/vnd.oasis.opendocument.chart", ".odb": "application/vnd.oasis.opendocument.database", ".odf": "application/vnd.oasis.opendocument.formula", ".odg": "application/vnd.oasis.opendocument.graphics", ".otg": "application/vnd.oasis.opendocument.graphics-template", ".odi": "application/vnd.oasis.opendocument.image", ".odp": "application/vnd.oasis.opendocument.presentation", ".otp": "application/vnd.oasis.opendocument.presentation-template", ".ods": "application/vnd.oasis.opendocument.spreadsheet", ".ots": "application/vnd.oasis.opendocument.spreadsheet-template", ".odt": "application/vnd.oasis.opendocument.text", ".odm": "application/vnd.oasis.opendocument.text-master", ".ott": "application/vnd.oasis.opendocument.text-template", ".oth": "application/vnd.oasis.opendocument.text-web", ".sxw": "application/vnd.sun.xml.writer", ".stw": "application/vnd.sun.xml.writer.template", ".sxc": "application/vnd.sun.xml.calc", ".stc": "application/vnd.sun.xml.calc.template", ".sxd": "application/vnd.sun.xml.draw", ".std": "application/vnd.sun.xml.draw.template", ".sxi": "application/vnd.sun.xml.impress", ".sti": "application/vnd.sun.xml.impress.template", ".sxg": "application/vnd.sun.xml.writer.global", ".sxm": "application/vnd.sun.xml.math", ".sis": "application/vnd.symbian.install", ".wbxml": "application/vnd.wap.wbxml", ".wmlc": "application/vnd.wap.wmlc", ".wmlsc": "application/vnd.wap.wmlscriptc", ".bcpio": "application/x-bcpio", ".torrent": "application/x-bittorrent", ".bz2": "application/x-bzip2", ".vcd": "application/x-cdlink", ".pgn": "application/x-chess-pgn", ".cpio": "application/x-cpio", ".csh": "application/x-csh", ".dvi": "application/x-dvi", ".spl": "application/x-futuresplash", ".gtar": "application/x-gtar", ".hdf": "application/x-hdf", ".jar": "application/x-java-archive", ".jnlp": "application/x-java-jnlp-file", ".js": "application/x-javascript", ".ksp": "application/x-kspread", ".chrt": "application/x-kchart", ".kil": "application/x-killustrator", ".latex": "application/x-latex", ".rpm": "application/x-rpm", ".sh": "application/x-sh", ".shar": "application/x-shar", ".swf": "application/x-shockwave-flash", ".sit": "application/x-stuffit", ".sv4cpio": "application/x-sv4cpio", ".sv4crc": "application/x-sv4crc", ".tar": "application/x-tar", ".tcl": "application/x-tcl", ".tex": "application/x-tex", ".man": "application/x-troff-man", ".me": "application/x-troff-me", ".ms": "application/x-troff-ms", ".ustar": "application/x-ustar", ".src": "application/x-wais-source", ".zip": "application/zip", ".m3u": "audio/x-mpegurl", ".ra": "audio/x-pn-realaudio", ".wav": "audio/x-wav", ".wma": "audio/x-ms-wma", ".wax": "audio/x-ms-wax", ".pdb": "chemical/x-pdb", ".xyz": "chemical/x-xyz", ".bmp": "image/bmp", ".gif": "image/gif", ".ief": "image/ief", ".png": "image/png", ".wbmp": "image/vnd.wap.wbmp", ".ras": "image/x-cmu-raster", ".pnm": "image/x-portable-anymap", ".pbm": "image/x-portable-bitmap", ".pgm": "image/x-portable-graymap", ".ppm": "image/x-portable-pixmap", ".rgb": "image/x-rgb", ".xbm": "image/x-xbitmap", ".xpm": "image/x-xpixmap", ".xwd": "image/x-xwindowdump", ".css": "text/css", ".rtx": "text/richtext", ".tsv": "text/tab-separated-values", ".jad": "text/vnd.sun.j2me.app-descriptor", ".wml": "text/vnd.wap.wml", ".wmls": "text/vnd.wap.wmlscript", ".etx": "text/x-setext", ".mxu": "video/vnd.mpegurl", ".flv": "video/x-flv", ".wm": "video/x-ms-wm", ".wmv": "video/x-ms-wmv", ".wmx": "video/x-ms-wmx", ".wvx": "video/x-ms-wvx", ".avi": "video/x-msvideo", ".movie": "video/x-sgi-movie", ".ice": "x-conference/x-cooltalk", ".3gp": "video/3gpp", ".ai": "application/postscript", ".aif": "audio/x-aiff", ".aifc": "audio/x-aiff", ".aiff": "audio/x-aiff", ".asc": "text/plain", ".atom": "application/atom+xml", ".au": "audio/basic", ".bin": "application/octet-stream", ".cdf": "application/x-netcdf", ".cgm": "image/cgm", ".class": "application/octet-stream", ".dcr": "application/x-director", ".dif": "video/x-dv", ".dir": "application/x-director", ".djv": "image/vnd.djvu", ".djvu": "image/vnd.djvu", ".dll": "application/octet-stream", ".dmg": "application/octet-stream", ".dms": "application/octet-stream", ".dtd": "application/xml-dtd", ".dv": "video/x-dv", ".dxr": "application/x-director", ".eps": "application/postscript", ".exe": "application/octet-stream", ".ez": "application/andrew-inset", ".gram": "application/srgs", ".grxml": "application/srgs+xml", ".gz": "application/x-gzip", ".htm": "text/html", ".html": "text/html", ".ico": "image/x-icon", ".ics": "text/calendar", ".ifb": "text/calendar", ".iges": "model/iges", ".igs": "model/iges", ".jp2": "image/jp2", ".jpe": "image/jpeg", ".jpeg": "image/jpeg", ".jpg": "image/jpeg", ".kar": "audio/midi", ".lha": "application/octet-stream", ".lzh": "application/octet-stream", ".m4a": "audio/mp4a-latm", ".m4p": "audio/mp4a-latm", ".m4u": "video/vnd.mpegurl", ".m4v": "video/x-m4v", ".mac": "image/x-macpaint", ".mathml": "application/mathml+xml", ".mesh": "model/mesh", ".mid": "audio/midi", ".midi": "audio/midi", ".mov": "video/quicktime", ".mp2": "audio/mpeg", ".mp3": "audio/mpeg", ".mp4": "video/mp4", ".mpe": "video/mpeg", ".mpeg": "video/mpeg", ".mpg": "video/mpeg", ".mpga": "audio/mpeg", ".msh": "model/mesh", ".nc": "application/x-netcdf", ".oda": "application/oda", ".ogv": "video/ogv", ".pct": "image/pict", ".pic": "image/pict", ".pict": "image/pict", ".pnt": "image/x-macpaint", ".pntg": "image/x-macpaint", ".ps": "application/postscript", ".qt": "video/quicktime", ".qti": "image/x-quicktime", ".qtif": "image/x-quicktime", ".ram": "audio/x-pn-realaudio", ".rdf": "application/rdf+xml", ".rm": "application/vnd.rn-realmedia", ".roff": "application/x-troff", ".sgm": "text/sgml", ".sgml": "text/sgml", ".silo": "model/mesh", ".skd": "application/x-koan", ".skm": "application/x-koan", ".skp": "application/x-koan", ".skt": "application/x-koan", ".smi": "application/smil", ".smil": "application/smil", ".snd": "audio/basic", ".so": "application/octet-stream", ".svg": "image/svg+xml", ".t": "application/x-troff", ".texi": "application/x-texinfo", ".texinfo": "application/x-texinfo", ".tif": "image/tiff", ".tiff": "image/tiff", ".tr": "application/x-troff", ".txt": "text/plain", ".vrml": "model/vrml", ".vxml": "application/voicexml+xml", ".webm": "video/webm", ".wrl": "model/vrml", ".xht": "application/xhtml+xml", ".xhtml": "application/xhtml+xml", ".xml": "application/xml", ".xsl": "application/xml", ".xslt": "application/xslt+xml", ".xul": "application/vnd.mozilla.xul+xml", } // TypeByExtension returns the MIME type associated with the file extension ext. // 获取文件类型，选项ContentType使用 func TypeByExtension(filePath string) string { typ := mime.TypeByExtension(path.Ext(filePath)) if typ == "" { typ = extToMimeType[strings.ToLower(path.Ext(filePath))] } return typ } aliyun-oss-go-sdk-1.5.0/oss/model.go000066400000000000000000000021531311621456000172350ustar00rootroot00000000000000package oss import ( "hash" "io" "net/http" ) // Response Http response from oss type Response struct { StatusCode int Headers http.Header Body io.ReadCloser ClientCRC uint64 ServerCRC uint64 } // PutObjectRequest The request of DoPutObject type PutObjectRequest struct { ObjectKey string Reader io.Reader } // GetObjectRequest The request of DoGetObject type GetObjectRequest struct { ObjectKey string } // GetObjectResult The result of DoGetObject type GetObjectResult struct { Response *Response ClientCRC hash.Hash64 ServerCRC uint64 } // AppendObjectRequest The requtest of DoAppendObject type AppendObjectRequest struct { ObjectKey string Reader io.Reader Position int64 } // AppendObjectResult The result of DoAppendObject type AppendObjectResult struct { NextPosition int64 CRC uint64 } // UploadPartRequest The request of DoUploadPart type UploadPartRequest struct { InitResult *InitiateMultipartUploadResult Reader io.Reader PartSize int64 PartNumber int } // UploadPartResult The result of DoUploadPart type UploadPartResult struct { Part UploadPart } aliyun-oss-go-sdk-1.5.0/oss/multicopy.go000066400000000000000000000271141311621456000201660ustar00rootroot00000000000000package oss import ( "crypto/md5" "encoding/base64" "encoding/json" "errors" "io/ioutil" "os" "path/filepath" "strconv" ) // // CopyFile 分片复制文件 // // srcBucketName 源Bucket名称。 // srcObjectKey 源Object名称。 // destObjectKey 目标Object名称。目标Bucket名称为Bucket.BucketName。 // partSize 复制文件片的大小，字节数。比如100 * 1024为每片100KB。 // options Object的属性限制项。详见InitiateMultipartUpload。 // // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) CopyFile(srcBucketName, srcObjectKey, destObjectKey string, partSize int64, options ...Option) error { destBucketName := bucket.BucketName if partSize < MinPartSize || partSize > MaxPartSize { return errors.New("oss: part size invalid range (1024KB, 5GB]") } cpConf, err := getCpConfig(options, filepath.Base(destObjectKey)) if err != nil { return err } routines := getRoutines(options) if cpConf.IsEnable { return bucket.copyFileWithCp(srcBucketName, srcObjectKey, destBucketName, destObjectKey, partSize, options, cpConf.FilePath, routines) } return bucket.copyFile(srcBucketName, srcObjectKey, destBucketName, destObjectKey, partSize, options, routines) } // ----- 并发无断点的下载 ----- // 工作协程参数 type copyWorkerArg struct { bucket *Bucket imur InitiateMultipartUploadResult srcBucketName string srcObjectKey string options []Option hook copyPartHook } // Hook用于测试 type copyPartHook func(part copyPart) error var copyPartHooker copyPartHook = defaultCopyPartHook func defaultCopyPartHook(part copyPart) error { return nil } // 工作协程 func copyWorker(id int, arg copyWorkerArg, jobs <-chan copyPart, results chan<- UploadPart, failed chan<- error, die <-chan bool) { for chunk := range jobs { if err := arg.hook(chunk); err != nil { failed <- err break } chunkSize := chunk.End - chunk.Start + 1 part, err := arg.bucket.UploadPartCopy(arg.imur, arg.srcBucketName, arg.srcObjectKey, chunk.Start, chunkSize, chunk.Number, arg.options...) if err != nil { failed <- err break } select { case <-die: return default: } results <- part } } // 调度协程 func copyScheduler(jobs chan copyPart, parts []copyPart) { for _, part := range parts { jobs <- part } close(jobs) } // 分片 type copyPart struct { Number int // 片序号[1, 10000] Start int64 // 片起始位置 End int64 // 片结束位置 } // 文件分片 func getCopyParts(bucket *Bucket, objectKey string, partSize int64) ([]copyPart, error) { meta, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { return nil, err } parts := []copyPart{} objectSize, err := strconv.ParseInt(meta.Get(HTTPHeaderContentLength), 10, 0) if err != nil { return nil, err } part := copyPart{} i := 0 for offset := int64(0); offset < objectSize; offset += partSize { part.Number = i + 1 part.Start = offset part.End = GetPartEnd(offset, objectSize, partSize) parts = append(parts, part) i++ } return parts, nil } // 获取源文件大小 func getSrcObjectBytes(parts []copyPart) int64 { var ob int64 for _, part := range parts { ob += (part.End - part.Start + 1) } return ob } // 并发无断点续传的下载 func (bucket Bucket) copyFile(srcBucketName, srcObjectKey, destBucketName, destObjectKey string, partSize int64, options []Option, routines int) error { descBucket, err := bucket.Client.Bucket(destBucketName) srcBucket, err := bucket.Client.Bucket(srcBucketName) listener := getProgressListener(options) // 分割文件 parts, err := getCopyParts(srcBucket, srcObjectKey, partSize) if err != nil { return err } // 初始化上传任务 imur, err := descBucket.InitiateMultipartUpload(destObjectKey, options...) if err != nil { return err } jobs := make(chan copyPart, len(parts)) results := make(chan UploadPart, len(parts)) failed := make(chan error) die := make(chan bool) var completedBytes int64 totalBytes := getSrcObjectBytes(parts) event := newProgressEvent(TransferStartedEvent, 0, totalBytes) publishProgress(listener, event) // 启动工作协程 arg := copyWorkerArg{descBucket, imur, srcBucketName, srcObjectKey, options, copyPartHooker} for w := 1; w <= routines; w++ { go copyWorker(w, arg, jobs, results, failed, die) } // 并发上传分片 go copyScheduler(jobs, parts) // 等待分片下载完成 completed := 0 ups := make([]UploadPart, len(parts)) for completed < len(parts) { select { case part := <-results: completed++ ups[part.PartNumber-1] = part completedBytes += (parts[part.PartNumber-1].End - parts[part.PartNumber-1].Start + 1) event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes) publishProgress(listener, event) case err := <-failed: close(die) descBucket.AbortMultipartUpload(imur) event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes) publishProgress(listener, event) return err } if completed >= len(parts) { break } } event = newProgressEvent(TransferCompletedEvent, completedBytes, totalBytes) publishProgress(listener, event) // 提交任务 _, err = descBucket.CompleteMultipartUpload(imur, ups) if err != nil { bucket.AbortMultipartUpload(imur) return err } return nil } // ----- 并发有断点的下载 ----- const copyCpMagic = "84F1F18C-FF1D-403B-A1D8-9DEB5F65910A" type copyCheckpoint struct { Magic string // magic MD5 string // cp内容的MD5 SrcBucketName string // 源Bucket SrcObjectKey string // 源Object DestBucketName string // 目标Bucket DestObjectKey string // 目标Bucket CopyID string // copy id ObjStat objectStat // 文件状态 Parts []copyPart // 全部分片 CopyParts []UploadPart // 分片上传成功后的返回值 PartStat []bool // 分片下载是否完成 } // CP数据是否有效，CP有效且Object没有更新时有效 func (cp copyCheckpoint) isValid(bucket *Bucket, objectKey string) (bool, error) { // 比较CP的Magic及MD5 cpb := cp cpb.MD5 = "" js, _ := json.Marshal(cpb) sum := md5.Sum(js) b64 := base64.StdEncoding.EncodeToString(sum[:]) if cp.Magic != downloadCpMagic || b64 != cp.MD5 { return false, nil } // 确认object没有更新 meta, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { return false, err } objectSize, err := strconv.ParseInt(meta.Get(HTTPHeaderContentLength), 10, 0) if err != nil { return false, err } // 比较Object的大小/最后修改时间/etag if cp.ObjStat.Size != objectSize || cp.ObjStat.LastModified != meta.Get(HTTPHeaderLastModified) || cp.ObjStat.Etag != meta.Get(HTTPHeaderEtag) { return false, nil } return true, nil } // 从文件中load func (cp *copyCheckpoint) load(filePath string) error { contents, err := ioutil.ReadFile(filePath) if err != nil { return err } err = json.Unmarshal(contents, cp) return err } // 更新分片状态 func (cp *copyCheckpoint) update(part UploadPart) { cp.CopyParts[part.PartNumber-1] = part cp.PartStat[part.PartNumber-1] = true } // dump到文件 func (cp *copyCheckpoint) dump(filePath string) error { bcp := *cp // 计算MD5 bcp.MD5 = "" js, err := json.Marshal(bcp) if err != nil { return err } sum := md5.Sum(js) b64 := base64.StdEncoding.EncodeToString(sum[:]) bcp.MD5 = b64 // 序列化 js, err = json.Marshal(bcp) if err != nil { return err } // dump return ioutil.WriteFile(filePath, js, FilePermMode) } // 未完成的分片 func (cp copyCheckpoint) todoParts() []copyPart { dps := []copyPart{} for i, ps := range cp.PartStat { if !ps { dps = append(dps, cp.Parts[i]) } } return dps } // 完成的字节数 func (cp copyCheckpoint) getCompletedBytes() int64 { var completedBytes int64 for i, part := range cp.Parts { if cp.PartStat[i] { completedBytes += (part.End - part.Start + 1) } } return completedBytes } // 初始化下载任务 func (cp *copyCheckpoint) prepare(srcBucket *Bucket, srcObjectKey string, destBucket *Bucket, destObjectKey string, partSize int64, options []Option) error { // cp cp.Magic = copyCpMagic cp.SrcBucketName = srcBucket.BucketName cp.SrcObjectKey = srcObjectKey cp.DestBucketName = destBucket.BucketName cp.DestObjectKey = destObjectKey // object meta, err := srcBucket.GetObjectDetailedMeta(srcObjectKey) if err != nil { return err } objectSize, err := strconv.ParseInt(meta.Get(HTTPHeaderContentLength), 10, 0) if err != nil { return err } cp.ObjStat.Size = objectSize cp.ObjStat.LastModified = meta.Get(HTTPHeaderLastModified) cp.ObjStat.Etag = meta.Get(HTTPHeaderEtag) // parts cp.Parts, err = getCopyParts(srcBucket, srcObjectKey, partSize) if err != nil { return err } cp.PartStat = make([]bool, len(cp.Parts)) for i := range cp.PartStat { cp.PartStat[i] = false } cp.CopyParts = make([]UploadPart, len(cp.Parts)) // init copy imur, err := destBucket.InitiateMultipartUpload(destObjectKey, options...) if err != nil { return err } cp.CopyID = imur.UploadID return nil } func (cp *copyCheckpoint) complete(bucket *Bucket, parts []UploadPart, cpFilePath string) error { imur := InitiateMultipartUploadResult{Bucket: cp.DestBucketName, Key: cp.DestObjectKey, UploadID: cp.CopyID} _, err := bucket.CompleteMultipartUpload(imur, parts) if err != nil { return err } os.Remove(cpFilePath) return err } // 并发带断点的下载 func (bucket Bucket) copyFileWithCp(srcBucketName, srcObjectKey, destBucketName, destObjectKey string, partSize int64, options []Option, cpFilePath string, routines int) error { descBucket, err := bucket.Client.Bucket(destBucketName) srcBucket, err := bucket.Client.Bucket(srcBucketName) listener := getProgressListener(options) // LOAD CP数据 ccp := copyCheckpoint{} err = ccp.load(cpFilePath) if err != nil { os.Remove(cpFilePath) } // LOAD出错或数据无效重新初始化下载 valid, err := ccp.isValid(srcBucket, srcObjectKey) if err != nil || !valid { if err = ccp.prepare(srcBucket, srcObjectKey, descBucket, destObjectKey, partSize, options); err != nil { return err } os.Remove(cpFilePath) } // 未完成的分片 parts := ccp.todoParts() imur := InitiateMultipartUploadResult{ Bucket: destBucketName, Key: destObjectKey, UploadID: ccp.CopyID} jobs := make(chan copyPart, len(parts)) results := make(chan UploadPart, len(parts)) failed := make(chan error) die := make(chan bool) completedBytes := ccp.getCompletedBytes() event := newProgressEvent(TransferStartedEvent, completedBytes, ccp.ObjStat.Size) publishProgress(listener, event) // 启动工作协程 arg := copyWorkerArg{descBucket, imur, srcBucketName, srcObjectKey, options, copyPartHooker} for w := 1; w <= routines; w++ { go copyWorker(w, arg, jobs, results, failed, die) } // 并发下载分片 go copyScheduler(jobs, parts) // 等待分片下载完成 completed := 0 for completed < len(parts) { select { case part := <-results: completed++ ccp.update(part) ccp.dump(cpFilePath) completedBytes += (parts[part.PartNumber-1].End - parts[part.PartNumber-1].Start + 1) event = newProgressEvent(TransferDataEvent, completedBytes, ccp.ObjStat.Size) publishProgress(listener, event) case err := <-failed: close(die) event = newProgressEvent(TransferFailedEvent, completedBytes, ccp.ObjStat.Size) publishProgress(listener, event) return err } if completed >= len(parts) { break } } event = newProgressEvent(TransferCompletedEvent, completedBytes, ccp.ObjStat.Size) publishProgress(listener, event) return ccp.complete(descBucket, ccp.CopyParts, cpFilePath) } aliyun-oss-go-sdk-1.5.0/oss/multicopy_test.go000066400000000000000000000336001311621456000212220ustar00rootroot00000000000000package oss import ( "fmt" "os" "time" . "gopkg.in/check.v1" ) type OssCopySuite struct { client *Client bucket *Bucket } var _ = Suite(&OssCopySuite{}) // Run once when the suite starts running func (s *OssCopySuite) SetUpSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) s.client = client s.client.CreateBucket(bucketName) time.Sleep(5 * time.Second) bucket, err := s.client.Bucket(bucketName) c.Assert(err, IsNil) s.bucket = bucket testLogger.Println("test copy started") } // Run before each test or benchmark starts running func (s *OssCopySuite) TearDownSuite(c *C) { // Delete Part lmur, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmur.Uploads { var imur = InitiateMultipartUploadResult{Bucket: bucketName, Key: upload.Key, UploadID: upload.UploadID} err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } //Delete Objects lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = s.bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } testLogger.Println("test copy completed") } // Run after each test or benchmark runs func (s *OssCopySuite) SetUpTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // Run once after all tests or benchmarks have finished running func (s *OssCopySuite) TearDownTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // TestCopyRoutineWithoutRecovery 多线程无断点恢复的复制 func (s *OssCopySuite) TestCopyRoutineWithoutRecovery(c *C) { srcObjectName := objectNamePrefix + "tcrwr" destObjectName := srcObjectName + "-copy" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "copy-new-file.jpg" // 上传源文件 err := s.bucket.UploadFile(srcObjectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) os.Remove(newFile) // 不指定Routines，默认单线程 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 指定线程数1 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(1)) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 指定线程数3，小于分片数5 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(3)) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 指定线程数5，等于分片数 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(5)) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 指定线程数10，大于分片数5 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(10)) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 线程值无效自动变成1 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(-1)) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // option err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(3), Meta("myprop", "mypropval")) meta, err := s.bucket.GetObjectDetailedMeta(destObjectName) c.Assert(err, IsNil) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.DeleteObject(srcObjectName) c.Assert(err, IsNil) } // CopyErrorHooker CopyPart请求Hook func CopyErrorHooker(part copyPart) error { if part.Number == 5 { time.Sleep(time.Second) return fmt.Errorf("ErrorHooker") } return nil } // TestCopyRoutineWithoutRecoveryNegative 多线程无断点恢复的复制 func (s *OssCopySuite) TestCopyRoutineWithoutRecoveryNegative(c *C) { srcObjectName := objectNamePrefix + "tcrwrn" destObjectName := srcObjectName + "-copy" fileName := "../sample/BingWallpaper-2015-11-07.jpg" // 上传源文件 err := s.bucket.UploadFile(srcObjectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) copyPartHooker = CopyErrorHooker // worker线程错误 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(2)) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") copyPartHooker = defaultCopyPartHook // 源Bucket不存在 err = s.bucket.CopyFile("NotExist", srcObjectName, destObjectName, 100*1024, Routines(2)) c.Assert(err, NotNil) // 源Object不存在 err = s.bucket.CopyFile(bucketName, "NotExist", destObjectName, 100*1024, Routines(2)) // 指定的分片大小无效 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024, Routines(2)) c.Assert(err, NotNil) err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*1024*1024*100, Routines(2)) c.Assert(err, NotNil) // 删除源文件 err = s.bucket.DeleteObject(srcObjectName) c.Assert(err, IsNil) } // TestCopyRoutineWithRecovery 多线程且有断点恢复的复制 func (s *OssCopySuite) TestCopyRoutineWithRecovery(c *C) { srcObjectName := objectNamePrefix + "tcrtr" destObjectName := srcObjectName + "-copy" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "copy-new-file.jpg" // 上传源文件 err := s.bucket.UploadFile(srcObjectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) os.Remove(newFile) // Routines默认值，CP开启默认路径是destObjectName+.cp // 第一次上传，上传4片 copyPartHooker = CopyErrorHooker err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Checkpoint(true, "")) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") copyPartHooker = defaultCopyPartHook // check cp ccp := copyCheckpoint{} err = ccp.load(destObjectName + ".cp") c.Assert(err, IsNil) c.Assert(ccp.Magic, Equals, copyCpMagic) c.Assert(len(ccp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ==")) c.Assert(ccp.SrcBucketName, Equals, bucketName) c.Assert(ccp.SrcObjectKey, Equals, srcObjectName) c.Assert(ccp.DestBucketName, Equals, bucketName) c.Assert(ccp.DestObjectKey, Equals, destObjectName) c.Assert(len(ccp.CopyID), Equals, len("3F79722737D1469980DACEDCA325BB52")) c.Assert(ccp.ObjStat.Size, Equals, int64(482048)) c.Assert(len(ccp.ObjStat.LastModified), Equals, len("2015-12-17 18:43:03 +0800 CST")) c.Assert(ccp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"") c.Assert(len(ccp.Parts), Equals, 5) c.Assert(len(ccp.todoParts()), Equals, 1) c.Assert(ccp.PartStat[4], Equals, false) // 第二次上传，完成剩余的一片 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Checkpoint(true, "")) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) err = ccp.load(fileName + ".cp") c.Assert(err, NotNil) // Routines指定，CP指定 copyPartHooker = CopyErrorHooker err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(2), Checkpoint(true, srcObjectName+".cp")) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") copyPartHooker = defaultCopyPartHook // check cp ccp = copyCheckpoint{} err = ccp.load(srcObjectName + ".cp") c.Assert(err, IsNil) c.Assert(ccp.Magic, Equals, copyCpMagic) c.Assert(len(ccp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ==")) c.Assert(ccp.SrcBucketName, Equals, bucketName) c.Assert(ccp.SrcObjectKey, Equals, srcObjectName) c.Assert(ccp.DestBucketName, Equals, bucketName) c.Assert(ccp.DestObjectKey, Equals, destObjectName) c.Assert(len(ccp.CopyID), Equals, len("3F79722737D1469980DACEDCA325BB52")) c.Assert(ccp.ObjStat.Size, Equals, int64(482048)) c.Assert(len(ccp.ObjStat.LastModified), Equals, len("2015-12-17 18:43:03 +0800 CST")) c.Assert(ccp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"") c.Assert(len(ccp.Parts), Equals, 5) c.Assert(len(ccp.todoParts()), Equals, 1) c.Assert(ccp.PartStat[4], Equals, false) // 第二次上传，完成剩余的一片 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(2), Checkpoint(true, srcObjectName+".cp")) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) err = ccp.load(srcObjectName + ".cp") c.Assert(err, NotNil) // 一次完成上传，中间没有错误 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(3), Checkpoint(true, "")) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 用多协程下载，中间没有错误 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(10), Checkpoint(true, "")) c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // option err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(5), Checkpoint(true, ""), Meta("myprop", "mypropval")) c.Assert(err, IsNil) meta, err := s.bucket.GetObjectDetailedMeta(destObjectName) c.Assert(err, IsNil) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") err = s.bucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 删除源文件 err = s.bucket.DeleteObject(srcObjectName) c.Assert(err, IsNil) } // TestCopyRoutineWithRecoveryNegative 多线程无断点恢复的复制 func (s *OssCopySuite) TestCopyRoutineWithRecoveryNegative(c *C) { srcObjectName := objectNamePrefix + "tcrwrn" destObjectName := srcObjectName + "-copy" // 源Bucket不存在 err := s.bucket.CopyFile("NotExist", srcObjectName, destObjectName, 100*1024, Checkpoint(true, "")) c.Assert(err, NotNil) c.Assert(err, NotNil) // 源Object不存在 err = s.bucket.CopyFile(bucketName, "NotExist", destObjectName, 100*1024, Routines(2), Checkpoint(true, "")) c.Assert(err, NotNil) // 指定的分片大小无效 err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024, Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*1024*1024*100, Routines(2), Checkpoint(true, "")) c.Assert(err, NotNil) } // TestCopyFileCrossBucket 跨Bucket直接的复制 func (s *OssCopySuite) TestCopyFileCrossBucket(c *C) { destBucketName := bucketName + "-cfcb-desc" srcObjectName := objectNamePrefix + "tcrtr" destObjectName := srcObjectName + "-copy" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "copy-new-file.jpg" destBucket, err := s.client.Bucket(destBucketName) c.Assert(err, IsNil) // 创建目标Bucket err = s.client.CreateBucket(destBucketName) // 上传源文件 err = s.bucket.UploadFile(srcObjectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) os.Remove(newFile) // 复制文件 err = destBucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(5), Checkpoint(true, "")) c.Assert(err, IsNil) err = destBucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = destBucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 带option的复制 err = destBucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(10), Checkpoint(true, "copy.cp"), Meta("myprop", "mypropval")) c.Assert(err, IsNil) err = destBucket.GetObjectToFile(destObjectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = destBucket.DeleteObject(destObjectName) c.Assert(err, IsNil) os.Remove(newFile) // 删除目标Bucket err = s.client.DeleteBucket(destBucketName) c.Assert(err, IsNil) } aliyun-oss-go-sdk-1.5.0/oss/multipart.go000066400000000000000000000230561311621456000201630ustar00rootroot00000000000000package oss import ( "bytes" "encoding/xml" "io" "net/http" "os" "sort" "strconv" ) // // InitiateMultipartUpload 初始化分片上传任务。 // // objectKey Object名称。 // options 上传时可以指定Object的属性，可选属性有CacheControl、ContentDisposition、ContentEncoding、Expires、 // ServerSideEncryption、Meta，具体含义请参考 // https://help.aliyun.com/document_detail/oss/api-reference/multipart-upload/InitiateMultipartUpload.html // // InitiateMultipartUploadResult 初始化后操作成功的返回值，用于后面的UploadPartFromFile、UploadPartCopy等操作。error为nil时有效。 // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) InitiateMultipartUpload(objectKey string, options ...Option) (InitiateMultipartUploadResult, error) { var imur InitiateMultipartUploadResult opts := addContentType(options, objectKey) resp, err := bucket.do("POST", objectKey, "uploads", "uploads", opts, nil, nil) if err != nil { return imur, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &imur) return imur, err } // // UploadPart 上传分片。 // // 初始化一个Multipart Upload之后，可以根据指定的Object名和Upload ID来分片（Part）上传数据。 // 每一个上传的Part都有一个标识它的号码（part number，范围是1~10000）。对于同一个Upload ID， // 该号码不但唯一标识这一片数据，也标识了这片数据在整个文件内的相对位置。如果您用同一个part号码，上传了新的数据， // 那么OSS上已有的这个号码的Part数据将被覆盖。除了最后一片Part以外，其他的part最小为100KB； // 最后一片Part没有大小限制。 // // imur InitiateMultipartUpload成功后的返回值。 // reader io.Reader 需要分片上传的reader。 // size 本次上传片Part的大小。 // partNumber 本次上传片(Part)的编号，范围是1~10000。如果超出范围，OSS将返回InvalidArgument错误。 // // UploadPart 上传成功的返回值，两个成员PartNumber、ETag。PartNumber片编号，即传入参数partNumber； // ETag及上传数据的MD5。error为nil时有效。 // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) UploadPart(imur InitiateMultipartUploadResult, reader io.Reader, partSize int64, partNumber int, options ...Option) (UploadPart, error) { request := &UploadPartRequest{ InitResult: &imur, Reader: reader, PartSize: partSize, PartNumber: partNumber, } result, err := bucket.DoUploadPart(request, options) return result.Part, err } // // UploadPartFromFile 上传分片。 // // imur InitiateMultipartUpload成功后的返回值。 // filePath 需要分片上传的本地文件。 // startPosition 本次上传文件片的起始位置。 // partSize 本次上传文件片的大小。 // partNumber 本次上传文件片的编号，范围是1~10000。 // // UploadPart 上传成功的返回值，两个成员PartNumber、ETag。PartNumber片编号，传入参数partNumber； // ETag上传数据的MD5。error为nil时有效。 // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) UploadPartFromFile(imur InitiateMultipartUploadResult, filePath string, startPosition, partSize int64, partNumber int, options ...Option) (UploadPart, error) { var part = UploadPart{} fd, err := os.Open(filePath) if err != nil { return part, err } defer fd.Close() fd.Seek(startPosition, os.SEEK_SET) request := &UploadPartRequest{ InitResult: &imur, Reader: fd, PartSize: partSize, PartNumber: partNumber, } result, err := bucket.DoUploadPart(request, options) return result.Part, err } // // DoUploadPart 上传分片。 // // request 上传分片请求。 // // UploadPartResult 上传分片请求返回值。 // error 操作无错误为nil，非nil为错误信息。 // func (bucket Bucket) DoUploadPart(request *UploadPartRequest, options []Option) (*UploadPartResult, error) { listener := getProgressListener(options) params := "partNumber=" + strconv.Itoa(request.PartNumber) + "&uploadId=" + request.InitResult.UploadID opts := []Option{ContentLength(request.PartSize)} resp, err := bucket.do("PUT", request.InitResult.Key, params, params, opts, &io.LimitedReader{R: request.Reader, N: request.PartSize}, listener) if err != nil { return &UploadPartResult{}, err } defer resp.Body.Close() part := UploadPart{ ETag: resp.Headers.Get(HTTPHeaderEtag), PartNumber: request.PartNumber, } if bucket.getConfig().IsEnableCRC { err = checkCRC(resp, "DoUploadPart") if err != nil { return &UploadPartResult{part}, err } } return &UploadPartResult{part}, nil } // // UploadPartCopy 拷贝分片。 // // imur InitiateMultipartUpload成功后的返回值。 // copySrc 源Object名称。 // startPosition 本次拷贝片(Part)在源Object的起始位置。 // partSize 本次拷贝片的大小。 // partNumber 本次拷贝片的编号，范围是1~10000。如果超出范围，OSS将返回InvalidArgument错误。 // options copy时源Object的限制条件，满足限制条件时copy，不满足时返回错误。可选条件有CopySourceIfMatch、 // CopySourceIfNoneMatch、CopySourceIfModifiedSince CopySourceIfUnmodifiedSince，具体含义请参看 // https://help.aliyun.com/document_detail/oss/api-reference/multipart-upload/UploadPartCopy.html // // UploadPart 上传成功的返回值，两个成员PartNumber、ETag。PartNumber片(Part)编号，即传入参数partNumber； // ETag及上传数据的MD5。error为nil时有效。 // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) UploadPartCopy(imur InitiateMultipartUploadResult, srcBucketName, srcObjectKey string, startPosition, partSize int64, partNumber int, options ...Option) (UploadPart, error) { var out UploadPartCopyResult var part UploadPart opts := []Option{CopySource(srcBucketName, srcObjectKey), CopySourceRange(startPosition, partSize)} opts = append(opts, options...) params := "partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + imur.UploadID resp, err := bucket.do("PUT", imur.Key, params, params, opts, nil, nil) if err != nil { return part, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) if err != nil { return part, err } part.ETag = out.ETag part.PartNumber = partNumber return part, nil } // // CompleteMultipartUpload 提交分片上传任务。 // // imur InitiateMultipartUpload的返回值。 // parts UploadPart/UploadPartFromFile/UploadPartCopy返回值组成的数组。 // // CompleteMultipartUploadResponse 操作成功后的返回值。error为nil时有效。 // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) CompleteMultipartUpload(imur InitiateMultipartUploadResult, parts []UploadPart) (CompleteMultipartUploadResult, error) { var out CompleteMultipartUploadResult sort.Sort(uploadParts(parts)) cxml := completeMultipartUploadXML{} cxml.Part = parts bs, err := xml.Marshal(cxml) if err != nil { return out, err } buffer := new(bytes.Buffer) buffer.Write(bs) params := "uploadId=" + imur.UploadID resp, err := bucket.do("POST", imur.Key, params, params, nil, buffer, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // AbortMultipartUpload 取消分片上传任务。 // // imur InitiateMultipartUpload的返回值。 // // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) AbortMultipartUpload(imur InitiateMultipartUploadResult) error { params := "uploadId=" + imur.UploadID resp, err := bucket.do("DELETE", imur.Key, params, params, nil, nil, nil) if err != nil { return err } defer resp.Body.Close() return checkRespCode(resp.StatusCode, []int{http.StatusNoContent}) } // // ListUploadedParts 列出指定上传任务已经上传的分片。 // // imur InitiateMultipartUpload的返回值。 // // ListUploadedPartsResponse 操作成功后的返回值，成员UploadedParts已经上传/拷贝的片。error为nil时该返回值有效。 // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) ListUploadedParts(imur InitiateMultipartUploadResult) (ListUploadedPartsResult, error) { var out ListUploadedPartsResult params := "uploadId=" + imur.UploadID resp, err := bucket.do("GET", imur.Key, params, params, nil, nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) return out, err } // // ListMultipartUploads 列出所有未上传完整的multipart任务列表。 // // options ListObject的筛选行为。Prefix返回object的前缀，KeyMarker返回object的起始位置，MaxUploads最大数目默认1000， // Delimiter用于对Object名字进行分组的字符，所有名字包含指定的前缀且第一次出现delimiter字符之间的object。 // // ListMultipartUploadResponse 操作成功后的返回值，error为nil时该返回值有效。 // error 操作成功error为nil，非nil为错误信息。 // func (bucket Bucket) ListMultipartUploads(options ...Option) (ListMultipartUploadResult, error) { var out ListMultipartUploadResult options = append(options, EncodingType("url")) params, err := handleParams(options) if err != nil { return out, err } resp, err := bucket.do("GET", "", "uploads&"+params, "uploads", nil, nil, nil) if err != nil { return out, err } defer resp.Body.Close() err = xmlUnmarshal(resp.Body, &out) if err != nil { return out, err } err = decodeListMultipartUploadResult(&out) return out, err } aliyun-oss-go-sdk-1.5.0/oss/multipart_test.go000066400000000000000000000656461311621456000212350ustar00rootroot00000000000000// multipart test package oss import ( "math/rand" "net/http" "os" "strconv" "time" . "gopkg.in/check.v1" ) type OssBucketMultipartSuite struct { client *Client bucket *Bucket } var _ = Suite(&OssBucketMultipartSuite{}) // Run once when the suite starts running func (s *OssBucketMultipartSuite) SetUpSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) s.client = client s.client.CreateBucket(bucketName) time.Sleep(5 * time.Second) bucket, err := s.client.Bucket(bucketName) c.Assert(err, IsNil) s.bucket = bucket // Delete Part lmur, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmur.Uploads { var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName, Key: upload.Key, UploadID: upload.UploadID} err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } // Delete Objects lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = s.bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } testLogger.Println("test multipart started") } // Run before each test or benchmark starts running func (s *OssBucketMultipartSuite) TearDownSuite(c *C) { // Delete Part lmur, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmur.Uploads { var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName, Key: upload.Key, UploadID: upload.UploadID} err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } // Delete Objects lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = s.bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } testLogger.Println("test multipart completed") } // Run after each test or benchmark runs func (s *OssBucketMultipartSuite) SetUpTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // Run once after all tests or benchmarks have finished running func (s *OssBucketMultipartSuite) TearDownTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".temp") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".txt1") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".txt2") c.Assert(err, IsNil) } // TestMultipartUpload func (s *OssBucketMultipartSuite) TestMultipartUpload(c *C) { objectName := objectNamePrefix + "tmu" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartNum(fileName, 3) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) options := []Option{ Expires(futureDate), Meta("my", "myprop"), } fd, err := os.Open(fileName) c.Assert(err, IsNil) defer fd.Close() imur, err := s.bucket.InitiateMultipartUpload(objectName, options...) c.Assert(err, IsNil) var parts []UploadPart for _, chunk := range chunks { fd.Seek(chunk.Offset, os.SEEK_SET) part, err := s.bucket.UploadPart(imur, fd, chunk.Size, chunk.Number) c.Assert(err, IsNil) parts = append(parts, part) } cmur, err := s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) testLogger.Println("cmur:", cmur) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) c.Assert(meta.Get("X-Oss-Meta-My"), Equals, "myprop") c.Assert(meta.Get("Expires"), Equals, futureDate.Format(http.TimeFormat)) c.Assert(meta.Get("X-Oss-Object-Type"), Equals, "Multipart") err = s.bucket.GetObjectToFile(objectName, "newpic1.jpg") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestMultipartUpload func (s *OssBucketMultipartSuite) TestMultipartUploadFromFile(c *C) { objectName := objectNamePrefix + "tmuff" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartNum(fileName, 3) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) options := []Option{ Expires(futureDate), Meta("my", "myprop"), } imur, err := s.bucket.InitiateMultipartUpload(objectName, options...) c.Assert(err, IsNil) var parts []UploadPart for _, chunk := range chunks { part, err := s.bucket.UploadPartFromFile(imur, fileName, chunk.Offset, chunk.Size, chunk.Number) c.Assert(err, IsNil) parts = append(parts, part) } cmur, err := s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) testLogger.Println("cmur:", cmur) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) c.Assert(meta.Get("X-Oss-Meta-My"), Equals, "myprop") c.Assert(meta.Get("Expires"), Equals, futureDate.Format(http.TimeFormat)) c.Assert(meta.Get("X-Oss-Object-Type"), Equals, "Multipart") err = s.bucket.GetObjectToFile(objectName, "newpic1.jpg") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestUploadPartCopy func (s *OssBucketMultipartSuite) TestUploadPartCopy(c *C) { objectSrc := objectNamePrefix + "tupc" + "src" objectDesc := objectNamePrefix + "tupc" + "desc" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartNum(fileName, 3) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) err = s.bucket.PutObjectFromFile(objectSrc, fileName) c.Assert(err, IsNil) options := []Option{ Expires(futureDate), Meta("my", "myprop"), } imur, err := s.bucket.InitiateMultipartUpload(objectDesc, options...) c.Assert(err, IsNil) var parts []UploadPart for _, chunk := range chunks { part, err := s.bucket.UploadPartCopy(imur, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) parts = append(parts, part) } cmur, err := s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) testLogger.Println("cmur:", cmur) meta, err := s.bucket.GetObjectDetailedMeta(objectDesc) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) c.Assert(meta.Get("X-Oss-Meta-My"), Equals, "myprop") c.Assert(meta.Get("Expires"), Equals, futureDate.Format(http.TimeFormat)) c.Assert(meta.Get("X-Oss-Object-Type"), Equals, "Multipart") err = s.bucket.GetObjectToFile(objectDesc, "newpic2.jpg") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectSrc) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectDesc) c.Assert(err, IsNil) } func (s *OssBucketMultipartSuite) TestListUploadedParts(c *C) { objectName := objectNamePrefix + "tlup" objectSrc := objectNamePrefix + "tlup" + "src" objectDesc := objectNamePrefix + "tlup" + "desc" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartSize(fileName, 100*1024) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) err = s.bucket.PutObjectFromFile(objectSrc, fileName) c.Assert(err, IsNil) // upload imurUpload, err := s.bucket.InitiateMultipartUpload(objectName) var partsUpload []UploadPart for _, chunk := range chunks { part, err := s.bucket.UploadPartFromFile(imurUpload, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) partsUpload = append(partsUpload, part) } // copy imurCopy, err := s.bucket.InitiateMultipartUpload(objectDesc) var partsCopy []UploadPart for _, chunk := range chunks { part, err := s.bucket.UploadPartCopy(imurCopy, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) partsCopy = append(partsCopy, part) } // list lupr, err := s.bucket.ListUploadedParts(imurUpload) c.Assert(err, IsNil) testLogger.Println("lupr:", lupr) c.Assert(len(lupr.UploadedParts), Equals, len(chunks)) lupr, err = s.bucket.ListUploadedParts(imurCopy) c.Assert(err, IsNil) testLogger.Println("lupr:", lupr) c.Assert(len(lupr.UploadedParts), Equals, len(chunks)) lmur, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) testLogger.Println("lmur:", lmur) // complete _, err = s.bucket.CompleteMultipartUpload(imurUpload, partsUpload) c.Assert(err, IsNil) _, err = s.bucket.CompleteMultipartUpload(imurCopy, partsCopy) c.Assert(err, IsNil) // download err = s.bucket.GetObjectToFile(objectDesc, "newpic3.jpg") c.Assert(err, IsNil) err = s.bucket.GetObjectToFile(objectName, "newpic4.jpg") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectDesc) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectSrc) c.Assert(err, IsNil) } func (s *OssBucketMultipartSuite) TestAbortMultipartUpload(c *C) { objectName := objectNamePrefix + "tamu" objectSrc := objectNamePrefix + "tamu" + "src" objectDesc := objectNamePrefix + "tamu" + "desc" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartSize(fileName, 100*1024) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) err = s.bucket.PutObjectFromFile(objectSrc, fileName) c.Assert(err, IsNil) // upload imurUpload, err := s.bucket.InitiateMultipartUpload(objectName) var partsUpload []UploadPart for _, chunk := range chunks { part, err := s.bucket.UploadPartFromFile(imurUpload, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) partsUpload = append(partsUpload, part) } // copy imurCopy, err := s.bucket.InitiateMultipartUpload(objectDesc) var partsCopy []UploadPart for _, chunk := range chunks { part, err := s.bucket.UploadPartCopy(imurCopy, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) partsCopy = append(partsCopy, part) } // list lupr, err := s.bucket.ListUploadedParts(imurUpload) c.Assert(err, IsNil) testLogger.Println("lupr:", lupr) c.Assert(len(lupr.UploadedParts), Equals, len(chunks)) lupr, err = s.bucket.ListUploadedParts(imurCopy) c.Assert(err, IsNil) testLogger.Println("lupr:", lupr) c.Assert(len(lupr.UploadedParts), Equals, len(chunks)) lmur, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) testLogger.Println("lmur:", lmur) c.Assert(len(lmur.Uploads), Equals, 2) // abort err = s.bucket.AbortMultipartUpload(imurUpload) c.Assert(err, IsNil) err = s.bucket.AbortMultipartUpload(imurCopy) c.Assert(err, IsNil) lmur, err = s.bucket.ListMultipartUploads() c.Assert(err, IsNil) testLogger.Println("lmur:", lmur) c.Assert(len(lmur.Uploads), Equals, 0) // download err = s.bucket.GetObjectToFile(objectDesc, "newpic3.jpg") c.Assert(err, NotNil) err = s.bucket.GetObjectToFile(objectName, "newpic4.jpg") c.Assert(err, NotNil) } // TestUploadPartCopyWithConstraints func (s *OssBucketMultipartSuite) TestUploadPartCopyWithConstraints(c *C) { objectSrc := objectNamePrefix + "tucwc" + "src" objectDesc := objectNamePrefix + "tucwc" + "desc" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartNum(fileName, 3) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) err = s.bucket.PutObjectFromFile(objectSrc, fileName) c.Assert(err, IsNil) imur, err := s.bucket.InitiateMultipartUpload(objectDesc) var parts []UploadPart for _, chunk := range chunks { _, err = s.bucket.UploadPartCopy(imur, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number), CopySourceIfModifiedSince(futureDate)) c.Assert(err, NotNil) } for _, chunk := range chunks { _, err = s.bucket.UploadPartCopy(imur, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number), CopySourceIfUnmodifiedSince(futureDate)) c.Assert(err, IsNil) } meta, err := s.bucket.GetObjectDetailedMeta(objectSrc) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) for _, chunk := range chunks { _, err = s.bucket.UploadPartCopy(imur, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number), CopySourceIfNoneMatch(meta.Get("Etag"))) c.Assert(err, NotNil) } for _, chunk := range chunks { part, err := s.bucket.UploadPartCopy(imur, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number), CopySourceIfMatch(meta.Get("Etag"))) c.Assert(err, IsNil) parts = append(parts, part) } cmur, err := s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) testLogger.Println("cmur:", cmur) err = s.bucket.GetObjectToFile(objectDesc, "newpic5.jpg") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectSrc) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectDesc) c.Assert(err, IsNil) } // TestMultipartUploadFromFileOutofOrder func (s *OssBucketMultipartSuite) TestMultipartUploadFromFileOutofOrder(c *C) { objectName := objectNamePrefix + "tmuffoo" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartSize(fileName, 1024*100) shuffleArray(chunks) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) imur, err := s.bucket.InitiateMultipartUpload(objectName) var parts []UploadPart for _, chunk := range chunks { _, err := s.bucket.UploadPartFromFile(imur, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) } // double upload for _, chunk := range chunks { part, err := s.bucket.UploadPartFromFile(imur, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) parts = append(parts, part) } cmur, err := s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) testLogger.Println("cmur:", cmur) err = s.bucket.GetObjectToFile(objectName, "newpic6.jpg") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestUploadPartCopyOutofOrder func (s *OssBucketMultipartSuite) TestUploadPartCopyOutofOrder(c *C) { objectSrc := objectNamePrefix + "tupcoo" + "src" objectDesc := objectNamePrefix + "tupcoo" + "desc" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartSize(fileName, 1024*100) shuffleArray(chunks) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) err = s.bucket.PutObjectFromFile(objectSrc, fileName) c.Assert(err, IsNil) imur, err := s.bucket.InitiateMultipartUpload(objectDesc) var parts []UploadPart for _, chunk := range chunks { _, err := s.bucket.UploadPartCopy(imur, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) } //double copy for _, chunk := range chunks { part, err := s.bucket.UploadPartCopy(imur, bucketName, objectSrc, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) parts = append(parts, part) } cmur, err := s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) testLogger.Println("cmur:", cmur) err = s.bucket.GetObjectToFile(objectDesc, "newpic7.jpg") c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectSrc) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectDesc) c.Assert(err, IsNil) } // TestMultipartUpload func (s *OssBucketMultipartSuite) TestMultipartUploadFromFileType(c *C) { objectName := objectNamePrefix + "tmuffwm" + ".jpg" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartNum(fileName, 4) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) imur, err := s.bucket.InitiateMultipartUpload(objectName) var parts []UploadPart for _, chunk := range chunks { part, err := s.bucket.UploadPartFromFile(imur, fileName, chunk.Offset, chunk.Size, chunk.Number) c.Assert(err, IsNil) parts = append(parts, part) } testLogger.Println("parts:", parts) cmur, err := s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) testLogger.Println("cmur:", cmur) err = s.bucket.GetObjectToFile(objectName, "newpic8.jpg") c.Assert(err, IsNil) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) c.Assert(meta.Get("Content-Type"), Equals, "image/jpeg") err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } func (s *OssBucketMultipartSuite) TestListMultipartUploads(c *C) { objectName := objectNamePrefix + "tlmu" imurs := []InitiateMultipartUploadResult{} for i := 0; i < 20; i++ { imur, err := s.bucket.InitiateMultipartUpload(objectName + strconv.Itoa(i)) c.Assert(err, IsNil) imurs = append(imurs, imur) } lmpu, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 20) lmpu, err = s.bucket.ListMultipartUploads(MaxUploads(3)) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 3) lmpu, err = s.bucket.ListMultipartUploads(Prefix(objectName)) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 20) lmpu, err = s.bucket.ListMultipartUploads(Prefix(objectName + "1")) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 11) lmpu, err = s.bucket.ListMultipartUploads(Prefix(objectName + "22")) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 0) lmpu, err = s.bucket.ListMultipartUploads(KeyMarker(objectName + "10")) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 17) lmpu, err = s.bucket.ListMultipartUploads(KeyMarker(objectName+"10"), MaxUploads(3)) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 3) lmpu, err = s.bucket.ListMultipartUploads(Prefix(objectName), Delimiter("4")) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 18) c.Assert(len(lmpu.CommonPrefixes), Equals, 2) // upload-id-marker lmpu, err = s.bucket.ListMultipartUploads(KeyMarker(objectName+"12"), UploadIDMarker("EEE")) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 15) //testLogger.Println("UploadIDMarker", lmpu.Uploads) for _, imur := range imurs { err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } } func (s *OssBucketMultipartSuite) TestListMultipartUploadsEncodingKey(c *C) { objectName := objectNamePrefix + "让你任性让你狂" + "tlmuek" imurs := []InitiateMultipartUploadResult{} for i := 0; i < 3; i++ { imur, err := s.bucket.InitiateMultipartUpload(objectName + strconv.Itoa(i)) c.Assert(err, IsNil) imurs = append(imurs, imur) } lmpu, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 3) lmpu, err = s.bucket.ListMultipartUploads(Prefix(objectNamePrefix + "让你任性让你狂tlmuek1")) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 1) lmpu, err = s.bucket.ListMultipartUploads(KeyMarker(objectNamePrefix + "让你任性让你狂tlmuek1")) c.Assert(err, IsNil) c.Assert(len(lmpu.Uploads), Equals, 1) lmpu, err = s.bucket.ListMultipartUploads(EncodingType("url")) c.Assert(err, IsNil) for i, upload := range lmpu.Uploads { c.Assert(upload.Key, Equals, objectNamePrefix+"让你任性让你狂tlmuek"+strconv.Itoa(i)) } for _, imur := range imurs { err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } } func (s *OssBucketMultipartSuite) TestMultipartNegative(c *C) { objectName := objectNamePrefix + "tmn" // key tool long data := make([]byte, 100*1024) imur, err := s.bucket.InitiateMultipartUpload(string(data)) c.Assert(err, NotNil) // imur invalid fileName := "../sample/BingWallpaper-2015-11-07.jpg" fd, err := os.Open(fileName) c.Assert(err, IsNil) defer fd.Close() _, err = s.bucket.UploadPart(imur, fd, 1024, 1) c.Assert(err, NotNil) _, err = s.bucket.UploadPartFromFile(imur, fileName, 0, 1024, 1) c.Assert(err, NotNil) _, err = s.bucket.UploadPartCopy(imur, bucketName, fileName, 0, 1024, 1) c.Assert(err, NotNil) err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, NotNil) _, err = s.bucket.ListUploadedParts(imur) c.Assert(err, NotNil) // invalid exist imur, err = s.bucket.InitiateMultipartUpload(objectName) c.Assert(err, IsNil) _, err = s.bucket.UploadPart(imur, fd, 1024, 1) c.Assert(err, IsNil) _, err = s.bucket.UploadPart(imur, fd, 102400, 10001) c.Assert(err, NotNil) // _, err = s.bucket.UploadPartFromFile(imur, fileName, 0, 1024, 1) // c.Assert(err, IsNil) _, err = s.bucket.UploadPartFromFile(imur, fileName, 0, 102400, 10001) c.Assert(err, NotNil) _, err = s.bucket.UploadPartCopy(imur, bucketName, fileName, 0, 1024, 1) c.Assert(err, NotNil) _, err = s.bucket.UploadPartCopy(imur, bucketName, fileName, 0, 1024, 1000) c.Assert(err, NotNil) err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) // option invalid _, err = s.bucket.InitiateMultipartUpload(objectName, IfModifiedSince(futureDate)) c.Assert(err, IsNil) } func (s *OssBucketMultipartSuite) TestMultipartUploadFromFileBigFile(c *C) { objectName := objectNamePrefix + "tmuffbf" bigFile := "D:\\tmp\\bigfile.zip" newFile := "D:\\tmp\\newbigfile.zip" exist, err := isFileExist(bigFile) c.Assert(err, IsNil) if !exist { return } chunks, err := SplitFileByPartNum(bigFile, 64) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) imur, err := s.bucket.InitiateMultipartUpload(objectName) var parts []UploadPart start := GetNowSec() for _, chunk := range chunks { part, err := s.bucket.UploadPartFromFile(imur, bigFile, chunk.Offset, chunk.Size, (int)(chunk.Number)) c.Assert(err, IsNil) parts = append(parts, part) } end := GetNowSec() testLogger.Println("Uplaod big file:", bigFile, "use sec:", end-start) testLogger.Println("parts:", parts) _, err = s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) start = GetNowSec() err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) end = GetNowSec() testLogger.Println("Download big file:", bigFile, "use sec:", end-start) start = GetNowSec() eq, err := compareFiles(bigFile, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) end = GetNowSec() testLogger.Println("Compare big file:", bigFile, "use sec:", end-start) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestUploadFile func (s *OssBucketMultipartSuite) TestUploadFile(c *C) { objectName := objectNamePrefix + "tuff" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" newFile := "newfiletuff.jpg" // 有余数 err := s.bucket.UploadFile(objectName, fileName, 100*1024) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 整除 err = s.bucket.UploadFile(objectName, fileName, 482048/4) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 等于文件大小 err = s.bucket.UploadFile(objectName, fileName, 482048) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 大于文件大小 err = s.bucket.UploadFile(objectName, fileName, 482049) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // option options := []Option{ Expires(futureDate), ObjectACL(ACLPublicRead), Meta("myprop", "mypropval")} err = s.bucket.UploadFile(objectName, fileName, 482049, options...) c.Assert(err, IsNil) // Check os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) acl, err := s.bucket.GetObjectACL(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectAcl:", acl) c.Assert(acl.ACL, Equals, "default") meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") } func (s *OssBucketMultipartSuite) TestUploadFileNegative(c *C) { objectName := objectNamePrefix + "tufn" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" // 小于最小文件片 err := s.bucket.UploadFile(objectName, fileName, 100*1024-1) c.Assert(err, NotNil) // 大于最大文件片 err = s.bucket.UploadFile(objectName, fileName, 1024*1024*1024*5+1) c.Assert(err, NotNil) // 文件不存在 err = s.bucket.UploadFile(objectName, "/root/123abc9874", 1024*1024*1024) c.Assert(err, NotNil) // Key无效 err = s.bucket.UploadFile("", fileName, 100*1024) c.Assert(err, NotNil) } // TestDownloadFile func (s *OssBucketMultipartSuite) TestDownloadFile(c *C) { objectName := objectNamePrefix + "tdff" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" newFile := "newfiletdff.jpg" err := s.bucket.UploadFile(objectName, fileName, 100*1024) c.Assert(err, IsNil) // 有余数 err = s.bucket.DownloadFile(objectName, newFile, 100*1024) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 整除 err = s.bucket.DownloadFile(objectName, newFile, 482048/4) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 等于文件大小 err = s.bucket.DownloadFile(objectName, newFile, 482048) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // 大于文件大小 err = s.bucket.DownloadFile(objectName, newFile, 482049) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // option meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) testLogger.Println("GetObjectDetailedMeta:", meta) // If-Match err = s.bucket.DownloadFile(objectName, newFile, 482048/4, IfMatch(meta.Get("Etag"))) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) // If-None-Match err = s.bucket.DownloadFile(objectName, newFile, 482048, IfNoneMatch(meta.Get("Etag"))) c.Assert(err, NotNil) os.Remove(newFile) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } func (s *OssBucketMultipartSuite) TestDownloadFileNegative(c *C) { objectName := objectNamePrefix + "tufn" newFile := "newfiletudff.jpg" // 小于最小文件片 err := s.bucket.DownloadFile(objectName, newFile, 100*1024-1) c.Assert(err, NotNil) // 大于最大文件片 err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024+1) c.Assert(err, NotNil) // 文件不存在 err = s.bucket.DownloadFile(objectName, "/OSS/TEMP/ZIBI/QUQU/BALA", 1024*1024*1024+1) c.Assert(err, NotNil) // Key不存在 err = s.bucket.DownloadFile(objectName, newFile, 100*1024) c.Assert(err, NotNil) } // private func shuffleArray(chunks []FileChunk) []FileChunk { for i := range chunks { j := rand.Intn(i + 1) chunks[i], chunks[j] = chunks[j], chunks[i] } return chunks } aliyun-oss-go-sdk-1.5.0/oss/option.go000066400000000000000000000232321311621456000174460ustar00rootroot00000000000000package oss import ( "bytes" "fmt" "net/http" "net/url" "sort" "strconv" "strings" "time" ) type optionType string const ( optionParam optionType = "HTTPParameter" // URL参数 optionHTTP optionType = "HTTPHeader" // HTTP头 optionArg optionType = "FuncArgument" // 函数参数 ) const ( deleteObjectsQuiet = "delete-objects-quiet" routineNum = "x-routine-num" checkpointConfig = "x-cp-config" initCRC64 = "init-crc64" progressListener = "x-progress-listener" storageClass = "storage-class" ) type ( optionValue struct { Value interface{} Type optionType } // Option http option Option func(map[string]optionValue) error ) // ACL is an option to set X-Oss-Acl header func ACL(acl ACLType) Option { return setHeader(HTTPHeaderOssACL, string(acl)) } // ContentType is an option to set Content-Type header func ContentType(value string) Option { return setHeader(HTTPHeaderContentType, value) } // ContentLength is an option to set Content-Length header func ContentLength(length int64) Option { return setHeader(HTTPHeaderContentLength, strconv.FormatInt(length, 10)) } // CacheControl is an option to set Cache-Control header func CacheControl(value string) Option { return setHeader(HTTPHeaderCacheControl, value) } // ContentDisposition is an option to set Content-Disposition header func ContentDisposition(value string) Option { return setHeader(HTTPHeaderContentDisposition, value) } // ContentEncoding is an option to set Content-Encoding header func ContentEncoding(value string) Option { return setHeader(HTTPHeaderContentEncoding, value) } // ContentMD5 is an option to set Content-MD5 header func ContentMD5(value string) Option { return setHeader(HTTPHeaderContentMD5, value) } // Expires is an option to set Expires header func Expires(t time.Time) Option { return setHeader(HTTPHeaderExpires, t.Format(http.TimeFormat)) } // Meta is an option to set Meta header func Meta(key, value string) Option { return setHeader(HTTPHeaderOssMetaPrefix+key, value) } // Range is an option to set Range header, [start, end] func Range(start, end int64) Option { return setHeader(HTTPHeaderRange, fmt.Sprintf("bytes=%d-%d", start, end)) } // NormalizedRange is an option to set Range header, such as 1024-2048 or 1024- or -2048 func NormalizedRange(nr string) Option { return setHeader(HTTPHeaderRange, fmt.Sprintf("bytes=%s", strings.TrimSpace(nr))) } // AcceptEncoding is an option to set Accept-Encoding header func AcceptEncoding(value string) Option { return setHeader(HTTPHeaderAcceptEncoding, value) } // IfModifiedSince is an option to set If-Modified-Since header func IfModifiedSince(t time.Time) Option { return setHeader(HTTPHeaderIfModifiedSince, t.Format(http.TimeFormat)) } // IfUnmodifiedSince is an option to set If-Unmodified-Since header func IfUnmodifiedSince(t time.Time) Option { return setHeader(HTTPHeaderIfUnmodifiedSince, t.Format(http.TimeFormat)) } // IfMatch is an option to set If-Match header func IfMatch(value string) Option { return setHeader(HTTPHeaderIfMatch, value) } // IfNoneMatch is an option to set IfNoneMatch header func IfNoneMatch(value string) Option { return setHeader(HTTPHeaderIfNoneMatch, value) } // CopySource is an option to set X-Oss-Copy-Source header func CopySource(sourceBucket, sourceObject string) Option { return setHeader(HTTPHeaderOssCopySource, "/"+sourceBucket+"/"+sourceObject) } // CopySourceRange is an option to set X-Oss-Copy-Source header func CopySourceRange(startPosition, partSize int64) Option { val := "bytes=" + strconv.FormatInt(startPosition, 10) + "-" + strconv.FormatInt((startPosition+partSize-1), 10) return setHeader(HTTPHeaderOssCopySourceRange, val) } // CopySourceIfMatch is an option to set X-Oss-Copy-Source-If-Match header func CopySourceIfMatch(value string) Option { return setHeader(HTTPHeaderOssCopySourceIfMatch, value) } // CopySourceIfNoneMatch is an option to set X-Oss-Copy-Source-If-None-Match header func CopySourceIfNoneMatch(value string) Option { return setHeader(HTTPHeaderOssCopySourceIfNoneMatch, value) } // CopySourceIfModifiedSince is an option to set X-Oss-CopySource-If-Modified-Since header func CopySourceIfModifiedSince(t time.Time) Option { return setHeader(HTTPHeaderOssCopySourceIfModifiedSince, t.Format(http.TimeFormat)) } // CopySourceIfUnmodifiedSince is an option to set X-Oss-Copy-Source-If-Unmodified-Since header func CopySourceIfUnmodifiedSince(t time.Time) Option { return setHeader(HTTPHeaderOssCopySourceIfUnmodifiedSince, t.Format(http.TimeFormat)) } // MetadataDirective is an option to set X-Oss-Metadata-Directive header func MetadataDirective(directive MetadataDirectiveType) Option { return setHeader(HTTPHeaderOssMetadataDirective, string(directive)) } // ServerSideEncryption is an option to set X-Oss-Server-Side-Encryption header func ServerSideEncryption(value string) Option { return setHeader(HTTPHeaderOssServerSideEncryption, value) } // ObjectACL is an option to set X-Oss-Object-Acl header func ObjectACL(acl ACLType) Option { return setHeader(HTTPHeaderOssObjectACL, string(acl)) } // symlinkTarget is an option to set X-Oss-Symlink-Target func symlinkTarget(targetObjectKey string) Option { return setHeader(HTTPHeaderOssSymlinkTarget, targetObjectKey) } // Origin is an option to set Origin header func Origin(value string) Option { return setHeader(HTTPHeaderOrigin, value) } // Delimiter is an option to set delimiler parameter func Delimiter(value string) Option { return addParam("delimiter", value) } // Marker is an option to set marker parameter func Marker(value string) Option { return addParam("marker", value) } // MaxKeys is an option to set maxkeys parameter func MaxKeys(value int) Option { return addParam("max-keys", strconv.Itoa(value)) } // Prefix is an option to set prefix parameter func Prefix(value string) Option { return addParam("prefix", value) } // EncodingType is an option to set encoding-type parameter func EncodingType(value string) Option { return addParam("encoding-type", value) } // MaxUploads is an option to set max-uploads parameter func MaxUploads(value int) Option { return addParam("max-uploads", strconv.Itoa(value)) } // KeyMarker is an option to set key-marker parameter func KeyMarker(value string) Option { return addParam("key-marker", value) } // UploadIDMarker is an option to set upload-id-marker parameter func UploadIDMarker(value string) Option { return addParam("upload-id-marker", value) } // DeleteObjectsQuiet DeleteObjects详细(verbose)模式或简单(quiet)模式，默认详细模式。 func DeleteObjectsQuiet(isQuiet bool) Option { return addArg(deleteObjectsQuiet, isQuiet) } // StorageClass bucket的存储方式 func StorageClass(value StorageClassType) Option { return addArg(storageClass, value) } // 断点续传配置，包括是否启用、cp文件 type cpConfig struct { IsEnable bool FilePath string } // Checkpoint DownloadFile/UploadFile是否开启checkpoint及checkpoint文件路径 func Checkpoint(isEnable bool, filePath string) Option { return addArg(checkpointConfig, &cpConfig{isEnable, filePath}) } // Routines DownloadFile/UploadFile并发数 func Routines(n int) Option { return addArg(routineNum, n) } // InitCRC AppendObject CRC的校验的初始值 func InitCRC(initCRC uint64) Option { return addArg(initCRC64, initCRC) } // Progress set progress listener func Progress(listener ProgressListener) Option { return addArg(progressListener, listener) } func setHeader(key string, value interface{}) Option { return func(params map[string]optionValue) error { if value == nil { return nil } params[key] = optionValue{value, optionHTTP} return nil } } func addParam(key string, value interface{}) Option { return func(params map[string]optionValue) error { if value == nil { return nil } params[key] = optionValue{value, optionParam} return nil } } func addArg(key string, value interface{}) Option { return func(params map[string]optionValue) error { if value == nil { return nil } params[key] = optionValue{value, optionArg} return nil } } func handleOptions(headers map[string]string, options []Option) error { params := map[string]optionValue{} for _, option := range options { if option != nil { if err := option(params); err != nil { return err } } } for k, v := range params { if v.Type == optionHTTP { headers[k] = v.Value.(string) } } return nil } func handleParams(options []Option) (string, error) { // option params := map[string]optionValue{} for _, option := range options { if option != nil { if err := option(params); err != nil { return "", err } } } // sort var buf bytes.Buffer keys := make([]string, 0, len(params)) for k, v := range params { if v.Type == optionParam { keys = append(keys, k) } } sort.Strings(keys) // serialize for _, k := range keys { vs := params[k] prefix := url.QueryEscape(k) + "=" if buf.Len() > 0 { buf.WriteByte('&') } buf.WriteString(prefix) buf.WriteString(url.QueryEscape(vs.Value.(string))) } return buf.String(), nil } func findOption(options []Option, param string, defaultVal interface{}) (interface{}, error) { params := map[string]optionValue{} for _, option := range options { if option != nil { if err := option(params); err != nil { return nil, err } } } if val, ok := params[param]; ok { return val.Value, nil } return defaultVal, nil } func isOptionSet(options []Option, option string) (bool, interface{}, error) { params := map[string]optionValue{} for _, option := range options { if option != nil { if err := option(params); err != nil { return false, nil, err } } } if val, ok := params[option]; ok { return true, val.Value, nil } return false, nil, nil } aliyun-oss-go-sdk-1.5.0/oss/option_test.go000066400000000000000000000120311311621456000205000ustar00rootroot00000000000000package oss import ( "net/http" . "gopkg.in/check.v1" ) type OssOptionSuite struct{} var _ = Suite(&OssOptionSuite{}) type optionTestCase struct { option Option key string value string } var headerTestcases = []optionTestCase{ { option: Meta("User", "baymax"), key: "X-Oss-Meta-User", value: "baymax", }, { option: ACL(ACLPrivate), key: "X-Oss-Acl", value: "private", }, { option: ContentType("plain/text"), key: "Content-Type", value: "plain/text", }, { option: CacheControl("no-cache"), key: "Cache-Control", value: "no-cache", }, { option: ContentDisposition("Attachment; filename=example.txt"), key: "Content-Disposition", value: "Attachment; filename=example.txt", }, { option: ContentEncoding("gzip"), key: "Content-Encoding", value: "gzip", }, { option: Expires(pastDate), key: "Expires", value: pastDate.Format(http.TimeFormat), }, { option: Range(0, 9), key: "Range", value: "bytes=0-9", }, { option: Origin("localhost"), key: "Origin", value: "localhost", }, { option: CopySourceRange(0, 9), key: "X-Oss-Copy-Source-Range", value: "bytes=0-8", }, { option: IfModifiedSince(pastDate), key: "If-Modified-Since", value: pastDate.Format(http.TimeFormat), }, { option: IfUnmodifiedSince(futureDate), key: "If-Unmodified-Since", value: futureDate.Format(http.TimeFormat), }, { option: IfMatch("xyzzy"), key: "If-Match", value: "xyzzy", }, { option: IfNoneMatch("xyzzy"), key: "If-None-Match", value: "xyzzy", }, { option: CopySource("bucket_name", "object_name"), key: "X-Oss-Copy-Source", value: "/bucket_name/object_name", }, { option: CopySourceIfModifiedSince(pastDate), key: "X-Oss-Copy-Source-If-Modified-Since", value: pastDate.Format(http.TimeFormat), }, { option: CopySourceIfUnmodifiedSince(futureDate), key: "X-Oss-Copy-Source-If-Unmodified-Since", value: futureDate.Format(http.TimeFormat), }, { option: CopySourceIfMatch("xyzzy"), key: "X-Oss-Copy-Source-If-Match", value: "xyzzy", }, { option: CopySourceIfNoneMatch("xyzzy"), key: "X-Oss-Copy-Source-If-None-Match", value: "xyzzy", }, { option: MetadataDirective(MetaCopy), key: "X-Oss-Metadata-Directive", value: "COPY", }, { option: ServerSideEncryption("AES256"), key: "X-Oss-Server-Side-Encryption", value: "AES256", }, { option: ObjectACL(ACLPrivate), key: "X-Oss-Object-Acl", value: "private", }, } func (s *OssOptionSuite) TestHeaderOptions(c *C) { for _, testcase := range headerTestcases { headers := make(map[string]optionValue) err := testcase.option(headers) c.Assert(err, IsNil) expected, actual := testcase.value, headers[testcase.key].Value c.Assert(expected, Equals, actual) } } var paramTestCases = []optionTestCase{ { option: Delimiter("/"), key: "delimiter", value: "/", }, { option: Marker("abc"), key: "marker", value: "abc", }, { option: MaxKeys(150), key: "max-keys", value: "150", }, { option: Prefix("fun"), key: "prefix", value: "fun", }, { option: EncodingType("ascii"), key: "encoding-type", value: "ascii", }, { option: MaxUploads(100), key: "max-uploads", value: "100", }, { option: KeyMarker("abc"), key: "key-marker", value: "abc", }, { option: UploadIDMarker("xyz"), key: "upload-id-marker", value: "xyz", }, } func (s *OssOptionSuite) TestParamOptions(c *C) { for _, testcase := range paramTestCases { params := make(map[string]optionValue) err := testcase.option(params) c.Assert(err, IsNil) expected, actual := testcase.value, params[testcase.key].Value c.Assert(expected, Equals, actual) } } func (s *OssOptionSuite) TestHandleOptions(c *C) { headers := make(map[string]string) options := []Option{} for _, testcase := range headerTestcases { options = append(options, testcase.option) } err := handleOptions(headers, options) c.Assert(err, IsNil) for _, testcase := range headerTestcases { expected, actual := testcase.value, headers[testcase.key] c.Assert(expected, Equals, actual) } options = []Option{IfMatch(""), nil} headers = map[string]string{} err = handleOptions(headers, options) c.Assert(err, IsNil) c.Assert(len(headers), Equals, 1) } func (s *OssOptionSuite) TestHandleParams(c *C) { options := []Option{} for _, testcase := range paramTestCases { options = append(options, testcase.option) } out, err := handleParams(options) c.Assert(err, IsNil) c.Assert(len(out), Equals, 120) options = []Option{KeyMarker(""), nil} out, err = handleParams(options) c.Assert(out, Equals, "key-marker=") c.Assert(err, IsNil) } func (s *OssOptionSuite) TestFindOption(c *C) { options := []Option{} for _, testcase := range headerTestcases { options = append(options, testcase.option) } str, err := findOption(options, "X-Oss-Acl", "") c.Assert(err, IsNil) c.Assert(str, Equals, "private") str, err = findOption(options, "MyProp", "") c.Assert(err, IsNil) c.Assert(str, Equals, "") } aliyun-oss-go-sdk-1.5.0/oss/progress.go000066400000000000000000000051561311621456000200070ustar00rootroot00000000000000package oss import "io" // ProgressEventType transfer progress event type type ProgressEventType int const ( // TransferStartedEvent transfer started, set TotalBytes TransferStartedEvent ProgressEventType = 1 + iota // TransferDataEvent transfer data, set ConsumedBytes anmd TotalBytes TransferDataEvent // TransferCompletedEvent transfer completed TransferCompletedEvent // TransferFailedEvent transfer encounters an error TransferFailedEvent ) // ProgressEvent progress event type ProgressEvent struct { ConsumedBytes int64 TotalBytes int64 EventType ProgressEventType } // ProgressListener listen progress change type ProgressListener interface { ProgressChanged(event *ProgressEvent) } // -------------------- private -------------------- func newProgressEvent(eventType ProgressEventType, consumed, total int64) *ProgressEvent { return &ProgressEvent{ ConsumedBytes: consumed, TotalBytes: total, EventType: eventType} } // publishProgress func publishProgress(listener ProgressListener, event *ProgressEvent) { if listener != nil && event != nil { listener.ProgressChanged(event) } } type readerTracker struct { completedBytes int64 } type teeReader struct { reader io.Reader writer io.Writer listener ProgressListener consumedBytes int64 totalBytes int64 tracker *readerTracker } // TeeReader returns a Reader that writes to w what it reads from r. // All reads from r performed through it are matched with // corresponding writes to w. There is no internal buffering - // the write must complete before the read completes. // Any error encountered while writing is reported as a read error. func TeeReader(reader io.Reader, writer io.Writer, totalBytes int64, listener ProgressListener, tracker *readerTracker) io.Reader { return &teeReader{ reader: reader, writer: writer, listener: listener, consumedBytes: 0, totalBytes: totalBytes, tracker: tracker, } } func (t *teeReader) Read(p []byte) (n int, err error) { n, err = t.reader.Read(p) // read encountered error if err != nil && err != io.EOF { event := newProgressEvent(TransferFailedEvent, t.consumedBytes, t.totalBytes) publishProgress(t.listener, event) } if n > 0 { t.consumedBytes += int64(n) // crc if t.writer != nil { if n, err := t.writer.Write(p[:n]); err != nil { return n, err } } // progress if t.listener != nil { event := newProgressEvent(TransferDataEvent, t.consumedBytes, t.totalBytes) publishProgress(t.listener, event) } // track if t.tracker != nil { t.tracker.completedBytes = t.consumedBytes } } return } aliyun-oss-go-sdk-1.5.0/oss/progress_test.go000066400000000000000000000255551311621456000210530ustar00rootroot00000000000000// bucket test package oss import ( "bytes" "io/ioutil" "math/rand" "os" "strings" "time" . "gopkg.in/check.v1" ) type OssProgressSuite struct { client *Client bucket *Bucket } var _ = Suite(&OssProgressSuite{}) // Run once when the suite starts running func (s *OssProgressSuite) SetUpSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) s.client = client s.client.CreateBucket(bucketName) time.Sleep(5 * time.Second) bucket, err := s.client.Bucket(bucketName) c.Assert(err, IsNil) s.bucket = bucket testLogger.Println("test progress started") } // Run before each test or benchmark starts running func (s *OssProgressSuite) TearDownSuite(c *C) { // Delete Multipart lmu, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmu.Uploads { imur := InitiateMultipartUploadResult{Bucket: bucketName, Key: upload.Key, UploadID: upload.UploadID} err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } // Delete Objects lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = s.bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } testLogger.Println("test progress completed") } // Run after each test or benchmark runs func (s *OssProgressSuite) SetUpTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".txt") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".html") c.Assert(err, IsNil) } // Run once after all tests or benchmarks have finished running func (s *OssProgressSuite) TearDownTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".txt") c.Assert(err, IsNil) err = removeTempFiles("../oss", ".html") c.Assert(err, IsNil) } // OssProgressListener progress listener type OssProgressListener struct { } // ProgressChanged handle progress event func (listener *OssProgressListener) ProgressChanged(event *ProgressEvent) { switch event.EventType { case TransferStartedEvent: testLogger.Printf("Transfer Started, ConsumedBytes: %d, TotalBytes %d.\n", event.ConsumedBytes, event.TotalBytes) case TransferDataEvent: testLogger.Printf("Transfer Data, ConsumedBytes: %d, TotalBytes %d, %d%%.\n", event.ConsumedBytes, event.TotalBytes, event.ConsumedBytes*100/event.TotalBytes) case TransferCompletedEvent: testLogger.Printf("Transfer Completed, ConsumedBytes: %d, TotalBytes %d.\n", event.ConsumedBytes, event.TotalBytes) case TransferFailedEvent: testLogger.Printf("Transfer Failed, ConsumedBytes: %d, TotalBytes %d.\n", event.ConsumedBytes, event.TotalBytes) default: } } // TestPutObject func (s *OssProgressSuite) TestPutObject(c *C) { objectName := objectNamePrefix + "tpo.html" localFile := "../sample/The Go Programming Language.html" // PutObject fd, err := os.Open(localFile) c.Assert(err, IsNil) defer fd.Close() err = s.bucket.PutObject(objectName, fd, Progress(&OssProgressListener{})) c.Assert(err, IsNil) // PutObjectFromFile err = s.bucket.PutObjectFromFile(objectName, localFile, Progress(&OssProgressListener{})) c.Assert(err, IsNil) // DoPutObject fd, err = os.Open(localFile) c.Assert(err, IsNil) defer fd.Close() request := &PutObjectRequest{ ObjectKey: objectName, Reader: fd, } options := []Option{Progress(&OssProgressListener{})} _, err = s.bucket.DoPutObject(request, options) c.Assert(err, IsNil) // PutObject size is 0 err = s.bucket.PutObject(objectName, strings.NewReader(""), Progress(&OssProgressListener{})) c.Assert(err, IsNil) testLogger.Println("OssProgressSuite.TestPutObject") } func (s *OssProgressSuite) TestPutObjectNegative(c *C) { objectName := objectNamePrefix + "tpon.html" localFile := "../sample/The Go Programming Language.html" // invalid endpoint client, err := New("http://oss-cn-taikang.aliyuncs.com", accessID, accessKey) c.Assert(err, IsNil) bucket, err := client.Bucket(bucketName) c.Assert(err, IsNil) err = bucket.PutObjectFromFile(objectName, localFile, Progress(&OssProgressListener{})) testLogger.Println(err) c.Assert(err, NotNil) testLogger.Println("OssProgressSuite.TestPutObjectNegative") } // TestAppendObject func (s *OssProgressSuite) TestAppendObject(c *C) { objectName := objectNamePrefix + "tao" objectValue := "昨夜雨疏风骤，浓睡不消残酒。试问卷帘人，却道海棠依旧。知否？知否？应是绿肥红瘦。" var val = []byte(objectValue) var nextPos int64 var midPos = 1 + rand.Intn(len(val)-1) // AppendObject nextPos, err := s.bucket.AppendObject(objectName, bytes.NewReader(val[0:midPos]), nextPos, Progress(&OssProgressListener{})) c.Assert(err, IsNil) // DoAppendObject request := &AppendObjectRequest{ ObjectKey: objectName, Reader: bytes.NewReader(val[midPos:]), Position: nextPos, } options := []Option{Progress(&OssProgressListener{})} _, err = s.bucket.DoAppendObject(request, options) c.Assert(err, IsNil) testLogger.Println("OssProgressSuite.TestAppendObject") } // TestMultipartUpload func (s *OssProgressSuite) TestMultipartUpload(c *C) { objectName := objectNamePrefix + "tmu.jpg" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartNum(fileName, 3) c.Assert(err, IsNil) testLogger.Println("chunks:", chunks) fd, err := os.Open(fileName) c.Assert(err, IsNil) defer fd.Close() // Initiate imur, err := s.bucket.InitiateMultipartUpload(objectName) c.Assert(err, IsNil) // UploadPart var parts []UploadPart for _, chunk := range chunks { fd.Seek(chunk.Offset, os.SEEK_SET) part, err := s.bucket.UploadPart(imur, fd, chunk.Size, chunk.Number, Progress(&OssProgressListener{})) c.Assert(err, IsNil) parts = append(parts, part) } // Complete _, err = s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) testLogger.Println("OssProgressSuite.TestMultipartUpload") } // TestMultipartUploadFromFile func (s *OssProgressSuite) TestMultipartUploadFromFile(c *C) { objectName := objectNamePrefix + "tmuff.jpg" var fileName = "../sample/BingWallpaper-2015-11-07.jpg" chunks, err := SplitFileByPartNum(fileName, 3) c.Assert(err, IsNil) // Initiate imur, err := s.bucket.InitiateMultipartUpload(objectName) c.Assert(err, IsNil) // UploadPart var parts []UploadPart for _, chunk := range chunks { part, err := s.bucket.UploadPartFromFile(imur, fileName, chunk.Offset, chunk.Size, chunk.Number, Progress(&OssProgressListener{})) c.Assert(err, IsNil) parts = append(parts, part) } // Complete _, err = s.bucket.CompleteMultipartUpload(imur, parts) c.Assert(err, IsNil) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) testLogger.Println("OssProgressSuite.TestMultipartUploadFromFile") } // TestGetObject func (s *OssProgressSuite) TestGetObject(c *C) { objectName := objectNamePrefix + "tgo.jpg" localFile := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "newpic-progress-1.jpg" // PutObject err := s.bucket.PutObjectFromFile(objectName, localFile, Progress(&OssProgressListener{})) c.Assert(err, IsNil) // GetObject body, err := s.bucket.GetObject(objectName, Progress(&OssProgressListener{})) c.Assert(err, IsNil) _, err = ioutil.ReadAll(body) c.Assert(err, IsNil) body.Close() // GetObjectToFile err = s.bucket.GetObjectToFile(objectName, newFile, Progress(&OssProgressListener{})) c.Assert(err, IsNil) // DoGetObject request := &GetObjectRequest{objectName} options := []Option{Progress(&OssProgressListener{})} result, err := s.bucket.DoGetObject(request, options) c.Assert(err, IsNil) _, err = ioutil.ReadAll(result.Response.Body) c.Assert(err, IsNil) result.Response.Body.Close() // GetObject with range body, err = s.bucket.GetObject(objectName, Range(1024, 4*1024), Progress(&OssProgressListener{})) c.Assert(err, IsNil) _, err = ioutil.ReadAll(body) c.Assert(err, IsNil) body.Close() // PutObject size is 0 err = s.bucket.PutObject(objectName, strings.NewReader(""), Progress(&OssProgressListener{})) c.Assert(err, IsNil) // GetObject size is 0 body, err = s.bucket.GetObject(objectName, Progress(&OssProgressListener{})) c.Assert(err, IsNil) _, err = ioutil.ReadAll(body) c.Assert(err, IsNil) body.Close() testLogger.Println("OssProgressSuite.TestGetObject") } // TestGetObjectNegative func (s *OssProgressSuite) TestGetObjectNegative(c *C) { objectName := objectNamePrefix + "tgon.jpg" localFile := "../sample/BingWallpaper-2015-11-07.jpg" // PutObject err := s.bucket.PutObjectFromFile(objectName, localFile) c.Assert(err, IsNil) // GetObject body, err := s.bucket.GetObject(objectName, Progress(&OssProgressListener{})) c.Assert(err, IsNil) buf := make([]byte, 4*1024) n, err := body.Read(buf) c.Assert(err, IsNil) //time.Sleep(70 * time.Second) TODO // read should fail for err == nil { n, err = body.Read(buf) n += n } c.Assert(err, NotNil) body.Close() testLogger.Println("OssProgressSuite.TestGetObjectNegative") } // TestUploadFile func (s *OssProgressSuite) TestUploadFile(c *C) { objectName := objectNamePrefix + "tuf.jpg" fileName := "../sample/BingWallpaper-2015-11-07.jpg" err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(5), Progress(&OssProgressListener{})) c.Assert(err, IsNil) err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, objectName+".cp"), Progress(&OssProgressListener{})) c.Assert(err, IsNil) testLogger.Println("OssProgressSuite.TestUploadFile") } // TestDownloadFile func (s *OssProgressSuite) TestDownloadFile(c *C) { objectName := objectNamePrefix + "tdf.jpg" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "down-new-file-progress-2.jpg" // upload err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(5), Progress(&OssProgressListener{})) c.Assert(err, IsNil) err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), Progress(&OssProgressListener{})) c.Assert(err, IsNil) err = s.bucket.DownloadFile(objectName, newFile, 50*1024, Routines(3), Checkpoint(true, ""), Progress(&OssProgressListener{})) c.Assert(err, IsNil) testLogger.Println("OssProgressSuite.TestDownloadFile") } // TestCopyFile func (s *OssProgressSuite) TestCopyFile(c *C) { srcObjectName := objectNamePrefix + "tcf.jpg" destObjectName := srcObjectName + "-copy" fileName := "../sample/BingWallpaper-2015-11-07.jpg" // upload err := s.bucket.UploadFile(srcObjectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(5), Progress(&OssProgressListener{})) c.Assert(err, IsNil) err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(3), Checkpoint(true, ""), Progress(&OssProgressListener{})) c.Assert(err, IsNil) testLogger.Println("OssProgressSuite.TestCopyFile") } aliyun-oss-go-sdk-1.5.0/oss/type.go000066400000000000000000000411441311621456000171210ustar00rootroot00000000000000package oss import ( "encoding/xml" "net/url" "time" ) // ListBucketsResult ListBuckets请求返回的结果 type ListBucketsResult struct { XMLName xml.Name `xml:"ListAllMyBucketsResult"` Prefix string `xml:"Prefix"` // 本次查询结果的前缀 Marker string `xml:"Marker"` // 标明查询的起点，未全部返回时有此节点 MaxKeys int `xml:"MaxKeys"` // 返回结果的最大数目，未全部返回时有此节点 IsTruncated bool `xml:"IsTruncated"` // 所有的结果是否已经全部返回 NextMarker string `xml:"NextMarker"` // 表示下一次查询的起点 Owner Owner `xml:"Owner"` // 拥有者信息 Buckets []BucketProperties `xml:"Buckets>Bucket"` // Bucket列表 } // BucketProperties Bucket信息 type BucketProperties struct { XMLName xml.Name `xml:"Bucket"` Name string `xml:"Name"` // Bucket名称 Location string `xml:"Location"` // Bucket所在的数据中心 CreationDate time.Time `xml:"CreationDate"` // Bucket创建时间 StorageClass string `xml:"StorageClass"` // Bucket的存储方式 } // GetBucketACLResult GetBucketACL请求返回的结果 type GetBucketACLResult struct { XMLName xml.Name `xml:"AccessControlPolicy"` ACL string `xml:"AccessControlList>Grant"` // Bucket权限 Owner Owner `xml:"Owner"` // Bucket拥有者信息 } // LifecycleConfiguration Bucket的Lifecycle配置 type LifecycleConfiguration struct { XMLName xml.Name `xml:"LifecycleConfiguration"` Rules []LifecycleRule `xml:"Rule"` } // LifecycleRule Lifecycle规则 type LifecycleRule struct { XMLName xml.Name `xml:"Rule"` ID string `xml:"ID"` // 规则唯一的ID Prefix string `xml:"Prefix"` // 规则所适用Object的前缀 Status string `xml:"Status"` // 规则是否生效 Expiration LifecycleExpiration `xml:"Expiration"` // 规则的过期属性 } // LifecycleExpiration 规则的过期属性 type LifecycleExpiration struct { XMLName xml.Name `xml:"Expiration"` Days int `xml:"Days,omitempty"` // 最后修改时间过后多少天生效 Date time.Time `xml:"Date,omitempty"` // 指定规则何时生效 } type lifecycleXML struct { XMLName xml.Name `xml:"LifecycleConfiguration"` Rules []lifecycleRule `xml:"Rule"` } type lifecycleRule struct { XMLName xml.Name `xml:"Rule"` ID string `xml:"ID"` Prefix string `xml:"Prefix"` Status string `xml:"Status"` Expiration lifecycleExpiration `xml:"Expiration"` } type lifecycleExpiration struct { XMLName xml.Name `xml:"Expiration"` Days int `xml:"Days,omitempty"` Date string `xml:"Date,omitempty"` } const expirationDateFormat = "2006-01-02T15:04:05.000Z" func convLifecycleRule(rules []LifecycleRule) []lifecycleRule { rs := []lifecycleRule{} for _, rule := range rules { r := lifecycleRule{} r.ID = rule.ID r.Prefix = rule.Prefix r.Status = rule.Status if rule.Expiration.Date.IsZero() { r.Expiration.Days = rule.Expiration.Days } else { r.Expiration.Date = rule.Expiration.Date.Format(expirationDateFormat) } rs = append(rs, r) } return rs } // BuildLifecycleRuleByDays 指定过期天数构建Lifecycle规则 func BuildLifecycleRuleByDays(id, prefix string, status bool, days int) LifecycleRule { var statusStr = "Enabled" if !status { statusStr = "Disabled" } return LifecycleRule{ID: id, Prefix: prefix, Status: statusStr, Expiration: LifecycleExpiration{Days: days}} } // BuildLifecycleRuleByDate 指定过期时间构建Lifecycle规则 func BuildLifecycleRuleByDate(id, prefix string, status bool, year, month, day int) LifecycleRule { var statusStr = "Enabled" if !status { statusStr = "Disabled" } date := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC) return LifecycleRule{ID: id, Prefix: prefix, Status: statusStr, Expiration: LifecycleExpiration{Date: date}} } // GetBucketLifecycleResult GetBucketLifecycle请求请求结果 type GetBucketLifecycleResult LifecycleConfiguration // RefererXML Referer配置 type RefererXML struct { XMLName xml.Name `xml:"RefererConfiguration"` AllowEmptyReferer bool `xml:"AllowEmptyReferer"` // 是否允许referer字段为空的请求访问 RefererList []string `xml:"RefererList>Referer"` // referer访问白名单 } // GetBucketRefererResult GetBucketReferer请教返回结果 type GetBucketRefererResult RefererXML // LoggingXML Logging配置 type LoggingXML struct { XMLName xml.Name `xml:"BucketLoggingStatus"` LoggingEnabled LoggingEnabled `xml:"LoggingEnabled"` // 访问日志信息容器 } type loggingXMLEmpty struct { XMLName xml.Name `xml:"BucketLoggingStatus"` } // LoggingEnabled 访问日志信息容器 type LoggingEnabled struct { XMLName xml.Name `xml:"LoggingEnabled"` TargetBucket string `xml:"TargetBucket"` //存放访问日志的Bucket TargetPrefix string `xml:"TargetPrefix"` //保存访问日志的文件前缀 } // GetBucketLoggingResult GetBucketLogging请求返回结果 type GetBucketLoggingResult LoggingXML // WebsiteXML Website配置 type WebsiteXML struct { XMLName xml.Name `xml:"WebsiteConfiguration"` IndexDocument IndexDocument `xml:"IndexDocument"` // 目录URL时添加的索引文件 ErrorDocument ErrorDocument `xml:"ErrorDocument"` // 404错误时使用的文件 } // IndexDocument 目录URL时添加的索引文件 type IndexDocument struct { XMLName xml.Name `xml:"IndexDocument"` Suffix string `xml:"Suffix"` // 目录URL时添加的索引文件名 } // ErrorDocument 404错误时使用的文件 type ErrorDocument struct { XMLName xml.Name `xml:"ErrorDocument"` Key string `xml:"Key"` // 404错误时使用的文件名 } // GetBucketWebsiteResult GetBucketWebsite请求返回结果 type GetBucketWebsiteResult WebsiteXML // CORSXML CORS配置 type CORSXML struct { XMLName xml.Name `xml:"CORSConfiguration"` CORSRules []CORSRule `xml:"CORSRule"` // CORS规则列表 } // CORSRule CORS规则 type CORSRule struct { XMLName xml.Name `xml:"CORSRule"` AllowedOrigin []string `xml:"AllowedOrigin"` // 允许的来源，默认通配符"*" AllowedMethod []string `xml:"AllowedMethod"` // 允许的方法 AllowedHeader []string `xml:"AllowedHeader"` // 允许的请求头 ExposeHeader []string `xml:"ExposeHeader"` // 允许的响应头 MaxAgeSeconds int `xml:"MaxAgeSeconds"` // 最大的缓存时间 } // GetBucketCORSResult GetBucketCORS请求返回的结果 type GetBucketCORSResult CORSXML // GetBucketInfoResult GetBucketInfo请求返回结果 type GetBucketInfoResult struct { XMLName xml.Name `xml:"BucketInfo"` BucketInfo BucketInfo `xml:"Bucket"` } // BucketInfo Bucket信息 type BucketInfo struct { XMLName xml.Name `xml:"Bucket"` Name string `xml:"Name"` // Bucket名称 Location string `xml:"Location"` // Bucket所在的数据中心 CreationDate time.Time `xml:"CreationDate"` // Bucket创建时间 ExtranetEndpoint string `xml:"ExtranetEndpoint"` // Bucket访问的外网域名 IntranetEndpoint string `xml:"IntranetEndpoint"` // Bucket访问的内网域名 ACL string `xml:"AccessControlList>Grant"` // Bucket权限 Owner Owner `xml:"Owner"` // Bucket拥有者信息 StorageClass string `xml:"StorageClass"` // Bucket存储类型 } // ListObjectsResult ListObjects请求返回结果 type ListObjectsResult struct { XMLName xml.Name `xml:"ListBucketResult"` Prefix string `xml:"Prefix"` // 本次查询结果的开始前缀 Marker string `xml:"Marker"` // 这次查询的起点 MaxKeys int `xml:"MaxKeys"` // 请求返回结果的最大数目 Delimiter string `xml:"Delimiter"` // 对Object名字进行分组的字符 IsTruncated bool `xml:"IsTruncated"` // 是否所有的结果都已经返回 NextMarker string `xml:"NextMarker"` // 下一次查询的起点 Objects []ObjectProperties `xml:"Contents"` // Object类别 CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` // 以delimiter结尾并有共同前缀的Object的集合 } // ObjectProperties Objecct属性 type ObjectProperties struct { XMLName xml.Name `xml:"Contents"` Key string `xml:"Key"` // Object的Key Type string `xml:"Type"` // Object Type Size int64 `xml:"Size"` // Object的长度字节数 ETag string `xml:"ETag"` // 标示Object的内容 Owner Owner `xml:"Owner"` // 保存Object拥有者信息的容器 LastModified time.Time `xml:"LastModified"` // Object最后修改时间 StorageClass string `xml:"StorageClass"` // Object的存储类型 } // Owner Bucket/Object的owner type Owner struct { XMLName xml.Name `xml:"Owner"` ID string `xml:"ID"` // 用户ID DisplayName string `xml:"DisplayName"` // Owner名字 } // CopyObjectResult CopyObject请求返回的结果 type CopyObjectResult struct { XMLName xml.Name `xml:"CopyObjectResult"` LastModified time.Time `xml:"LastModified"` // 新Object最后更新时间 ETag string `xml:"ETag"` // 新Object的ETag值 } // GetObjectACLResult GetObjectACL请求返回的结果 type GetObjectACLResult GetBucketACLResult type deleteXML struct { XMLName xml.Name `xml:"Delete"` Objects []DeleteObject `xml:"Object"` // 删除的所有Object Quiet bool `xml:"Quiet"` // 安静响应模式 } // DeleteObject 删除的Object type DeleteObject struct { XMLName xml.Name `xml:"Object"` Key string `xml:"Key"` // Object名称 } // DeleteObjectsResult DeleteObjects请求返回结果 type DeleteObjectsResult struct { XMLName xml.Name `xml:"DeleteResult"` DeletedObjects []string `xml:"Deleted>Key"` // 删除的Object列表 } // InitiateMultipartUploadResult InitiateMultipartUpload请求返回结果 type InitiateMultipartUploadResult struct { XMLName xml.Name `xml:"InitiateMultipartUploadResult"` Bucket string `xml:"Bucket"` // Bucket名称 Key string `xml:"Key"` // 上传Object名称 UploadID string `xml:"UploadId"` // 生成的UploadId } // UploadPart 上传/拷贝的分片 type UploadPart struct { XMLName xml.Name `xml:"Part"` PartNumber int `xml:"PartNumber"` // Part编号 ETag string `xml:"ETag"` // ETag缓存码 } type uploadParts []UploadPart func (slice uploadParts) Len() int { return len(slice) } func (slice uploadParts) Less(i, j int) bool { return slice[i].PartNumber < slice[j].PartNumber } func (slice uploadParts) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } // UploadPartCopyResult 拷贝分片请求返回的结果 type UploadPartCopyResult struct { XMLName xml.Name `xml:"CopyPartResult"` LastModified time.Time `xml:"LastModified"` // 最后修改时间 ETag string `xml:"ETag"` // ETag } type completeMultipartUploadXML struct { XMLName xml.Name `xml:"CompleteMultipartUpload"` Part []UploadPart `xml:"Part"` } // CompleteMultipartUploadResult 提交分片上传任务返回结果 type CompleteMultipartUploadResult struct { XMLName xml.Name `xml:"CompleteMultipartUploadResult"` Location string `xml:"Location"` // Object的URL Bucket string `xml:"Bucket"` // Bucket名称 ETag string `xml:"ETag"` // Object的ETag Key string `xml:"Key"` // Object的名字 } // ListUploadedPartsResult ListUploadedParts请求返回结果 type ListUploadedPartsResult struct { XMLName xml.Name `xml:"ListPartsResult"` Bucket string `xml:"Bucket"` // Bucket名称 Key string `xml:"Key"` // Object名称 UploadID string `xml:"UploadId"` // 上传Id NextPartNumberMarker string `xml:"NextPartNumberMarker"` // 下一个Part的位置 MaxParts int `xml:"MaxParts"` // 最大Part个数 IsTruncated bool `xml:"IsTruncated"` // 是否完全上传完成 UploadedParts []UploadedPart `xml:"Part"` // 已完成的Part } // UploadedPart 该任务已经上传的分片 type UploadedPart struct { XMLName xml.Name `xml:"Part"` PartNumber int `xml:"PartNumber"` // Part编号 LastModified time.Time `xml:"LastModified"` // 最后一次修改时间 ETag string `xml:"ETag"` // ETag缓存码 Size int `xml:"Size"` // Part大小 } // ListMultipartUploadResult ListMultipartUpload请求返回结果 type ListMultipartUploadResult struct { XMLName xml.Name `xml:"ListMultipartUploadsResult"` Bucket string `xml:"Bucket"` // Bucket名称 Delimiter string `xml:"Delimiter"` // 分组分割符 Prefix string `xml:"Prefix"` // 筛选前缀 KeyMarker string `xml:"KeyMarker"` // 起始Object位置 UploadIDMarker string `xml:"UploadIdMarker"` // 起始UploadId位置 NextKeyMarker string `xml:"NextKeyMarker"` // 如果没有全部返回，标明接下去的KeyMarker位置 NextUploadIDMarker string `xml:"NextUploadIdMarker"` // 如果没有全部返回，标明接下去的UploadId位置 MaxUploads int `xml:"MaxUploads"` // 返回最大Upload数目 IsTruncated bool `xml:"IsTruncated"` // 是否完全返回 Uploads []UncompletedUpload `xml:"Upload"` // 未完成上传的MultipartUpload CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` // 所有名字包含指定的前缀且第一次出现delimiter字符之间的object作为一组的分组结果 } // UncompletedUpload 未完成的Upload任务 type UncompletedUpload struct { XMLName xml.Name `xml:"Upload"` Key string `xml:"Key"` // Object名称 UploadID string `xml:"UploadId"` // 对应UploadId Initiated time.Time `xml:"Initiated"` // 初始化时间，格式2012-02-23T04:18:23.000Z } // 解析URL编码 func decodeDeleteObjectsResult(result *DeleteObjectsResult) error { var err error for i := 0; i < len(result.DeletedObjects); i++ { result.DeletedObjects[i], err = url.QueryUnescape(result.DeletedObjects[i]) if err != nil { return err } } return nil } // 解析URL编码 func decodeListObjectsResult(result *ListObjectsResult) error { var err error result.Prefix, err = url.QueryUnescape(result.Prefix) if err != nil { return err } result.Marker, err = url.QueryUnescape(result.Marker) if err != nil { return err } result.Delimiter, err = url.QueryUnescape(result.Delimiter) if err != nil { return err } result.NextMarker, err = url.QueryUnescape(result.NextMarker) if err != nil { return err } for i := 0; i < len(result.Objects); i++ { result.Objects[i].Key, err = url.QueryUnescape(result.Objects[i].Key) if err != nil { return err } } for i := 0; i < len(result.CommonPrefixes); i++ { result.CommonPrefixes[i], err = url.QueryUnescape(result.CommonPrefixes[i]) if err != nil { return err } } return nil } // 解析URL编码 func decodeListMultipartUploadResult(result *ListMultipartUploadResult) error { var err error result.Prefix, err = url.QueryUnescape(result.Prefix) if err != nil { return err } result.Delimiter, err = url.QueryUnescape(result.Delimiter) if err != nil { return err } result.KeyMarker, err = url.QueryUnescape(result.KeyMarker) if err != nil { return err } result.NextKeyMarker, err = url.QueryUnescape(result.NextKeyMarker) if err != nil { return err } for i := 0; i < len(result.Uploads); i++ { result.Uploads[i].Key, err = url.QueryUnescape(result.Uploads[i].Key) if err != nil { return err } } for i := 0; i < len(result.CommonPrefixes); i++ { result.CommonPrefixes[i], err = url.QueryUnescape(result.CommonPrefixes[i]) if err != nil { return err } } return nil } // createBucketConfiguration 规则的过期属性 type createBucketConfiguration struct { XMLName xml.Name `xml:"CreateBucketConfiguration"` StorageClass StorageClassType `xml:"StorageClass,omitempty"` } aliyun-oss-go-sdk-1.5.0/oss/type_test.go000066400000000000000000000070501311621456000201560ustar00rootroot00000000000000package oss import ( "net/url" "sort" . "gopkg.in/check.v1" ) type OssTypeSuite struct{} var _ = Suite(&OssTypeSuite{}) var ( goStr = "go go + go <> go" chnStr = "试问闲情几许" goURLStr = url.QueryEscape(goStr) chnURLStr = url.QueryEscape(chnStr) ) func (s *OssTypeSuite) TestConvLifecycleRule(c *C) { r1 := BuildLifecycleRuleByDate("id1", "one", true, 2015, 11, 11) r2 := BuildLifecycleRuleByDays("id2", "two", false, 3) rs := convLifecycleRule([]LifecycleRule{r1}) c.Assert(rs[0].ID, Equals, "id1") c.Assert(rs[0].Prefix, Equals, "one") c.Assert(rs[0].Status, Equals, "Enabled") c.Assert(rs[0].Expiration.Date, Equals, "2015-11-11T00:00:00.000Z") c.Assert(rs[0].Expiration.Days, Equals, 0) rs = convLifecycleRule([]LifecycleRule{r2}) c.Assert(rs[0].ID, Equals, "id2") c.Assert(rs[0].Prefix, Equals, "two") c.Assert(rs[0].Status, Equals, "Disabled") c.Assert(rs[0].Expiration.Date, Equals, "") c.Assert(rs[0].Expiration.Days, Equals, 3) } func (s *OssTypeSuite) TestDecodeDeleteObjectsResult(c *C) { var res DeleteObjectsResult err := decodeDeleteObjectsResult(&res) c.Assert(err, IsNil) res.DeletedObjects = []string{""} err = decodeDeleteObjectsResult(&res) c.Assert(err, IsNil) c.Assert(res.DeletedObjects[0], Equals, "") res.DeletedObjects = []string{goURLStr, chnURLStr} err = decodeDeleteObjectsResult(&res) c.Assert(err, IsNil) c.Assert(res.DeletedObjects[0], Equals, goStr) c.Assert(res.DeletedObjects[1], Equals, chnStr) } func (s *OssTypeSuite) TestDecodeListObjectsResult(c *C) { var res ListObjectsResult err := decodeListObjectsResult(&res) c.Assert(err, IsNil) res = ListObjectsResult{} err = decodeListObjectsResult(&res) c.Assert(err, IsNil) res = ListObjectsResult{Prefix: goURLStr, Marker: goURLStr, Delimiter: goURLStr, NextMarker: goURLStr, Objects: []ObjectProperties{{Key: chnURLStr}}, CommonPrefixes: []string{chnURLStr}} err = decodeListObjectsResult(&res) c.Assert(err, IsNil) c.Assert(res.Prefix, Equals, goStr) c.Assert(res.Marker, Equals, goStr) c.Assert(res.Delimiter, Equals, goStr) c.Assert(res.NextMarker, Equals, goStr) c.Assert(res.Objects[0].Key, Equals, chnStr) c.Assert(res.CommonPrefixes[0], Equals, chnStr) } func (s *OssTypeSuite) TestDecodeListMultipartUploadResult(c *C) { res := ListMultipartUploadResult{} err := decodeListMultipartUploadResult(&res) c.Assert(err, IsNil) res = ListMultipartUploadResult{Prefix: goURLStr, KeyMarker: goURLStr, Delimiter: goURLStr, NextKeyMarker: goURLStr, Uploads: []UncompletedUpload{{Key: chnURLStr}}} err = decodeListMultipartUploadResult(&res) c.Assert(err, IsNil) c.Assert(res.Prefix, Equals, goStr) c.Assert(res.KeyMarker, Equals, goStr) c.Assert(res.Delimiter, Equals, goStr) c.Assert(res.NextKeyMarker, Equals, goStr) c.Assert(res.Uploads[0].Key, Equals, chnStr) } func (s *OssTypeSuite) TestSortUploadPart(c *C) { parts := []UploadPart{} sort.Sort(uploadParts(parts)) c.Assert(len(parts), Equals, 0) parts = []UploadPart{ {PartNumber: 5, ETag: "E5"}, {PartNumber: 1, ETag: "E1"}, {PartNumber: 4, ETag: "E4"}, {PartNumber: 2, ETag: "E2"}, {PartNumber: 3, ETag: "E3"}, } sort.Sort(uploadParts(parts)) c.Assert(parts[0].PartNumber, Equals, 1) c.Assert(parts[0].ETag, Equals, "E1") c.Assert(parts[1].PartNumber, Equals, 2) c.Assert(parts[1].ETag, Equals, "E2") c.Assert(parts[2].PartNumber, Equals, 3) c.Assert(parts[2].ETag, Equals, "E3") c.Assert(parts[3].PartNumber, Equals, 4) c.Assert(parts[3].ETag, Equals, "E4") c.Assert(parts[4].PartNumber, Equals, 5) c.Assert(parts[4].ETag, Equals, "E5") } aliyun-oss-go-sdk-1.5.0/oss/upload.go000066400000000000000000000260121311621456000174210ustar00rootroot00000000000000package oss import ( "crypto/md5" "encoding/base64" "encoding/json" "errors" "io/ioutil" "os" "time" ) // // UploadFile 分片上传文件 // // objectKey object名称。 // filePath 本地文件。需要上传的文件。 // partSize 本次上传文件片的大小，字节数。比如100 * 1024为每片100KB。 // options 上传Object时可以指定Object的属性。详见InitiateMultipartUpload。 // // error 操作成功为nil，非nil为错误信息。 // func (bucket Bucket) UploadFile(objectKey, filePath string, partSize int64, options ...Option) error { if partSize < MinPartSize || partSize > MaxPartSize { return errors.New("oss: part size invalid range (1024KB, 5GB]") } cpConf, err := getCpConfig(options, filePath) if err != nil { return err } routines := getRoutines(options) if cpConf.IsEnable { return bucket.uploadFileWithCp(objectKey, filePath, partSize, options, cpConf.FilePath, routines) } return bucket.uploadFile(objectKey, filePath, partSize, options, routines) } // ----- 并发无断点的上传 ----- // 获取Checkpoint配置 func getCpConfig(options []Option, filePath string) (*cpConfig, error) { cpc := &cpConfig{} cpcOpt, err := findOption(options, checkpointConfig, nil) if err != nil || cpcOpt == nil { return cpc, err } cpc = cpcOpt.(*cpConfig) if cpc.IsEnable && cpc.FilePath == "" { cpc.FilePath = filePath + CheckpointFileSuffix } return cpc, nil } // 获取并发数，默认并发数1 func getRoutines(options []Option) int { rtnOpt, err := findOption(options, routineNum, nil) if err != nil || rtnOpt == nil { return 1 } rs := rtnOpt.(int) if rs < 1 { rs = 1 } else if rs > 100 { rs = 100 } return rs } // 获取进度回调 func getProgressListener(options []Option) ProgressListener { isSet, listener, _ := isOptionSet(options, progressListener) if !isSet { return nil } return listener.(ProgressListener) } // 测试使用 type uploadPartHook func(id int, chunk FileChunk) error var uploadPartHooker uploadPartHook = defaultUploadPart func defaultUploadPart(id int, chunk FileChunk) error { return nil } // 工作协程参数 type workerArg struct { bucket *Bucket filePath string imur InitiateMultipartUploadResult hook uploadPartHook } // 工作协程 func worker(id int, arg workerArg, jobs <-chan FileChunk, results chan<- UploadPart, failed chan<- error, die <-chan bool) { for chunk := range jobs { if err := arg.hook(id, chunk); err != nil { failed <- err break } part, err := arg.bucket.UploadPartFromFile(arg.imur, arg.filePath, chunk.Offset, chunk.Size, chunk.Number) if err != nil { failed <- err break } select { case <-die: return default: } results <- part } } // 调度协程 func scheduler(jobs chan FileChunk, chunks []FileChunk) { for _, chunk := range chunks { jobs <- chunk } close(jobs) } func getTotalBytes(chunks []FileChunk) int64 { var tb int64 for _, chunk := range chunks { tb += chunk.Size } return tb } // 并发上传，不带断点续传功能 func (bucket Bucket) uploadFile(objectKey, filePath string, partSize int64, options []Option, routines int) error { listener := getProgressListener(options) chunks, err := SplitFileByPartSize(filePath, partSize) if err != nil { return err } // 初始化上传任务 imur, err := bucket.InitiateMultipartUpload(objectKey, options...) if err != nil { return err } jobs := make(chan FileChunk, len(chunks)) results := make(chan UploadPart, len(chunks)) failed := make(chan error) die := make(chan bool) var completedBytes int64 totalBytes := getTotalBytes(chunks) event := newProgressEvent(TransferStartedEvent, 0, totalBytes) publishProgress(listener, event) // 启动工作协程 arg := workerArg{&bucket, filePath, imur, uploadPartHooker} for w := 1; w <= routines; w++ { go worker(w, arg, jobs, results, failed, die) } // 并发上传分片 go scheduler(jobs, chunks) // 等待分配分片上传完成 completed := 0 parts := make([]UploadPart, len(chunks)) for completed < len(chunks) { select { case part := <-results: completed++ parts[part.PartNumber-1] = part completedBytes += chunks[part.PartNumber-1].Size event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes) publishProgress(listener, event) case err := <-failed: close(die) event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes) publishProgress(listener, event) bucket.AbortMultipartUpload(imur) return err } if completed >= len(chunks) { break } } event = newProgressEvent(TransferStartedEvent, completedBytes, totalBytes) publishProgress(listener, event) // 提交任务 _, err = bucket.CompleteMultipartUpload(imur, parts) if err != nil { bucket.AbortMultipartUpload(imur) return err } return nil } // ----- 并发带断点的上传 ----- const uploadCpMagic = "FE8BB4EA-B593-4FAC-AD7A-2459A36E2E62" type uploadCheckpoint struct { Magic string // magic MD5 string // cp内容的MD5 FilePath string // 本地文件 FileStat cpStat // 文件状态 ObjectKey string // key UploadID string // upload id Parts []cpPart // 本地文件的全部分片 } type cpStat struct { Size int64 // 文件大小 LastModified time.Time // 本地文件最后修改时间 MD5 string // 本地文件MD5 } type cpPart struct { Chunk FileChunk // 分片 Part UploadPart // 上传完成的分片 IsCompleted bool // upload是否完成 } // CP数据是否有效，CP有效且文件没有更新时有效 func (cp uploadCheckpoint) isValid(filePath string) (bool, error) { // 比较CP的Magic及MD5 cpb := cp cpb.MD5 = "" js, _ := json.Marshal(cpb) sum := md5.Sum(js) b64 := base64.StdEncoding.EncodeToString(sum[:]) if cp.Magic != uploadCpMagic || b64 != cp.MD5 { return false, nil } // 确认本地文件是否更新 fd, err := os.Open(filePath) if err != nil { return false, err } defer fd.Close() st, err := fd.Stat() if err != nil { return false, err } md, err := calcFileMD5(filePath) if err != nil { return false, err } // 比较文件大小/文件最后更新时间/文件MD5 if cp.FileStat.Size != st.Size() || cp.FileStat.LastModified != st.ModTime() || cp.FileStat.MD5 != md { return false, nil } return true, nil } // 从文件中load func (cp *uploadCheckpoint) load(filePath string) error { contents, err := ioutil.ReadFile(filePath) if err != nil { return err } err = json.Unmarshal(contents, cp) return err } // dump到文件 func (cp *uploadCheckpoint) dump(filePath string) error { bcp := *cp // 计算MD5 bcp.MD5 = "" js, err := json.Marshal(bcp) if err != nil { return err } sum := md5.Sum(js) b64 := base64.StdEncoding.EncodeToString(sum[:]) bcp.MD5 = b64 // 序列化 js, err = json.Marshal(bcp) if err != nil { return err } // dump return ioutil.WriteFile(filePath, js, FilePermMode) } // 更新分片状态 func (cp *uploadCheckpoint) updatePart(part UploadPart) { cp.Parts[part.PartNumber-1].Part = part cp.Parts[part.PartNumber-1].IsCompleted = true } // 未完成的分片 func (cp *uploadCheckpoint) todoParts() []FileChunk { fcs := []FileChunk{} for _, part := range cp.Parts { if !part.IsCompleted { fcs = append(fcs, part.Chunk) } } return fcs } // 所有的分片 func (cp *uploadCheckpoint) allParts() []UploadPart { ps := []UploadPart{} for _, part := range cp.Parts { ps = append(ps, part.Part) } return ps } // 完成的字节数 func (cp *uploadCheckpoint) getCompletedBytes() int64 { var completedBytes int64 for _, part := range cp.Parts { if part.IsCompleted { completedBytes += part.Chunk.Size } } return completedBytes } // 计算文件文件MD5 func calcFileMD5(filePath string) (string, error) { return "", nil } // 初始化分片上传 func prepare(cp *uploadCheckpoint, objectKey, filePath string, partSize int64, bucket *Bucket, options []Option) error { // cp cp.Magic = uploadCpMagic cp.FilePath = filePath cp.ObjectKey = objectKey // localfile fd, err := os.Open(filePath) if err != nil { return err } defer fd.Close() st, err := fd.Stat() if err != nil { return err } cp.FileStat.Size = st.Size() cp.FileStat.LastModified = st.ModTime() md, err := calcFileMD5(filePath) if err != nil { return err } cp.FileStat.MD5 = md // chunks parts, err := SplitFileByPartSize(filePath, partSize) if err != nil { return err } cp.Parts = make([]cpPart, len(parts)) for i, part := range parts { cp.Parts[i].Chunk = part cp.Parts[i].IsCompleted = false } // init load imur, err := bucket.InitiateMultipartUpload(objectKey, options...) if err != nil { return err } cp.UploadID = imur.UploadID return nil } // 提交分片上传，删除CP文件 func complete(cp *uploadCheckpoint, bucket *Bucket, parts []UploadPart, cpFilePath string) error { imur := InitiateMultipartUploadResult{Bucket: bucket.BucketName, Key: cp.ObjectKey, UploadID: cp.UploadID} _, err := bucket.CompleteMultipartUpload(imur, parts) if err != nil { return err } os.Remove(cpFilePath) return err } // 并发带断点的上传 func (bucket Bucket) uploadFileWithCp(objectKey, filePath string, partSize int64, options []Option, cpFilePath string, routines int) error { listener := getProgressListener(options) // LOAD CP数据 ucp := uploadCheckpoint{} err := ucp.load(cpFilePath) if err != nil { os.Remove(cpFilePath) } // LOAD出错或数据无效重新初始化上传 valid, err := ucp.isValid(filePath) if err != nil || !valid { if err = prepare(&ucp, objectKey, filePath, partSize, &bucket, options); err != nil { return err } os.Remove(cpFilePath) } chunks := ucp.todoParts() imur := InitiateMultipartUploadResult{ Bucket: bucket.BucketName, Key: objectKey, UploadID: ucp.UploadID} jobs := make(chan FileChunk, len(chunks)) results := make(chan UploadPart, len(chunks)) failed := make(chan error) die := make(chan bool) completedBytes := ucp.getCompletedBytes() event := newProgressEvent(TransferStartedEvent, completedBytes, ucp.FileStat.Size) publishProgress(listener, event) // 启动工作协程 arg := workerArg{&bucket, filePath, imur, uploadPartHooker} for w := 1; w <= routines; w++ { go worker(w, arg, jobs, results, failed, die) } // 并发上传分片 go scheduler(jobs, chunks) // 等待分配分片上传完成 completed := 0 for completed < len(chunks) { select { case part := <-results: completed++ ucp.updatePart(part) ucp.dump(cpFilePath) completedBytes += ucp.Parts[part.PartNumber-1].Chunk.Size event = newProgressEvent(TransferDataEvent, completedBytes, ucp.FileStat.Size) publishProgress(listener, event) case err := <-failed: close(die) event = newProgressEvent(TransferFailedEvent, completedBytes, ucp.FileStat.Size) publishProgress(listener, event) return err } if completed >= len(chunks) { break } } event = newProgressEvent(TransferCompletedEvent, completedBytes, ucp.FileStat.Size) publishProgress(listener, event) // 提交分片上传 err = complete(&ucp, &bucket, ucp.allParts(), cpFilePath) return err } aliyun-oss-go-sdk-1.5.0/oss/upload_test.go000066400000000000000000000310511311621456000204570ustar00rootroot00000000000000package oss import ( "fmt" "io" "os" "time" . "gopkg.in/check.v1" ) type OssUploadSuite struct { client *Client bucket *Bucket } var _ = Suite(&OssUploadSuite{}) // Run once when the suite starts running func (s *OssUploadSuite) SetUpSuite(c *C) { client, err := New(endpoint, accessID, accessKey) c.Assert(err, IsNil) s.client = client s.client.CreateBucket(bucketName) time.Sleep(5 * time.Second) bucket, err := s.client.Bucket(bucketName) c.Assert(err, IsNil) s.bucket = bucket testLogger.Println("test upload started") } // Run before each test or benchmark starts running func (s *OssUploadSuite) TearDownSuite(c *C) { // Delete Part lmur, err := s.bucket.ListMultipartUploads() c.Assert(err, IsNil) for _, upload := range lmur.Uploads { var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName, Key: upload.Key, UploadID: upload.UploadID} err = s.bucket.AbortMultipartUpload(imur) c.Assert(err, IsNil) } // Delete Objects lor, err := s.bucket.ListObjects() c.Assert(err, IsNil) for _, object := range lor.Objects { err = s.bucket.DeleteObject(object.Key) c.Assert(err, IsNil) } testLogger.Println("test upload completed") } // Run after each test or benchmark runs func (s *OssUploadSuite) SetUpTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // Run once after all tests or benchmarks have finished running func (s *OssUploadSuite) TearDownTest(c *C) { err := removeTempFiles("../oss", ".jpg") c.Assert(err, IsNil) } // TestUploadRoutineWithoutRecovery 多线程无断点恢复的上传 func (s *OssUploadSuite) TestUploadRoutineWithoutRecovery(c *C) { objectName := objectNamePrefix + "turwr" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "upload-new-file.jpg" // 不指定Routines，默认单线程 err := s.bucket.UploadFile(objectName, fileName, 100*1024) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 指定线程数1 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(1)) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 指定线程数3，小于分片数5 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3)) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 指定线程数5，等于分片数 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(5)) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 指定线程数10，大于分片数5 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(10)) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 线程值无效自动变成1 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(0)) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 线程值无效自动变成1 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(-1)) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // option err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Meta("myprop", "mypropval")) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // ErrorHooker UploadPart请求Hook func ErrorHooker(id int, chunk FileChunk) error { if chunk.Number == 5 { time.Sleep(time.Second) return fmt.Errorf("ErrorHooker") } return nil } // TestUploadRoutineWithoutRecovery 多线程无断点恢复的上传 func (s *OssUploadSuite) TestUploadRoutineWithoutRecoveryNegative(c *C) { objectName := objectNamePrefix + "turwrn" fileName := "../sample/BingWallpaper-2015-11-07.jpg" uploadPartHooker = ErrorHooker // worker线程错误 err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(2)) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") uploadPartHooker = defaultUploadPart // 本地文件不存在 err = s.bucket.UploadFile(objectName, "NotExist", 100*1024, Routines(2)) c.Assert(err, NotNil) // 指定的分片大小无效 err = s.bucket.UploadFile(objectName, fileName, 1024, Routines(2)) c.Assert(err, NotNil) err = s.bucket.UploadFile(objectName, fileName, 1024*1024*1024*100, Routines(2)) c.Assert(err, NotNil) } // TestUploadRoutineWithRecovery 多线程且有断点恢复的上传 func (s *OssUploadSuite) TestUploadRoutineWithRecovery(c *C) { objectName := objectNamePrefix + "turtr" fileName := "../sample/BingWallpaper-2015-11-07.jpg" newFile := "upload-new-file-2.jpg" // Routines默认值，CP开启默认路径是fileName+.cp // 第一次上传，上传4片 uploadPartHooker = ErrorHooker err := s.bucket.UploadFile(objectName, fileName, 100*1024, Checkpoint(true, "")) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") uploadPartHooker = defaultUploadPart // check cp ucp := uploadCheckpoint{} err = ucp.load(fileName + ".cp") c.Assert(err, IsNil) c.Assert(ucp.Magic, Equals, uploadCpMagic) c.Assert(len(ucp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ==")) c.Assert(ucp.FilePath, Equals, fileName) c.Assert(ucp.FileStat.Size, Equals, int64(482048)) c.Assert(len(ucp.FileStat.LastModified.String()), Equals, len("2015-12-17 18:43:03 +0800 CST")) c.Assert(ucp.FileStat.MD5, Equals, "") c.Assert(ucp.ObjectKey, Equals, objectName) c.Assert(len(ucp.UploadID), Equals, len("3F79722737D1469980DACEDCA325BB52")) c.Assert(len(ucp.Parts), Equals, 5) c.Assert(len(ucp.todoParts()), Equals, 1) c.Assert(len(ucp.allParts()), Equals, 5) // 第二次上传，完成剩余的一片 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Checkpoint(true, "")) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) err = ucp.load(fileName + ".cp") c.Assert(err, NotNil) // Routines指定，CP指定 uploadPartHooker = ErrorHooker err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(2), Checkpoint(true, objectName+".cp")) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") uploadPartHooker = defaultUploadPart // check cp ucp = uploadCheckpoint{} err = ucp.load(objectName + ".cp") c.Assert(err, IsNil) c.Assert(ucp.Magic, Equals, uploadCpMagic) c.Assert(len(ucp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ==")) c.Assert(ucp.FilePath, Equals, fileName) c.Assert(ucp.FileStat.Size, Equals, int64(482048)) c.Assert(len(ucp.FileStat.LastModified.String()), Equals, len("2015-12-17 18:43:03 +0800 CST")) c.Assert(ucp.FileStat.MD5, Equals, "") c.Assert(ucp.ObjectKey, Equals, objectName) c.Assert(len(ucp.UploadID), Equals, len("3F79722737D1469980DACEDCA325BB52")) c.Assert(len(ucp.Parts), Equals, 5) c.Assert(len(ucp.todoParts()), Equals, 1) c.Assert(len(ucp.allParts()), Equals, 5) err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, objectName+".cp")) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) err = ucp.load(objectName + ".cp") c.Assert(err, NotNil) // 一次完成上传，中间没有错误 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, "")) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // 用多协程下载，中间没有错误 err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(10), Checkpoint(true, "")) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) // option err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, ""), Meta("myprop", "mypropval")) meta, err := s.bucket.GetObjectDetailedMeta(objectName) c.Assert(err, IsNil) c.Assert(meta.Get("X-Oss-Meta-Myprop"), Equals, "mypropval") os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err = compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } // TestUploadRoutineWithoutRecovery 多线程无断点恢复的上传 func (s *OssUploadSuite) TestUploadRoutineWithRecoveryNegative(c *C) { objectName := objectNamePrefix + "turrn" fileName := "../sample/BingWallpaper-2015-11-07.jpg" // 本地文件不存在 err := s.bucket.UploadFile(objectName, "NotExist", 100*1024, Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.UploadFile(objectName, "NotExist", 100*1024, Routines(2), Checkpoint(true, "")) c.Assert(err, NotNil) // 指定的分片大小无效 err = s.bucket.UploadFile(objectName, fileName, 1024, Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.UploadFile(objectName, fileName, 1024, Routines(2), Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.UploadFile(objectName, fileName, 1024*1024*1024*100, Checkpoint(true, "")) c.Assert(err, NotNil) err = s.bucket.UploadFile(objectName, fileName, 1024*1024*1024*100, Routines(2), Checkpoint(true, "")) c.Assert(err, NotNil) } // TestUploadLocalFileChange 上传过程中文件修改了 func (s *OssUploadSuite) TestUploadLocalFileChange(c *C) { objectName := objectNamePrefix + "tulfc" fileName := "../sample/BingWallpaper-2015-11-07.jpg" localFile := "BingWallpaper-2015-11-07.jpg" newFile := "upload-new-file-3.jpg" os.Remove(localFile) err := copyFile(fileName, localFile) c.Assert(err, IsNil) // 第一次上传，上传4片 uploadPartHooker = ErrorHooker err = s.bucket.UploadFile(objectName, localFile, 100*1024, Checkpoint(true, "")) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "ErrorHooker") uploadPartHooker = defaultUploadPart os.Remove(localFile) err = copyFile(fileName, localFile) c.Assert(err, IsNil) // 文件修改，第二次上传全部分片重新上传 err = s.bucket.UploadFile(objectName, localFile, 100*1024, Checkpoint(true, "")) c.Assert(err, IsNil) os.Remove(newFile) err = s.bucket.GetObjectToFile(objectName, newFile) c.Assert(err, IsNil) eq, err := compareFiles(fileName, newFile) c.Assert(err, IsNil) c.Assert(eq, Equals, true) err = s.bucket.DeleteObject(objectName) c.Assert(err, IsNil) } func copyFile(src, dst string) error { srcFile, err := os.Open(src) if err != nil { return err } defer srcFile.Close() dstFile, err := os.Create(dst) if err != nil { return err } defer dstFile.Close() _, err = io.Copy(dstFile, srcFile) return err } aliyun-oss-go-sdk-1.5.0/oss/utils.go000066400000000000000000000151431311621456000173000ustar00rootroot00000000000000package oss import ( "bytes" "errors" "fmt" "hash/crc64" "net/http" "os" "os/exec" "runtime" "strconv" "strings" "time" ) // Get User Agent // Go sdk相关信息，包括sdk版本，操作系统类型，GO版本 var userAgent = func() string { sys := getSysInfo() return fmt.Sprintf("aliyun-sdk-go/%s (%s/%s/%s;%s)", Version, sys.name, sys.release, sys.machine, runtime.Version()) }() type sysInfo struct { name string // 操作系统名称windows/Linux release string // 操作系统版本 2.6.32-220.23.2.ali1089.el5.x86_64等 machine string // 机器类型amd64/x86_64 } // Get　system info // 获取操作系统信息、机器类型 func getSysInfo() sysInfo { name := runtime.GOOS release := "-" machine := runtime.GOARCH if out, err := exec.Command("uname", "-s").CombinedOutput(); err == nil { name = string(bytes.TrimSpace(out)) } if out, err := exec.Command("uname", "-r").CombinedOutput(); err == nil { release = string(bytes.TrimSpace(out)) } if out, err := exec.Command("uname", "-m").CombinedOutput(); err == nil { machine = string(bytes.TrimSpace(out)) } return sysInfo{name: name, release: release, machine: machine} } // unpackedRange type unpackedRange struct { hasStart bool // 是否指定了起点 hasEnd bool // 是否指定了终点 start int64 // 起点 end int64 // 终点 } // invalid Range Error func invalidRangeError(r string) error { return fmt.Errorf("InvalidRange %s", r) } // parseRange parse various styles of range such as bytes=M-N func parseRange(normalizedRange string) (*unpackedRange, error) { var err error hasStart := false hasEnd := false var start int64 var end int64 // bytes==M-N or ranges=M-N nrSlice := strings.Split(normalizedRange, "=") if len(nrSlice) != 2 || nrSlice[0] != "bytes" { return nil, invalidRangeError(normalizedRange) } // bytes=M-N,X-Y rSlice := strings.Split(nrSlice[1], ",") rStr := rSlice[0] if strings.HasSuffix(rStr, "-") { // M- startStr := rStr[:len(rStr)-1] start, err = strconv.ParseInt(startStr, 10, 64) if err != nil { return nil, invalidRangeError(normalizedRange) } hasStart = true } else if strings.HasPrefix(rStr, "-") { // -N len := rStr[1:] end, err = strconv.ParseInt(len, 10, 64) if err != nil { return nil, invalidRangeError(normalizedRange) } if end == 0 { // -0 return nil, invalidRangeError(normalizedRange) } hasEnd = true } else { // M-N valSlice := strings.Split(rStr, "-") if len(valSlice) != 2 { return nil, invalidRangeError(normalizedRange) } start, err = strconv.ParseInt(valSlice[0], 10, 64) if err != nil { return nil, invalidRangeError(normalizedRange) } hasStart = true end, err = strconv.ParseInt(valSlice[1], 10, 64) if err != nil { return nil, invalidRangeError(normalizedRange) } hasEnd = true } return &unpackedRange{hasStart, hasEnd, start, end}, nil } // adjustRange return adjusted range, adjust the range according to the length of the file func adjustRange(ur *unpackedRange, size int64) (start, end int64) { if ur == nil { return 0, size } if ur.hasStart && ur.hasEnd { start = ur.start end = ur.end + 1 if ur.start < 0 || ur.start >= size || ur.end > size || ur.start > ur.end { start = 0 end = size } } else if ur.hasStart { start = ur.start end = size if ur.start < 0 || ur.start >= size { start = 0 } } else if ur.hasEnd { start = size - ur.end end = size if ur.end < 0 || ur.end > size { start = 0 end = size } } return } // GetNowSec returns Unix time, the number of seconds elapsed since January 1, 1970 UTC. // 获取当前时间，从UTC开始的秒数。 func GetNowSec() int64 { return time.Now().Unix() } // GetNowNanoSec returns t as a Unix time, the number of nanoseconds elapsed // since January 1, 1970 UTC. The result is undefined if the Unix time // in nanoseconds cannot be represented by an int64. Note that this // means the result of calling UnixNano on the zero Time is undefined. // 获取当前时间，从UTC开始的纳秒。 func GetNowNanoSec() int64 { return time.Now().UnixNano() } // GetNowGMT 获取当前时间，格式形如"Mon, 02 Jan 2006 15:04:05 GMT"，HTTP中使用的时间格式 func GetNowGMT() string { return time.Now().UTC().Format(http.TimeFormat) } // FileChunk 文件片定义 type FileChunk struct { Number int // 块序号 Offset int64 // 块在文件中的偏移量 Size int64 // 块大小 } // SplitFileByPartNum Split big file to part by the num of part // 按指定的块数分割文件。返回值FileChunk为分割结果，error为nil时有效。 func SplitFileByPartNum(fileName string, chunkNum int) ([]FileChunk, error) { if chunkNum <= 0 || chunkNum > 10000 { return nil, errors.New("chunkNum invalid") } file, err := os.Open(fileName) if err != nil { return nil, err } defer file.Close() stat, err := file.Stat() if err != nil { return nil, err } if int64(chunkNum) > stat.Size() { return nil, errors.New("oss: chunkNum invalid") } var chunks []FileChunk var chunk = FileChunk{} var chunkN = (int64)(chunkNum) for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * (stat.Size() / chunkN) if i == chunkN-1 { chunk.Size = stat.Size()/chunkN + stat.Size()%chunkN } else { chunk.Size = stat.Size() / chunkN } chunks = append(chunks, chunk) } return chunks, nil } // SplitFileByPartSize Split big file to part by the size of part // 按块大小分割文件。返回值FileChunk为分割结果，error为nil时有效。 func SplitFileByPartSize(fileName string, chunkSize int64) ([]FileChunk, error) { if chunkSize <= 0 { return nil, errors.New("chunkSize invalid") } file, err := os.Open(fileName) if err != nil { return nil, err } defer file.Close() stat, err := file.Stat() if err != nil { return nil, err } var chunkN = stat.Size() / chunkSize if chunkN >= 10000 { return nil, errors.New("Too many parts, please increase part size.") } var chunks []FileChunk var chunk = FileChunk{} for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * chunkSize chunk.Size = chunkSize chunks = append(chunks, chunk) } if stat.Size()%chunkSize > 0 { chunk.Number = len(chunks) + 1 chunk.Offset = int64(len(chunks)) * chunkSize chunk.Size = stat.Size() % chunkSize chunks = append(chunks, chunk) } return chunks, nil } // GetPartEnd 计算结束位置 func GetPartEnd(begin int64, total int64, per int64) int64 { if begin+per > total { return total - 1 } return begin + per - 1 } // crcTable returns the Table constructed from the specified polynomial var crcTable = func() *crc64.Table { return crc64.MakeTable(crc64.ECMA) } aliyun-oss-go-sdk-1.5.0/oss/utils_test.go000066400000000000000000000144171311621456000203420ustar00rootroot00000000000000package oss import . "gopkg.in/check.v1" type OssUtilsSuite struct{} var _ = Suite(&OssUtilsSuite{}) func (s *OssUtilsSuite) TestUtilsTime(c *C) { c.Assert(GetNowSec() > 1448597674, Equals, true) c.Assert(GetNowNanoSec() > 1448597674000000000, Equals, true) c.Assert(len(GetNowGMT()), Equals, len("Fri, 27 Nov 2015 04:14:34 GMT")) } func (s *OssUtilsSuite) TestUtilsSplitFile(c *C) { localFile := "../sample/BingWallpaper-2015-11-07.jpg" // Num parts, err := SplitFileByPartNum(localFile, 4) c.Assert(err, IsNil) c.Assert(len(parts), Equals, 4) testLogger.Println("parts 4:", parts) for i, part := range parts { c.Assert(part.Number, Equals, i+1) c.Assert(part.Offset, Equals, int64(i*120512)) c.Assert(part.Size, Equals, int64(120512)) } parts, err = SplitFileByPartNum(localFile, 5) c.Assert(err, IsNil) c.Assert(len(parts), Equals, 5) testLogger.Println("parts 5:", parts) for i, part := range parts { c.Assert(part.Number, Equals, i+1) c.Assert(part.Offset, Equals, int64(i*96409)) } _, err = SplitFileByPartNum(localFile, 10001) c.Assert(err, NotNil) _, err = SplitFileByPartNum(localFile, 0) c.Assert(err, NotNil) _, err = SplitFileByPartNum(localFile, -1) c.Assert(err, NotNil) _, err = SplitFileByPartNum("notexist", 1024) c.Assert(err, NotNil) // Size parts, err = SplitFileByPartSize(localFile, 120512) c.Assert(err, IsNil) c.Assert(len(parts), Equals, 4) testLogger.Println("parts 4:", parts) for i, part := range parts { c.Assert(part.Number, Equals, i+1) c.Assert(part.Offset, Equals, int64(i*120512)) c.Assert(part.Size, Equals, int64(120512)) } parts, err = SplitFileByPartSize(localFile, 96409) c.Assert(err, IsNil) c.Assert(len(parts), Equals, 6) testLogger.Println("parts 6:", parts) for i, part := range parts { c.Assert(part.Number, Equals, i+1) c.Assert(part.Offset, Equals, int64(i*96409)) } _, err = SplitFileByPartSize(localFile, 0) c.Assert(err, NotNil) _, err = SplitFileByPartSize(localFile, -1) c.Assert(err, NotNil) _, err = SplitFileByPartSize(localFile, 10) c.Assert(err, NotNil) _, err = SplitFileByPartSize("noexist", 120512) c.Assert(err, NotNil) } func (s *OssUtilsSuite) TestUtilsFileExt(c *C) { c.Assert(TypeByExtension("test.txt"), Equals, "text/plain; charset=utf-8") c.Assert(TypeByExtension("test.jpg"), Equals, "image/jpeg") c.Assert(TypeByExtension("test.pdf"), Equals, "application/pdf") c.Assert(TypeByExtension("test"), Equals, "") c.Assert(TypeByExtension("/root/dir/test.txt"), Equals, "text/plain; charset=utf-8") c.Assert(TypeByExtension("root/dir/test.txt"), Equals, "text/plain; charset=utf-8") c.Assert(TypeByExtension("root\\dir\\test.txt"), Equals, "text/plain; charset=utf-8") c.Assert(TypeByExtension("D:\\work\\dir\\test.txt"), Equals, "text/plain; charset=utf-8") } func (s *OssUtilsSuite) TestGetPartEnd(c *C) { end := GetPartEnd(3, 10, 3) c.Assert(end, Equals, int64(5)) end = GetPartEnd(9, 10, 3) c.Assert(end, Equals, int64(9)) end = GetPartEnd(7, 10, 3) c.Assert(end, Equals, int64(9)) } func (s *OssUtilsSuite) TestParseRange(c *C) { // InvalidRange bytes==M-N _, err := parseRange("bytes==M-N") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "InvalidRange bytes==M-N") // InvalidRange ranges=M-N _, err = parseRange("ranges=M-N") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "InvalidRange ranges=M-N") // InvalidRange ranges=M-N _, err = parseRange("bytes=M-N") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "InvalidRange bytes=M-N") // InvalidRange ranges=M- _, err = parseRange("bytes=M-") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "InvalidRange bytes=M-") // InvalidRange ranges=-N _, err = parseRange("bytes=-N") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "InvalidRange bytes=-N") // InvalidRange ranges=-0 _, err = parseRange("bytes=-0") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "InvalidRange bytes=-0") // InvalidRange bytes=1-2-3 _, err = parseRange("bytes=1-2-3") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "InvalidRange bytes=1-2-3") // InvalidRange bytes=1-N _, err = parseRange("bytes=1-N") c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "InvalidRange bytes=1-N") // ranges=M-N ur, err := parseRange("bytes=1024-4096") c.Assert(err, IsNil) c.Assert(ur.start, Equals, (int64)(1024)) c.Assert(ur.end, Equals, (int64)(4096)) c.Assert(ur.hasStart, Equals, true) c.Assert(ur.hasEnd, Equals, true) // ranges=M-N,X-Y ur, err = parseRange("bytes=1024-4096,2048-4096") c.Assert(err, IsNil) c.Assert(ur.start, Equals, (int64)(1024)) c.Assert(ur.end, Equals, (int64)(4096)) c.Assert(ur.hasStart, Equals, true) c.Assert(ur.hasEnd, Equals, true) // ranges=M- ur, err = parseRange("bytes=1024-") c.Assert(err, IsNil) c.Assert(ur.start, Equals, (int64)(1024)) c.Assert(ur.end, Equals, (int64)(0)) c.Assert(ur.hasStart, Equals, true) c.Assert(ur.hasEnd, Equals, false) // ranges=-N ur, err = parseRange("bytes=-4096") c.Assert(err, IsNil) c.Assert(ur.start, Equals, (int64)(0)) c.Assert(ur.end, Equals, (int64)(4096)) c.Assert(ur.hasStart, Equals, false) c.Assert(ur.hasEnd, Equals, true) } func (s *OssUtilsSuite) TestAdjustRange(c *C) { // nil start, end := adjustRange(nil, 8192) c.Assert(start, Equals, (int64)(0)) c.Assert(end, Equals, (int64)(8192)) // 1024-4096 ur := &unpackedRange{true, true, 1024, 4095} start, end = adjustRange(ur, 8192) c.Assert(start, Equals, (int64)(1024)) c.Assert(end, Equals, (int64)(4096)) // 1024- ur = &unpackedRange{true, false, 1024, 4096} start, end = adjustRange(ur, 8192) c.Assert(start, Equals, (int64)(1024)) c.Assert(end, Equals, (int64)(8192)) // -4096 ur = &unpackedRange{false, true, 1024, 4096} start, end = adjustRange(ur, 8192) c.Assert(start, Equals, (int64)(4096)) c.Assert(end, Equals, (int64)(8192)) // Invalid range 4096-1024 ur = &unpackedRange{true, true, 4096, 1024} start, end = adjustRange(ur, 8192) c.Assert(start, Equals, (int64)(0)) c.Assert(end, Equals, (int64)(8192)) // Invalid range -1- ur = &unpackedRange{true, false, -1, 0} start, end = adjustRange(ur, 8192) c.Assert(start, Equals, (int64)(0)) c.Assert(end, Equals, (int64)(8192)) // Invalid range -9999 ur = &unpackedRange{false, true, 0, 9999} start, end = adjustRange(ur, 8192) c.Assert(start, Equals, (int64)(0)) c.Assert(end, Equals, (int64)(8192)) } aliyun-oss-go-sdk-1.5.0/sample.go000066400000000000000000000011141311621456000166060ustar00rootroot00000000000000// main of samples package main import ( "fmt" "sample" ) func main() { sample.CreateBucketSample() sample.NewBucketSample() sample.ListBucketsSample() sample.BucketACLSample() sample.BucketLifecycleSample() sample.BucketRefererSample() sample.BucketLoggingSample() sample.BucketCORSSample() sample.ObjectACLSample() sample.ObjectMetaSample() sample.ListObjectsSample() sample.DeleteObjectSample() sample.AppendObjectSample() sample.CopyObjectSample() sample.PutObjectSample() sample.GetObjectSample() sample.CnameSample() fmt.Println("All samples completed") } aliyun-oss-go-sdk-1.5.0/sample/000077500000000000000000000000001311621456000162625ustar00rootroot00000000000000aliyun-oss-go-sdk-1.5.0/sample/append_object.go000066400000000000000000000067721311621456000214220ustar00rootroot00000000000000// Package sample examples package sample import ( "bytes" "fmt" "io/ioutil" "os" "strconv" "strings" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // AppendObjectSample 展示了追加上传的用法 func AppendObjectSample() { // 创建Bucket bucket, err := GetTestBucket(bucketName) if err != nil { HandleError(err) } err = bucket.DeleteObject(objectKey) var str = "弃我去者，昨日之日不可留。 乱我心者，今日之日多烦忧！" var nextPos int64 // 场景1：追加字符串到object // 第一次追加的位置是0，返回值为下一次追加的位置 nextPos, err = bucket.AppendObject(objectKey, strings.NewReader(str), nextPos) if err != nil { HandleError(err) } // 第二次追加 nextPos, err = bucket.AppendObject(objectKey, strings.NewReader(str), nextPos) if err != nil { HandleError(err) } // 下载 body, err := bucket.GetObject(objectKey) if err != nil { HandleError(err) } data, err := ioutil.ReadAll(body) body.Close() if err != nil { HandleError(err) } fmt.Println(objectKey, ":", string(data)) err = bucket.DeleteObject(objectKey) if err != nil { HandleError(err) } // 场景2：追加[]byte到object nextPos = 0 // 第一次追加的位置是0，返回值为下一次追加的位置 nextPos, err = bucket.AppendObject(objectKey, bytes.NewReader([]byte(str)), nextPos) if err != nil { HandleError(err) } // 第二次追加 nextPos, err = bucket.AppendObject(objectKey, bytes.NewReader([]byte(str)), nextPos) if err != nil { HandleError(err) } // 下载 body, err = bucket.GetObject(objectKey) if err != nil { HandleError(err) } data, err = ioutil.ReadAll(body) body.Close() if err != nil { HandleError(err) } fmt.Println(objectKey, ":", string(data)) err = bucket.DeleteObject(objectKey) if err != nil { HandleError(err) } //场景3：本地文件追加到Object fd, err := os.Open(localFile) if err != nil { HandleError(err) } defer fd.Close() nextPos = 0 nextPos, err = bucket.AppendObject(objectKey, fd, nextPos) if err != nil { HandleError(err) } // 场景4，您可以通过GetObjectDetailedMeta获取下次追加的位置 props, err := bucket.GetObjectDetailedMeta(objectKey) nextPos, err = strconv.ParseInt(props.Get(oss.HTTPHeaderOssNextAppendPosition), 10, 0) if err != nil { HandleError(err) } nextPos, err = bucket.AppendObject(objectKey, strings.NewReader(str), nextPos) if err != nil { HandleError(err) } err = bucket.DeleteObject(objectKey) if err != nil { HandleError(err) } // 场景5：第一次追加操作时，可以指定Object的Properties，包括以"x-oss-meta-my"为前缀的用户自定义属性 options := []oss.Option{ oss.Expires(futureDate), oss.ObjectACL(oss.ACLPublicRead), oss.Meta("myprop", "mypropval")} nextPos = 0 fd.Seek(0, os.SEEK_SET) nextPos, err = bucket.AppendObject(objectKey, strings.NewReader(str), nextPos, options...) if err != nil { HandleError(err) } // 第二次追加 fd.Seek(0, os.SEEK_SET) nextPos, err = bucket.AppendObject(objectKey, strings.NewReader(str), nextPos) if err != nil { HandleError(err) } props, err = bucket.GetObjectDetailedMeta(objectKey) if err != nil { HandleError(err) } fmt.Println("myprop:", props.Get("x-oss-meta-myprop")) goar, err := bucket.GetObjectACL(objectKey) if err != nil { HandleError(err) } fmt.Println("Object ACL:", goar.ACL) // 删除object和bucket err = DeleteTestBucketAndObject(bucketName) if err != nil { HandleError(err) } fmt.Println("AppendObjectSample completed") } aliyun-oss-go-sdk-1.5.0/sample/bucket_acl.go000066400000000000000000000015671311621456000207160ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // BucketACLSample 展示了如何读取/设置存储空间的权限(Bucket ACL) func BucketACLSample() { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { HandleError(err) } // 使用默认参数创建bucket err = client.CreateBucket(bucketName) if err != nil { HandleError(err) } // 场景：设置Bucket ACL，可选权限有ACLPrivate、ACLPublicRead、ACLPublicReadWrite err = client.SetBucketACL(bucketName, oss.ACLPublicRead) if err != nil { HandleError(err) } // 查看Bucket ACL gbar, err := client.GetBucketACL(bucketName) if err != nil { HandleError(err) } fmt.Println("Bucket ACL:", gbar.ACL) // 删除bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } fmt.Println("BucketACLSample completed") } aliyun-oss-go-sdk-1.5.0/sample/bucket_cors.go000066400000000000000000000031521311621456000211150ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // BucketCORSSample 展示了如何设置/读取/清除存储空间的跨域访问(Bucket CORS) func BucketCORSSample() { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { HandleError(err) } // 使用默认参数创建bucket err = client.CreateBucket(bucketName) if err != nil { HandleError(err) } rule1 := oss.CORSRule{ AllowedOrigin: []string{"*"}, AllowedMethod: []string{"PUT", "GET", "POST"}, AllowedHeader: []string{}, ExposeHeader: []string{}, MaxAgeSeconds: 100, } rule2 := oss.CORSRule{ AllowedOrigin: []string{"http://www.a.com", "http://www.b.com"}, AllowedMethod: []string{"GET"}, AllowedHeader: []string{"Authorization"}, ExposeHeader: []string{"x-oss-test", "x-oss-test1"}, MaxAgeSeconds: 100, } // 场景1：设置Bucket的CORS规则 err = client.SetBucketCORS(bucketName, []oss.CORSRule{rule1}) if err != nil { HandleError(err) } // 场景2：设置Bucket的CORS规则，如果该Bucket上已经设置了CORS规则，则会覆盖。 err = client.SetBucketCORS(bucketName, []oss.CORSRule{rule1, rule2}) if err != nil { HandleError(err) } // 获取Bucket上设置的CORS gbl, err := client.GetBucketCORS(bucketName) if err != nil { HandleError(err) } fmt.Println("Bucket CORS:", gbl.CORSRules) // 删除Bucket上的CORS设置 err = client.DeleteBucketCORS(bucketName) if err != nil { HandleError(err) } // 删除bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } fmt.Println("BucketCORSSample completed") } aliyun-oss-go-sdk-1.5.0/sample/bucket_lifecycle.go000066400000000000000000000037061311621456000221130ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // BucketLifecycleSample 展示了如何设置/读取/清除存储空间中文件的生命周期(Bucket Lifecycle) func BucketLifecycleSample() { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { HandleError(err) } // 使用默认参数创建bucket err = client.CreateBucket(bucketName) if err != nil { HandleError(err) } // 场景1：设置Lifecycle，其中规则的id是id1，规则生效的object前缀是one，符合的Object绝对过期时间2015/11/11 var rule1 = oss.BuildLifecycleRuleByDate("id1", "one", true, 2015, 11, 11) var rules = []oss.LifecycleRule{rule1} err = client.SetBucketLifecycle(bucketName, rules) if err != nil { HandleError(err) } // 场景2：设置Lifecycle，其中规则的id是id2，规则生效的object前缀是two，符合的Object相对过期时间是3天后 var rule2 = oss.BuildLifecycleRuleByDays("id2", "two", true, 3) rules = []oss.LifecycleRule{rule2} err = client.SetBucketLifecycle(bucketName, rules) if err != nil { HandleError(err) } // 场景3：在Bucket上同时设置两条规格，两个规则分别作用与不同的对象。规则id相同是会覆盖老的规则。 var rule3 = oss.BuildLifecycleRuleByDays("id1", "two", true, 365) var rule4 = oss.BuildLifecycleRuleByDate("id2", "one", true, 2016, 11, 11) rules = []oss.LifecycleRule{rule3, rule4} err = client.SetBucketLifecycle(bucketName, rules) if err != nil { HandleError(err) } // 获取Bucket上设置的Lifecycle gbl, err := client.GetBucketLifecycle(bucketName) if err != nil { HandleError(err) } fmt.Println("Bucket Lifecycle:", gbl.Rules) // 删除Bucket上的Lifecycle设置 err = client.DeleteBucketLifecycle(bucketName) if err != nil { HandleError(err) } // 删除bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } fmt.Println("BucketLifecycleSample completed") } aliyun-oss-go-sdk-1.5.0/sample/bucket_logging.go000066400000000000000000000042421311621456000215760ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // BucketLoggingSample 展示了如何设置/读取/清除存储空间的日志(Bucket Logging) func BucketLoggingSample() { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { HandleError(err) } // 创建bucket err = client.CreateBucket(bucketName) if err != nil { HandleError(err) } // 创建Target bucket，存储访问日志 var targetBucketName = "target-bucket" err = client.CreateBucket(targetBucketName) if err != nil { HandleError(err) } // 场景1：设置Logging，bucketName中以"prefix"为前缀的object的访问日志将被记录到targetBucketName err = client.SetBucketLogging(bucketName, targetBucketName, "prefix-1", true) if err != nil { HandleError(err) } // 场景2：设置Logging，bucketName中以"prefix"为前缀的object的访问日志将被记录到bucketName // 注意：相同bucket，相同prefix，多次设置后者会覆盖前者 err = client.SetBucketLogging(bucketName, bucketName, "prefix-2", true) if err != nil { HandleError(err) } // 删除Bucket上的Logging设置 err = client.DeleteBucketLogging(bucketName) if err != nil { HandleError(err) } // 场景3：设置但不生效 err = client.SetBucketLogging(bucketName, targetBucketName, "prefix-3", false) if err != nil { HandleError(err) } // 获取Bucket上设置的Logging gbl, err := client.GetBucketLogging(bucketName) if err != nil { HandleError(err) } fmt.Println("Bucket Logging:", gbl.LoggingEnabled) err = client.SetBucketLogging(bucketName, bucketName, "prefix2", true) if err != nil { HandleError(err) } // 获取Bucket上设置的Logging gbl, err = client.GetBucketLogging(bucketName) if err != nil { HandleError(err) } fmt.Println("Bucket Logging:", gbl.LoggingEnabled) // 删除Bucket上的Logging设置 err = client.DeleteBucketLogging(bucketName) if err != nil { HandleError(err) } // 删除bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } err = client.DeleteBucket(targetBucketName) if err != nil { HandleError(err) } fmt.Println("BucketLoggingSample completed") } aliyun-oss-go-sdk-1.5.0/sample/bucket_referer.go000066400000000000000000000023431311621456000216020ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // BucketRefererSample 展示了如何设置/读取/清除存储空间的白名单(Bucket Referer) func BucketRefererSample() { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { HandleError(err) } // 使用默认参数创建bucket err = client.CreateBucket(bucketName) if err != nil { HandleError(err) } var referers = []string{ "http://www.aliyun.com", "http://www.???.aliyuncs.com", "http://www.*.com", } // 场景1：设置referers，referer中支持?和*，分布代替一个或多个字符 err = client.SetBucketReferer(bucketName, referers, false) if err != nil { HandleError(err) } // 场景2：清空referers referers = []string{} err = client.SetBucketReferer(bucketName, referers, true) if err != nil { HandleError(err) } // 获取Bucket上设置的Lifecycle gbr, err := client.GetBucketReferer(bucketName) if err != nil { HandleError(err) } fmt.Println("Bucket Referers:", gbr.RefererList, "AllowEmptyReferer:", gbr.AllowEmptyReferer) // 删除bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } fmt.Println("BucketRefererSample completed") } aliyun-oss-go-sdk-1.5.0/sample/cname_sample.go000066400000000000000000000036051311621456000212410ustar00rootroot00000000000000package sample import ( "fmt" "io/ioutil" "strings" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // CnameSample 展示了Cname的用法 func CnameSample() { // NewClient client, err := oss.New(endpoint4Cname, accessID4Cname, accessKey4Cname, oss.UseCname(true)) if err != nil { HandleError(err) } // CreateBucket err = client.CreateBucket(bucketName4Cname) if err != nil { HandleError(err) } // SetBucketACL err = client.SetBucketACL(bucketName4Cname, oss.ACLPrivate) if err != nil { HandleError(err) } // 查看Bucket ACL gbar, err := client.GetBucketACL(bucketName4Cname) if err != nil { HandleError(err) } fmt.Println("Bucket ACL:", gbar.ACL) // ListBuckets， cname用户不能使用该操作 _, err = client.ListBuckets() if err == nil { HandleError(err) } bucket, err := client.Bucket(bucketName4Cname) if err != nil { HandleError(err) } objectValue := "长忆观潮，满郭人争江上望。来疑沧海尽成空，万面鼓声中。弄潮儿向涛头立，手把红旗旗不湿。别来几向梦中看，梦觉尚心寒。" // PutObject err = bucket.PutObject(objectKey, strings.NewReader(objectValue)) if err != nil { HandleError(err) } // GetObject body, err := bucket.GetObject(objectKey) if err != nil { HandleError(err) } data, err := ioutil.ReadAll(body) body.Close() if err != nil { HandleError(err) } fmt.Println(objectKey, ":", string(data)) // PutObjectFromFile err = bucket.PutObjectFromFile(objectKey, localFile) if err != nil { HandleError(err) } // GetObjectToFile err = bucket.GetObjectToFile(objectKey, newPicName) if err != nil { HandleError(err) } // ListObjects lor, err := bucket.ListObjects() if err != nil { HandleError(err) } fmt.Println("objects:", lor.Objects) // DeleteObject err = bucket.DeleteObject(objectKey) if err != nil { HandleError(err) } fmt.Println("CnameSample completed") } aliyun-oss-go-sdk-1.5.0/sample/comm.go000066400000000000000000000050501311621456000175440ustar00rootroot00000000000000package sample import ( "fmt" "os" "strings" "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) var ( pastDate = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) futureDate = time.Date(2049, time.January, 10, 23, 0, 0, 0, time.UTC) ) // HandleError sample中的错误处理 func HandleError(err error) { fmt.Println("occurred error:", err) os.Exit(-1) } // GetTestBucket 创建sample的Bucket并返回OssBucket对象，该函数为了简化sample，让sample代码更明了 func GetTestBucket(bucketName string) (*oss.Bucket, error) { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { return nil, err } // Create Bucket err = client.CreateBucket(bucketName) if err != nil { return nil, err } // Get Bucket bucket, err := client.Bucket(bucketName) if err != nil { return nil, err } return bucket, nil } // DeleteTestBucketAndObject 删除sample的object和bucket，该函数为了简化sample，让sample代码更明了 func DeleteTestBucketAndObject(bucketName string) error { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { return err } // Get Bucket bucket, err := client.Bucket(bucketName) if err != nil { return err } // Delete Part lmur, err := bucket.ListMultipartUploads() if err != nil { return err } for _, upload := range lmur.Uploads { var imur = oss.InitiateMultipartUploadResult{Bucket: bucket.BucketName, Key: upload.Key, UploadID: upload.UploadID} err = bucket.AbortMultipartUpload(imur) if err != nil { return err } } // Delete Objects lor, err := bucket.ListObjects() if err != nil { return err } for _, object := range lor.Objects { err = bucket.DeleteObject(object.Key) if err != nil { return err } } // Delete Bucket err = client.DeleteBucket(bucketName) if err != nil { return err } return nil } // Object pair of key and value type Object struct { Key string Value string } // CreateObjects 创建一组对象，该函数为了简化sample，让sample代码更明了 func CreateObjects(bucket *oss.Bucket, objects []Object) error { for _, object := range objects { err := bucket.PutObject(object.Key, strings.NewReader(object.Value)) if err != nil { return err } } return nil } // DeleteObjects 删除sample的object和bucket，该函数为了简化sample，让sample代码更明了 func DeleteObjects(bucket *oss.Bucket, objects []Object) error { for _, object := range objects { err := bucket.DeleteObject(object.Key) if err != nil { return err } } return nil } aliyun-oss-go-sdk-1.5.0/sample/config.go000066400000000000000000000020051311621456000200530ustar00rootroot00000000000000package sample const ( // sample运行的环境配置。如果您需要运行sample，请先修成您的配置。 endpoint string = "" accessID string = "" accessKey string = "" bucketName string = "" // 运行cname的示例程序sample/cname_sample的示例程序的配置。 // 如果您需要运行sample/cname_sample，请先修成您的配置。 endpoint4Cname string = "" accessID4Cname string = "" accessKey4Cname string = "" bucketName4Cname string = "" // 运行sample时的Object名称 objectKey string = "my-object" // 运行sample需要的资源，即sample目录目录下的BingWallpaper-2015-11-07.jpg // 和The Go Programming Language.html，请根据实际情况修改 localFile string = "src/sample/BingWallpaper-2015-11-07.jpg" htmlLocalFile string = "src/sample/The Go Programming Language.html" newPicName string = "src/sample/NewBingWallpaper-2015-11-07.jpg" ) aliyun-oss-go-sdk-1.5.0/sample/copy_object.go000066400000000000000000000065061311621456000211200ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // CopyObjectSample 展示了拷贝文件的用法 func CopyObjectSample() { // 创建Bucket bucket, err := GetTestBucket(bucketName) if err != nil { HandleError(err) } // 创建一个Object err = bucket.PutObjectFromFile(objectKey, localFile) if err != nil { HandleError(err) } // 场景1：把已经存在的对象copy成一个新对象 var descObjectKey = "descobject" _, err = bucket.CopyObject(objectKey, descObjectKey) if err != nil { HandleError(err) } // 场景2：把已经存在的对象copy成一个新对象，目标对象存在时，会覆盖 _, err = bucket.CopyObject(objectKey, descObjectKey) if err != nil { HandleError(err) } err = bucket.DeleteObject(descObjectKey) if err != nil { HandleError(err) } // 场景3：对象copy时对源对象执行约束条件，满足时候copy，不满足时返回错误，不执行copy // 约束条件不满足，copy没有执行 _, err = bucket.CopyObject(objectKey, descObjectKey, oss.CopySourceIfModifiedSince(futureDate)) if err == nil { HandleError(err) } fmt.Println("CopyObjectError:", err) // 约束条件满足，copy执行 _, err = bucket.CopyObject(objectKey, descObjectKey, oss.CopySourceIfUnmodifiedSince(futureDate)) if err != nil { HandleError(err) } // 场景4：对象copy时，可以指定目标对象的Properties，同时一定要指定MetadataDirective为MetaReplace options := []oss.Option{ oss.Expires(futureDate), oss.Meta("myprop", "mypropval"), oss.MetadataDirective(oss.MetaReplace)} _, err = bucket.CopyObject(objectKey, descObjectKey, options...) if err != nil { HandleError(err) } meta, err := bucket.GetObjectDetailedMeta(descObjectKey) if err != nil { HandleError(err) } fmt.Println("meta:", meta) // 场景5：当源对象和目标对象相同时，目的是用来修改源对象的meta options = []oss.Option{ oss.Expires(futureDate), oss.Meta("myprop", "mypropval"), oss.MetadataDirective(oss.MetaReplace)} _, err = bucket.CopyObject(objectKey, objectKey, options...) if err != nil { HandleError(err) } fmt.Println("meta:", meta) // 场景6：大文件分片拷贝，支持并发、断点续传功能。 // 分片上传，分片大小为100K。默认使用不使用并发上传，不使用断点续传。 err = bucket.CopyFile(bucketName, objectKey, descObjectKey, 100*1024) if err != nil { HandleError(err) } // 分片大小为100K，3个协程并发拷贝。 err = bucket.CopyFile(bucketName, objectKey, descObjectKey, 100*1024, oss.Routines(3)) if err != nil { HandleError(err) } // 分片大小为100K，3个协程并发拷贝，使用断点续传拷贝文件。 err = bucket.CopyFile(bucketName, objectKey, descObjectKey, 100*1024, oss.Routines(3), oss.Checkpoint(true, "")) if err != nil { HandleError(err) } // 断点续传功能需要使用本地文件，记录哪些分片已经上传。该文件路径可以Checkpoint的第二个参数指定，如果为空，则为当前目录下的{descObjectKey}.cp。 err = bucket.CopyFile(bucketName, objectKey, descObjectKey, 100*1024, oss.Checkpoint(true, localFile+".cp")) if err != nil { HandleError(err) } // 删除object和bucket err = DeleteTestBucketAndObject(bucketName) if err != nil { HandleError(err) } fmt.Println("CopyObjectSample completed") } aliyun-oss-go-sdk-1.5.0/sample/create_bucket.go000066400000000000000000000020061311621456000214070ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // CreateBucketSample 展示了如何创建存储空间 func CreateBucketSample() { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { HandleError(err) } DeleteTestBucketAndObject(bucketName) // 场景1：使用默认参数创建bucket err = client.CreateBucket(bucketName) if err != nil { HandleError(err) } // 删除bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } // 场景2：创建bucket时指定其权限 err = client.CreateBucket(bucketName, oss.ACL(oss.ACLPublicRead)) if err != nil { HandleError(err) } // 场景3：重复创建OSS不会报错，但是不做任何操作，指定的ACL无效 err = client.CreateBucket(bucketName, oss.ACL(oss.ACLPublicReadWrite)) if err != nil { HandleError(err) } // 删除bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } fmt.Println("CreateBucketSample completed") } aliyun-oss-go-sdk-1.5.0/sample/delete_object.go000066400000000000000000000050171311621456000214040ustar00rootroot00000000000000package sample import ( "fmt" "strings" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // DeleteObjectSample 展示了删除单个文件、批量删除文件的方法 func DeleteObjectSample() { // 创建Bucket bucket, err := GetTestBucket(bucketName) if err != nil { HandleError(err) } var val = "抽刀断水水更流，举杯销愁愁更愁。 人生在世不称意，明朝散发弄扁舟。" // 场景1：删除Object err = bucket.PutObject(objectKey, strings.NewReader(val)) if err != nil { HandleError(err) } err = bucket.DeleteObject(objectKey) if err != nil { HandleError(err) } // 场景2：删除多个Object err = bucket.PutObject(objectKey+"1", strings.NewReader(val)) if err != nil { HandleError(err) } err = bucket.PutObject(objectKey+"2", strings.NewReader(val)) if err != nil { HandleError(err) } delRes, err := bucket.DeleteObjects([]string{objectKey + "1", objectKey + "2"}) if err != nil { HandleError(err) } fmt.Println("Del Res:", delRes) lsRes, err := bucket.ListObjects() if err != nil { HandleError(err) } fmt.Println("Objects:", getObjectsFormResponse(lsRes)) // 场景3：删除多个Object，详细模式时返回的结果中会包含成功删除的Object，默认该模式 err = bucket.PutObject(objectKey+"1", strings.NewReader(val)) if err != nil { HandleError(err) } err = bucket.PutObject(objectKey+"2", strings.NewReader(val)) if err != nil { HandleError(err) } delRes, err = bucket.DeleteObjects([]string{objectKey + "1", objectKey + "2"}, oss.DeleteObjectsQuiet(false)) if err != nil { HandleError(err) } fmt.Println("Detail Del Res:", delRes) lsRes, err = bucket.ListObjects() if err != nil { HandleError(err) } fmt.Println("Objects:", getObjectsFormResponse(lsRes)) // 场景4：删除多个Object，简单模式返回的消息体中只包含删除出错的Object结果 err = bucket.PutObject(objectKey+"1", strings.NewReader(val)) if err != nil { HandleError(err) } err = bucket.PutObject(objectKey+"2", strings.NewReader(val)) if err != nil { HandleError(err) } delRes, err = bucket.DeleteObjects([]string{objectKey + "1", objectKey + "2"}, oss.DeleteObjectsQuiet(true)) if err != nil { HandleError(err) } fmt.Println("Sample Del Res:", delRes) lsRes, err = bucket.ListObjects() if err != nil { HandleError(err) } fmt.Println("Objects:", getObjectsFormResponse(lsRes)) // 删除object和bucket err = DeleteTestBucketAndObject(bucketName) if err != nil { HandleError(err) } fmt.Println("DeleteObjectSample completed") } aliyun-oss-go-sdk-1.5.0/sample/get_object.go000066400000000000000000000073771311621456000207340ustar00rootroot00000000000000package sample import ( "bytes" "fmt" "io" "io/ioutil" "os" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // GetObjectSample 展示了流式下载、范围下载、断点续传下载的用法 func GetObjectSample() { // 创建Bucket bucket, err := GetTestBucket(bucketName) if err != nil { HandleError(err) } // 上传对象 err = bucket.PutObjectFromFile(objectKey, localFile) if err != nil { HandleError(err) } // 场景1：下载object存储到ReadCloser，注意需要Close。 body, err := bucket.GetObject(objectKey) if err != nil { HandleError(err) } data, err := ioutil.ReadAll(body) body.Close() if err != nil { HandleError(err) } data = data // use data // 场景2：下载object存储到bytes数组，适合小对象。 buf := new(bytes.Buffer) body, err = bucket.GetObject(objectKey) if err != nil { HandleError(err) } io.Copy(buf, body) body.Close() // 场景3：下载object存储到本地文件，用户打开文件传入句柄。 fd, err := os.OpenFile("mynewfile-1.jpg", os.O_WRONLY|os.O_CREATE, 0660) if err != nil { HandleError(err) } defer fd.Close() body, err = bucket.GetObject(objectKey) if err != nil { HandleError(err) } io.Copy(fd, body) body.Close() // 场景4：下载object存储到本地文件。 err = bucket.GetObjectToFile(objectKey, "mynewfile-2.jpg") if err != nil { HandleError(err) } // 场景5：满足约束条件下载，否则返回错误。GetObject/GetObjectToFile/DownloadFile都支持该功能。 // 修改时间，约束条件满足，执行下载。 body, err = bucket.GetObject(objectKey, oss.IfModifiedSince(pastDate)) if err != nil { HandleError(err) } body.Close() // 修改时间，约束条件不满足，不执行下载。 _, err = bucket.GetObject(objectKey, oss.IfUnmodifiedSince(pastDate)) if err == nil { HandleError(err) } meta, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { HandleError(err) } etag := meta.Get(oss.HTTPHeaderEtag) // 校验内容，约束条件满足，执行下载。 body, err = bucket.GetObject(objectKey, oss.IfMatch(etag)) if err != nil { HandleError(err) } body.Close() // 校验内容，约束条件不满足，不执行下载。 body, err = bucket.GetObject(objectKey, oss.IfNoneMatch(etag)) if err == nil { HandleError(err) } // 场景6：大文件分片下载，支持并发下载，断点续传功能。 // 分片下载，分片大小为100K。默认使用不使用并发下载，不使用断点续传。 err = bucket.DownloadFile(objectKey, "mynewfile-3.jpg", 100*1024) if err != nil { HandleError(err) } // 分片大小为100K，3个协程并发下载。 err = bucket.DownloadFile(objectKey, "mynewfile-3.jpg", 100*1024, oss.Routines(3)) if err != nil { HandleError(err) } // 分片大小为100K，3个协程并发下载，使用断点续传下载文件。 err = bucket.DownloadFile(objectKey, "mynewfile-3.jpg", 100*1024, oss.Routines(3), oss.Checkpoint(true, "")) if err != nil { HandleError(err) } // 断点续传功能需要使用本地文件，记录哪些分片已经下载。该文件路径可以Checkpoint的第二个参数指定，如果为空，则为下载文件目录。 err = bucket.DownloadFile(objectKey, "mynewfile-3.jpg", 100*1024, oss.Checkpoint(true, "mynewfile.cp")) if err != nil { HandleError(err) } // 场景7：内容进行 GZIP压缩传输的用户。GetObject/GetObjectToFile具有相同功能。 err = bucket.PutObjectFromFile(objectKey, htmlLocalFile) if err != nil { HandleError(err) } err = bucket.GetObjectToFile(objectKey, "myhtml.gzip", oss.AcceptEncoding("gzip")) if err != nil { HandleError(err) } // 删除object和bucket err = DeleteTestBucketAndObject(bucketName) if err != nil { HandleError(err) } fmt.Println("GetObjectSample completed") } aliyun-oss-go-sdk-1.5.0/sample/list_buckets.go000066400000000000000000000054101311621456000213040ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // ListBucketsSample 展示了列举存储空间的用法，包括默认参数列举、指定参数列举 func ListBucketsSample() { var myBuckets = []string{ "my-bucket-1", "my-bucket-11", "my-bucket-2", "my-bucket-21", "my-bucket-22", "my-bucket-3", "my-bucket-31", "my-bucket-32"} // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { HandleError(err) } // remove other bucket lbr, err := client.ListBuckets() if err != nil { HandleError(err) } for _, bucket := range lbr.Buckets { err = client.DeleteBucket(bucket.Name) if err != nil { //HandleError(err) } } // 创建bucket for _, bucketName := range myBuckets { err = client.CreateBucket(bucketName) if err != nil { HandleError(err) } } // 场景1：使用默认参数参数 lbr, err = client.ListBuckets() if err != nil { HandleError(err) } fmt.Println("my buckets:", lbr.Buckets) // 场景2：指定最大返回数量 lbr, err = client.ListBuckets(oss.MaxKeys(3)) if err != nil { HandleError(err) } fmt.Println("my buckets max num:", lbr.Buckets) // 场景3：返回指定前缀的Bucket lbr, err = client.ListBuckets(oss.Prefix("my-bucket-2")) if err != nil { HandleError(err) } fmt.Println("my buckets prefix :", lbr.Buckets) // 场景4：指定从某个之后返回 lbr, err = client.ListBuckets(oss.Marker("my-bucket-22")) if err != nil { HandleError(err) } fmt.Println("my buckets marker :", lbr.Buckets) // 场景5：分页获取所有bucket，每次返回3个 marker := oss.Marker("") for { lbr, err = client.ListBuckets(oss.MaxKeys(3), marker) if err != nil { HandleError(err) } marker = oss.Marker(lbr.NextMarker) fmt.Println("my buckets page :", lbr.Buckets) if !lbr.IsTruncated { break } } // 场景6：分页所有获取从某个之后的bucket，每次返回3个 marker = oss.Marker("my-bucket-22") for { lbr, err = client.ListBuckets(oss.MaxKeys(3), marker) if err != nil { HandleError(err) } marker = oss.Marker(lbr.NextMarker) fmt.Println("my buckets marker&page :", lbr.Buckets) if !lbr.IsTruncated { break } } // 场景7：分页所有获取前缀的bucket，每次返回3个 pre := oss.Prefix("my-bucket-2") marker = oss.Marker("") for { lbr, err = client.ListBuckets(oss.MaxKeys(3), pre, marker) if err != nil { HandleError(err) } pre = oss.Prefix(lbr.Prefix) marker = oss.Marker(lbr.NextMarker) fmt.Println("my buckets prefix&page :", lbr.Buckets) if !lbr.IsTruncated { break } } // 删除bucket for _, bucketName := range myBuckets { err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } } fmt.Println("ListsBucketSample completed") } aliyun-oss-go-sdk-1.5.0/sample/list_objects.go000066400000000000000000000067761311621456000213150ustar00rootroot00000000000000package sample import ( "fmt" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // ListObjectsSample 展示了列举文件的用法，包括默认参数列举、指定参数列举 func ListObjectsSample() { var myObjects = []Object{ {"my-object-1", ""}, {"my-object-11", ""}, {"my-object-2", ""}, {"my-object-21", ""}, {"my-object-22", ""}, {"my-object-3", ""}, {"my-object-31", ""}, {"my-object-32", ""}} // 创建Bucket bucket, err := GetTestBucket(bucketName) if err != nil { HandleError(err) } // 创建object err = CreateObjects(bucket, myObjects) if err != nil { HandleError(err) } // 场景1：使用默认参数参数 lor, err := bucket.ListObjects() if err != nil { HandleError(err) } fmt.Println("my objects:", getObjectsFormResponse(lor)) // 场景2：指定最大返回数量 lor, err = bucket.ListObjects(oss.MaxKeys(3)) if err != nil { HandleError(err) } fmt.Println("my objects max num:", getObjectsFormResponse(lor)) // 场景3：返回指定前缀的Bucket lor, err = bucket.ListObjects(oss.Prefix("my-object-2")) if err != nil { HandleError(err) } fmt.Println("my objects prefix :", getObjectsFormResponse(lor)) // 场景4：指定从某个之后返回 lor, err = bucket.ListObjects(oss.Marker("my-object-22")) if err != nil { HandleError(err) } fmt.Println("my objects marker :", getObjectsFormResponse(lor)) // 场景5：分页获取所有object，每次返回3个 marker := oss.Marker("") for { lor, err = bucket.ListObjects(oss.MaxKeys(3), marker) if err != nil { HandleError(err) } marker = oss.Marker(lor.NextMarker) fmt.Println("my objects page :", getObjectsFormResponse(lor)) if !lor.IsTruncated { break } } // 场景6：分页所有获取从某个之后的object，每次返回3个 marker = oss.Marker("my-object-22") for { lor, err = bucket.ListObjects(oss.MaxKeys(3), marker) if err != nil { HandleError(err) } marker = oss.Marker(lor.NextMarker) fmt.Println("my objects marker&page :", getObjectsFormResponse(lor)) if !lor.IsTruncated { break } } // 场景7：分页所有获取前缀的object，每次返回2个 pre := oss.Prefix("my-object-2") marker = oss.Marker("") for { lor, err = bucket.ListObjects(oss.MaxKeys(2), marker, pre) if err != nil { HandleError(err) } pre = oss.Prefix(lor.Prefix) marker = oss.Marker(lor.NextMarker) fmt.Println("my objects prefix&page :", getObjectsFormResponse(lor)) if !lor.IsTruncated { break } } err = DeleteObjects(bucket, myObjects) if err != nil { HandleError(err) } // 场景8：prefix和delimiter结合，完成分组功能，ListObjectsResponse.Objects表示不再组中， // ListObjectsResponse.CommonPrefixes分组结果 myObjects = []Object{ {"fun/test.txt", ""}, {"fun/test.jpg", ""}, {"fun/movie/001.avi", ""}, {"fun/movie/007.avi", ""}, {"fun/music/001.mp3", ""}, {"fun/music/001.mp3", ""}} // 创建object err = CreateObjects(bucket, myObjects) if err != nil { HandleError(err) } lor, err = bucket.ListObjects(oss.Prefix("fun/"), oss.Delimiter("/")) if err != nil { HandleError(err) } fmt.Println("my objects prefix :", getObjectsFormResponse(lor), "common prefixes:", lor.CommonPrefixes) // 删除object和bucket err = DeleteTestBucketAndObject(bucketName) if err != nil { HandleError(err) } fmt.Println("ListObjectsSample completed") } func getObjectsFormResponse(lor oss.ListObjectsResult) string { var output string for _, object := range lor.Objects { output += object.Key + " " } return output } aliyun-oss-go-sdk-1.5.0/sample/new_bucket.go000066400000000000000000000016131311621456000207400ustar00rootroot00000000000000package sample import ( "fmt" "strings" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // NewBucketSample 展示了如何初始化Client、Bucket func NewBucketSample() { // New Client client, err := oss.New(endpoint, accessID, accessKey) if err != nil { HandleError(err) } // Create Bucket err = client.CreateBucket(bucketName) if err != nil { HandleError(err) } // New Bucket bucket, err := client.Bucket(bucketName) if err != nil { HandleError(err) } // Put Object，上传一个Object var objectName = "myobject" err = bucket.PutObject(objectName, strings.NewReader("MyObjectValue")) if err != nil { HandleError(err) } // Delete Object，删除Object err = bucket.DeleteObject(objectName) if err != nil { HandleError(err) } // 删除bucket err = client.DeleteBucket(bucketName) if err != nil { HandleError(err) } fmt.Println("NewBucketSample completed") } aliyun-oss-go-sdk-1.5.0/sample/object_acl.go000066400000000000000000000017331311621456000207020ustar00rootroot00000000000000package sample import ( "fmt" "strings" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // ObjectACLSample 展示了如何设置、读取文件权限(object acl) func ObjectACLSample() { // 创建Bucket bucket, err := GetTestBucket(bucketName) if err != nil { HandleError(err) } // 创建object err = bucket.PutObject(objectKey, strings.NewReader("YoursObjectValue")) if err != nil { HandleError(err) } // 场景：设置Bucket ACL，可选权限有ACLPrivate、ACLPublicRead、ACLPublicReadWrite err = bucket.SetObjectACL(objectKey, oss.ACLPrivate) if err != nil { HandleError(err) } // 查看Object ACL，返回的权限标识为private、public-read、public-read-write其中之一 goar, err := bucket.GetObjectACL(objectKey) if err != nil { HandleError(err) } fmt.Println("Object ACL:", goar.ACL) // 删除object和bucket err = DeleteTestBucketAndObject(bucketName) if err != nil { HandleError(err) } fmt.Println("ObjectACLSample completed") } aliyun-oss-go-sdk-1.5.0/sample/object_meta.go000066400000000000000000000035011311621456000210640ustar00rootroot00000000000000package sample import ( "fmt" "strings" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // ObjectMetaSample 展示了如何设置、读取文件元数据(object meta) func ObjectMetaSample() { // 创建Bucket bucket, err := GetTestBucket(bucketName) if err != nil { HandleError(err) } // 创建object err = bucket.PutObject(objectKey, strings.NewReader("YoursObjectValue")) if err != nil { HandleError(err) } // 场景：设置Bucket Meta，可以设置一个或多个属性。 // 注意：Meta不区分大小写 options := []oss.Option{ oss.Expires(futureDate), oss.Meta("myprop", "mypropval")} err = bucket.SetObjectMeta(objectKey, options...) if err != nil { HandleError(err) } // 场景1：查看Object的meta，只返回少量基本meta信息，如ETag、Size、LastModified。 props, err := bucket.GetObjectMeta(objectKey) if err != nil { HandleError(err) } fmt.Println("Object Meta:", props) // 场景2：查看Object的所有Meta，包括自定义的meta。 props, err = bucket.GetObjectDetailedMeta(objectKey) if err != nil { HandleError(err) } fmt.Println("Expires:", props.Get("Expires")) // 场景3：查看Object的所有Meta，符合约束条件返回，不符合约束条件保存，包括自定义的meta。 props, err = bucket.GetObjectDetailedMeta(objectKey, oss.IfUnmodifiedSince(futureDate)) if err != nil { HandleError(err) } fmt.Println("MyProp:", props.Get("X-Oss-Meta-Myprop")) _, err = bucket.GetObjectDetailedMeta(objectKey, oss.IfModifiedSince(futureDate)) if err == nil { HandleError(err) } goar, err := bucket.GetObjectACL(objectKey) if err != nil { HandleError(err) } fmt.Println("Object ACL:", goar.ACL) // 删除object和bucket err = DeleteTestBucketAndObject(bucketName) if err != nil { HandleError(err) } fmt.Println("ObjectMetaSample completed") } aliyun-oss-go-sdk-1.5.0/sample/put_object.go000066400000000000000000000052341311621456000207530ustar00rootroot00000000000000package sample import ( "bytes" "fmt" "os" "strings" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) // PutObjectSample 展示了简单上传、断点续传的使用方法 func PutObjectSample() { // 创建Bucket bucket, err := GetTestBucket(bucketName) if err != nil { HandleError(err) } var val = "花间一壶酒，独酌无相亲。 举杯邀明月，对影成三人。" // 场景1：上传object，value是字符串。 err = bucket.PutObject(objectKey, strings.NewReader(val)) if err != nil { HandleError(err) } // 场景2：上传object，value是[]byte。 err = bucket.PutObject(objectKey, bytes.NewReader([]byte(val))) if err != nil { HandleError(err) } // 场景3：上传本地文件，用户打开文件，传入句柄。 fd, err := os.Open(localFile) if err != nil { HandleError(err) } defer fd.Close() err = bucket.PutObject(objectKey, fd) if err != nil { HandleError(err) } // 场景4：上传本地文件，不需要打开文件。 err = bucket.PutObjectFromFile(objectKey, localFile) if err != nil { HandleError(err) } // 场景5：上传object，上传时指定对象属性。PutObject/PutObjectFromFile/UploadFile都支持该功能。 options := []oss.Option{ oss.Expires(futureDate), oss.ObjectACL(oss.ACLPublicRead), oss.Meta("myprop", "mypropval"), } err = bucket.PutObject(objectKey, strings.NewReader(val), options...) if err != nil { HandleError(err) } props, err := bucket.GetObjectDetailedMeta(objectKey) if err != nil { HandleError(err) } fmt.Println("Object Meta:", props) // 场景6：大文件分片上传，支持并发上传，断点续传功能。 // 分片上传，分片大小为100K。默认使用不使用并发上传，不使用断点续传。 err = bucket.UploadFile(objectKey, localFile, 100*1024) if err != nil { HandleError(err) } // 分片大小为100K，3个协程并发上传。 err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Routines(3)) if err != nil { HandleError(err) } // 分片大小为100K，3个协程并发下载，使用断点续传上传文件。 err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Routines(3), oss.Checkpoint(true, "")) if err != nil { HandleError(err) } // 断点续传功能需要使用本地文件，记录哪些分片已经上传。该文件路径可以Checkpoint的第二个参数指定，如果为空，则为上传文件目录。 err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Checkpoint(true, localFile+".cp")) if err != nil { HandleError(err) } // 删除object和bucket err = DeleteTestBucketAndObject(bucketName) if err != nil { HandleError(err) } fmt.Println("PutObjectSample completed") }