11package client
22
33import (
4+ "context"
5+ "crypto/tls"
6+ "net/http"
47 "os"
8+ "strings"
9+ "time"
510
6- "github.com/aws/aws-sdk-go/aws"
7- "github.com/aws/aws-sdk-go/aws/awserr"
8- "github.com/aws/aws-sdk-go/aws/credentials"
9- "github.com/aws/aws-sdk-go/aws/session"
10- "github.com/aws/aws-sdk-go/service/s3"
11- "github.com/aws/aws-sdk-go/service/s3/s3manager"
11+ "github.com/minio/minio-go/v7"
12+ "github.com/minio/minio-go/v7/pkg/credentials"
13+ )
14+
15+ const (
16+ s3DefaultTimeout = 30 * time .Second
17+ s3TransferTimeout = 24 * time .Hour
1218)
1319
1420type s3Client struct {
1521 scType string
1622 bucket string
17- Sess session.Session
23+ client * minio.Client
24+ }
25+
26+ func (s * s3Client ) ctx (timeout time.Duration ) (context.Context , context.CancelFunc ) {
27+ return context .WithTimeout (context .Background (), timeout )
1828}
1929
2030func NewS3Client (vars map [string ]interface {}) (* s3Client , error ) {
@@ -31,69 +41,80 @@ func NewS3Client(vars map[string]interface{}) (*s3Client, error) {
3141 if len (mode ) == 0 {
3242 mode = "virtual hosted"
3343 }
34- sess , err := session .NewSession (& aws.Config {
35- Credentials : credentials .NewStaticCredentials (accessKey , secretKey , "" ),
36- Endpoint : aws .String (endpoint ),
37- Region : aws .String (region ),
38- DisableSSL : aws .Bool (true ), S3ForcePathStyle : aws .Bool (mode == "path" ),
44+
45+ lookupStyle := minio .BucketLookupDNS
46+ if mode == "path" {
47+ lookupStyle = minio .BucketLookupPath
48+ }
49+
50+ ssl := strings .Split (endpoint , ":" )[0 ]
51+ secure := false
52+ tlsConfig := & tls.Config {}
53+ if ssl == "https" {
54+ secure = true
55+ tlsConfig .InsecureSkipVerify = true
56+ }
57+ var transport http.RoundTripper = & http.Transport {
58+ TLSClientConfig : tlsConfig ,
59+ }
60+
61+ endpoint = strings .TrimPrefix (endpoint , ssl + "://" )
62+
63+ client , err := minio .New (endpoint , & minio.Options {
64+ Creds : credentials .NewStaticV4 (accessKey , secretKey , "" ),
65+ Secure : secure ,
66+ Region : region ,
67+ BucketLookup : lookupStyle ,
68+ Transport : transport ,
3969 })
4070 if err != nil {
4171 return nil , err
4272 }
43- return & s3Client {scType : scType , bucket : bucket , Sess : * sess }, nil
73+ return & s3Client {scType : scType , bucket : bucket , client : client }, nil
4474}
4575
4676func (s s3Client ) ListBuckets () ([]interface {}, error ) {
47- var result [] interface {}
48- svc := s3 . New ( & s . Sess )
49- res , err := svc . ListBuckets (nil )
77+ ctx , cancel := s . ctx ( s3DefaultTimeout )
78+ defer cancel ( )
79+ buckets , err := s . client . ListBuckets (ctx )
5080 if err != nil {
5181 return nil , err
5282 }
53- for _ , b := range res .Buckets {
83+ var result []interface {}
84+ for _ , b := range buckets {
5485 result = append (result , b .Name )
5586 }
5687 return result , nil
5788}
5889
5990func (s s3Client ) Exist (path string ) (bool , error ) {
60- svc := s3 .New (& s .Sess )
61- if _ , err := svc .HeadObject (& s3.HeadObjectInput {
62- Bucket : & s .bucket ,
63- Key : & path ,
64- }); err != nil {
65- if aerr , ok := err .(awserr.RequestFailure ); ok {
66- if aerr .StatusCode () == 404 {
67- return false , nil
68- }
69- } else {
70- return false , aerr
91+ ctx , cancel := s .ctx (s3DefaultTimeout )
92+ defer cancel ()
93+ _ , err := s .client .StatObject (ctx , s .bucket , path , minio.StatObjectOptions {})
94+ if err != nil {
95+ resp := minio .ToErrorResponse (err )
96+ if resp .StatusCode == 404 {
97+ return false , nil
7198 }
99+ return false , err
72100 }
73101 return true , nil
74102}
75103
76104func (s * s3Client ) Size (path string ) (int64 , error ) {
77- svc := s3 .New (& s .Sess )
78- file , err := svc .GetObject (& s3.GetObjectInput {
79- Bucket : & s .bucket ,
80- Key : & path ,
81- })
105+ ctx , cancel := s .ctx (s3DefaultTimeout )
106+ defer cancel ()
107+ info , err := s .client .StatObject (ctx , s .bucket , path , minio.StatObjectOptions {})
82108 if err != nil {
83109 return 0 , err
84110 }
85- return * file . ContentLength , nil
111+ return info . Size , nil
86112}
87113
88114func (s s3Client ) Delete (path string ) (bool , error ) {
89- svc := s3 .New (& s .Sess )
90- if _ , err := svc .DeleteObject (& s3.DeleteObjectInput {Bucket : aws .String (s .bucket ), Key : aws .String (path )}); err != nil {
91- return false , err
92- }
93- if err := svc .WaitUntilObjectNotExists (& s3.HeadObjectInput {
94- Bucket : aws .String (s .bucket ),
95- Key : aws .String (path ),
96- }); err != nil {
115+ ctx , cancel := s .ctx (s3DefaultTimeout )
116+ defer cancel ()
117+ if err := s .client .RemoveObject (ctx , s .bucket , path , minio.RemoveObjectOptions {}); err != nil {
97118 return false , err
98119 }
99120 return true , nil
@@ -110,16 +131,21 @@ func (s s3Client) Upload(src, target string) (bool, error) {
110131 }
111132 defer file .Close ()
112133
113- uploader := s3manager .NewUploader (& s .Sess )
114- if fileInfo .Size () > s3manager .MaxUploadParts * s3manager .DefaultUploadPartSize {
115- uploader .PartSize = fileInfo .Size () / (s3manager .MaxUploadParts - 1 )
134+ opts := minio.PutObjectOptions {
135+ StorageClass : s .scType ,
136+ }
137+
138+ const maxParts = 10000
139+ const defaultPartSize = 64 * 1024 * 1024 // 64 MiB
140+ partSize := uint64 (defaultPartSize )
141+ if fileInfo .Size () > int64 (maxParts )* int64 (defaultPartSize ) {
142+ partSize = uint64 (fileInfo .Size ()) / (maxParts - 1 )
116143 }
117- if _ , err := uploader .Upload (& s3manager.UploadInput {
118- Bucket : aws .String (s .bucket ),
119- Key : aws .String (target ),
120- Body : file ,
121- StorageClass : & s .scType ,
122- }); err != nil {
144+ opts .PartSize = partSize
145+
146+ ctx , cancel := s .ctx (s3TransferTimeout )
147+ defer cancel ()
148+ if _ , err := s .client .PutObject (ctx , s .bucket , target , file , fileInfo .Size (), opts ); err != nil {
123149 return false , err
124150 }
125151 return true , nil
@@ -129,34 +155,28 @@ func (s s3Client) Download(src, target string) (bool, error) {
129155 if _ , err := os .Stat (target ); err == nil {
130156 _ = os .Remove (target )
131157 }
132- file , err := os .Create (target )
133- if err != nil {
134- return false , err
135- }
136- defer file .Close ()
137- downloader := s3manager .NewDownloader (& s .Sess )
138- if _ , err = downloader .Download (file , & s3.GetObjectInput {
139- Bucket : aws .String (s .bucket ),
140- Key : aws .String (src ),
141- }); err != nil {
158+ ctx , cancel := s .ctx (s3TransferTimeout )
159+ defer cancel ()
160+ if err := s .client .FGetObject (ctx , s .bucket , src , target , minio.GetObjectOptions {}); err != nil {
142161 os .Remove (target )
143162 return false , err
144163 }
145164 return true , nil
146165}
147166
148167func (s * s3Client ) ListObjects (prefix string ) ([]string , error ) {
149- svc := s3 .New (& s .Sess )
150- var result []string
151- outputs , err := svc .ListObjects (& s3.ListObjectsInput {
152- Bucket : & s .bucket ,
153- Prefix : & prefix ,
154- })
155- if err != nil {
156- return result , err
168+ opts := minio.ListObjectsOptions {
169+ Recursive : true ,
170+ Prefix : prefix ,
157171 }
158- for _ , item := range outputs .Contents {
159- result = append (result , * item .Key )
172+ var result []string
173+ ctx , cancel := s .ctx (s3DefaultTimeout )
174+ defer cancel ()
175+ for object := range s .client .ListObjects (ctx , s .bucket , opts ) {
176+ if object .Err != nil {
177+ return result , object .Err
178+ }
179+ result = append (result , object .Key )
160180 }
161181 return result , nil
162182}
0 commit comments