Skip to content

Commit

Permalink
Merge pull request #18 from rhnvrm/feat-file-put
Browse files Browse the repository at this point in the history 
feat: add support for FilePut
  • Loading branch information
@rhnvrm
rhnvrm authored Dec 29, 2021
2 parents b23aec1 + a9d0bbf commit ad0419e
Show file tree
Hide file tree
Showing 3 changed files with 163 additions and 25 deletions.
10 changes: 5 additions & 5 deletions policy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ type UploadConfig struct {
MetaData map[string]string
}

// UploadPolicies Amazon s3 upload policies
// UploadPolicies Amazon s3 upload policies.
type UploadPolicies struct {
URL string
Form map[string]string
}

// PolicyJSON is policy rule
// PolicyJSON is policy rule.
type PolicyJSON struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
Expand All @@ -57,12 +57,12 @@ const (
defaultExpirationHour = 1 * time.Hour
)

// nowTime mockable time.Now()
var nowTime = func() time.Time {
// nowTime is returns UTC time.
func nowTime() time.Time {
return time.Now().UTC()
}

var newLine = []byte{'\n'}
var newLine = []byte{'\n'} //nolint

// CreateUploadPolicies creates amazon s3 sigv4 compatible
// policy and signing keys with the signature returns the upload policy.
Expand Down
121 changes: 105 additions & 16 deletions simples3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,9 @@ type UploadInput struct {
// optional fields
ContentDisposition string
ACL string
CustomMetadata map[string]string // Setting key/value pairs adds user-defined metadata keys to the object, prefixed with AMZMetaPrefix.
// Setting key/value pairs adds user-defined metadata
// keys to the object, prefixed with AMZMetaPrefix.
CustomMetadata map[string]string

Body io.ReadSeeker
}
Expand All @@ -101,14 +103,22 @@ type UploadResponse struct {
ETag string `xml:"ETag"`
}

// PutResponse is returned when the action is successful,
// and the service sends back an HTTP 200 response. The response
// returns ETag along with HTTP headers.
type PutResponse struct {
ETag string
Headers http.Header
}

// DeleteInput is passed to FileDelete as a parameter.
type DeleteInput struct {
Bucket string
ObjectKey string
}

// IAMResponse is used by NewUsingIAM to auto
// detect the credentials
// detect the credentials.
type IAMResponse struct {
Code string `json:"Code"`
LastUpdated string `json:"LastUpdated"`
Expand Down Expand Up @@ -143,7 +153,7 @@ func newUsingIAMImpl(baseURL, region string) (*S3, error) {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
return nil, errors.New(http.StatusText(resp.StatusCode))
}

Expand All @@ -157,7 +167,7 @@ func newUsingIAMImpl(baseURL, region string) (*S3, error) {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
return nil, errors.New(http.StatusText(resp.StatusCode))
}

Expand Down Expand Up @@ -241,7 +251,7 @@ func detectFileSize(body io.Seeker) (int64, error) {
}
defer body.Seek(pos, 0)

n, err := body.Seek(0, 2)
n, err := body.Seek(0, 2) //nolint:gomnd
if err != nil {
return -1, err
}
Expand Down Expand Up @@ -280,8 +290,11 @@ func (s3 *S3) signRequest(req *http.Request) error {
// Signature Version 4 requests. It provides a hash of the
// request payload. If there is no payload, you must provide
// the hash of an empty string.
emptyhash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
req.Header.Set("x-amz-content-sha256", emptyhash)

if req.Header.Get("x-amz-content-sha256") == "" {
emptyhash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
req.Header.Set("x-amz-content-sha256", emptyhash)
}

k := s3.signKeys(t)
h := hmac.New(sha256.New, k)
Expand Down Expand Up @@ -319,13 +332,87 @@ func (s3 *S3) FileDownload(u DownloadInput) (io.ReadCloser, error) {
return nil, err
}

if res.StatusCode != 200 {
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code: %s", res.Status)
}

return res.Body, nil
}

// FilePut makes a PUT call to S3.
func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) {
fSize, err := detectFileSize(u.Body)
if err != nil {
return PutResponse{}, err
}

content := make([]byte, fSize)
_, err = u.Body.Read(content)
if err != nil {
return PutResponse{}, err
}
u.Body.Seek(0, 0)

req, er := http.NewRequest(http.MethodPut, s3.getURL(u.Bucket, u.ObjectKey), u.Body)
if er != nil {
return PutResponse{}, err
}

if u.ContentType == "" {
u.ContentType = "application/octet-stream"
}

h := sha256.New()
h.Write(content)
req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil)))

req.Header.Set("Content-Type", u.ContentType)
req.Header.Set("Content-Length", fmt.Sprintf("%d", fSize))
req.Header.Set("Host", req.URL.Host)

for k, v := range u.CustomMetadata {
req.Header.Set("x-amz-meta-"+k, v)
}

if u.ContentDisposition != "" {
req.Header.Set("Content-Disposition", u.ContentDisposition)
}

if u.ACL != "" {
req.Header.Set("x-amz-acl", u.ACL)
}

req.ContentLength = fSize

if err := s3.signRequest(req); err != nil {
return PutResponse{}, err
}

// debug(httputil.DumpRequest(req, true))
// Submit the request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return PutResponse{}, err
}
defer res.Body.Close()

data, err := ioutil.ReadAll(res.Body)
if err != nil {
return PutResponse{}, err
}

// Check the response
if res.StatusCode != http.StatusOK {
return PutResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}

return PutResponse{
ETag: res.Header.Get("ETag"),
Headers: res.Header.Clone(),
}, nil
}

// FileUpload makes a POST call with the file written as multipart
// and on successful upload, checks for 200 OK.
func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
Expand Down Expand Up @@ -404,8 +491,9 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
if err != nil {
return UploadResponse{}, err
}

// Check the response
if res.StatusCode != 201 {
if res.StatusCode != http.StatusCreated {
return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}

Expand Down Expand Up @@ -436,7 +524,7 @@ func (s3 *S3) FileDelete(u DeleteInput) error {
}

// Check the response
if res.StatusCode != 204 {
if res.StatusCode != http.StatusNoContent {
return fmt.Errorf("status code: %s", res.Status)
}

Expand All @@ -460,7 +548,7 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
return DetailsResponse{}, err
}

if res.StatusCode != 200 {
if res.StatusCode != http.StatusOK {
return DetailsResponse{}, fmt.Errorf("status code: %s", res.Status)
}

Expand Down Expand Up @@ -515,7 +603,7 @@ func getFirstString(s []string) string {
return ""
}

// if object matches reserved string, no need to encode them
// if object matches reserved string, no need to encode them.
var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")

// encodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
Expand All @@ -525,7 +613,8 @@ var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
// adapted from https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285
// adapted from
// https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285
func encodePath(pathName string) string {
if reservedObjectNames.MatchString(pathName) {
return pathName
Expand All @@ -541,12 +630,12 @@ func encodePath(pathName string) string {
encodedPathname.WriteRune(s)
continue
default:
len := utf8.RuneLen(s)
if len < 0 {
lenR := utf8.RuneLen(s)
if lenR < 0 {
// if utf8 cannot convert, return the same string as is
return pathName
}
u := make([]byte, len)
u := make([]byte, lenR)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
Expand Down
57 changes: 53 additions & 4 deletions simples3_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ type tConfig struct {
Region string
}

func TestS3_FileUpload(t *testing.T) {
func TestS3_FileUploadPostAndPut(t *testing.T) {
testTxt, err := os.Open("testdata/test.txt")
if err != nil {
return
Expand Down Expand Up @@ -128,9 +128,10 @@ func TestS3_FileUpload(t *testing.T) {
}
for _, testcase := range tests {
tt := testcase
t.Run(tt.name, func(t *testing.T) {
t.Run(tt.name+"_post", func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)

resp, err := s3.FileUpload(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
Expand All @@ -154,8 +155,40 @@ func TestS3_FileUpload(t *testing.T) {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}

if len(dResp.AmzMeta) != 1 {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %v", dResp.AmzMeta)
if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp)
}
}
})
t.Run(tt.name+"_put", func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)

resp, err := s3.FilePut(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}

// reset file, to reuse in further tests.
tt.args.u.Body.Seek(0, 0)

// check for empty response
if resp.ETag == "" {
t.Errorf("S3.FileUpload() returned empty response, %v", resp)
}

if tt.testDetails {
dResp, err := s3.FileDetails(DetailsInput{
Bucket: tt.args.u.Bucket,
ObjectKey: tt.args.u.ObjectKey,
})

if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}

if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp)
}
}
})
Expand Down Expand Up @@ -296,6 +329,22 @@ func TestS3_FileDelete(t *testing.T) {
},
wantErr: false,
},
{
name: "Delete test_metadata.txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test_metadata.txt",
},
},
wantErr: false,
},
{
name: "Delete avatar.png",
fields: tConfig{
Expand Down

0 comments on commit ad0419e

Please sign in to comment.