imagekit: Added ImageKit backend

backend/imagekit/util.go

@@ -0,0 +1,193 @@
+package imagekit
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/rclone/rclone/backend/imagekit/client"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/pacer"
+)
+
+func (f *Fs) getFiles(ctx context.Context, path string, includeVersions bool) (files []client.File, err error) {
+
+	files = make([]client.File, 0)
+
+	var hasMore = true
+
+	for hasMore {
+		err = f.pacer.Call(func() (bool, error) {
+			var data *[]client.File
+			var res *http.Response
+			res, data, err = f.ik.Files(ctx, client.FilesOrFolderParam{
+				Skip:  len(files),
+				Limit: 100,
+				Path:  path,
+			}, includeVersions)
+
+			hasMore = !(len(*data) == 0 || len(*data) < 100)
+
+			if len(*data) > 0 {
+				files = append(files, *data...)
+			}
+
+			return f.shouldRetry(ctx, res, err)
+		})
+	}
+
+	if err != nil {
+		return make([]client.File, 0), err
+	}
+
+	return files, nil
+}
+
+func (f *Fs) getFolders(ctx context.Context, path string) (folders []client.Folder, err error) {
+
+	folders = make([]client.Folder, 0)
+
+	var hasMore = true
+
+	for hasMore {
+		err = f.pacer.Call(func() (bool, error) {
+			var data *[]client.Folder
+			var res *http.Response
+			res, data, err = f.ik.Folders(ctx, client.FilesOrFolderParam{
+				Skip:  len(folders),
+				Limit: 100,
+				Path:  path,
+			})
+
+			hasMore = !(len(*data) == 0 || len(*data) < 100)
+
+			if len(*data) > 0 {
+				folders = append(folders, *data...)
+			}
+
+			return f.shouldRetry(ctx, res, err)
+		})
+	}
+
+	if err != nil {
+		return make([]client.Folder, 0), err
+	}
+
+	return folders, nil
+}
+
+func (f *Fs) getFileByName(ctx context.Context, path string, name string) (file *client.File) {
+
+	err := f.pacer.Call(func() (bool, error) {
+		res, data, err := f.ik.Files(ctx, client.FilesOrFolderParam{
+			Limit:       1,
+			Path:        path,
+			SearchQuery: fmt.Sprintf(`type = "file" AND name = %s`, strconv.Quote(name)),
+		}, false)
+
+		if len(*data) == 0 {
+			file = nil
+		} else {
+			file = &(*data)[0]
+		}
+
+		return f.shouldRetry(ctx, res, err)
+	})
+
+	if err != nil {
+		return nil
+	}
+
+	return file
+}
+
+func (f *Fs) getFolderByName(ctx context.Context, path string, name string) (folder *client.Folder, err error) {
+	err = f.pacer.Call(func() (bool, error) {
+		res, data, err := f.ik.Folders(ctx, client.FilesOrFolderParam{
+			Limit:       1,
+			Path:        path,
+			SearchQuery: fmt.Sprintf(`type = "folder" AND name = %s`, strconv.Quote(name)),
+		})
+
+		if len(*data) == 0 {
+			folder = nil
+		} else {
+			folder = &(*data)[0]
+		}
+
+		return f.shouldRetry(ctx, res, err)
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return folder, nil
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	401, // Unauthorized (e.g. "Token has expired")
+	408, // Request Timeout
+	429, // Rate exceeded.
+	500, // Get occasional 500 Internal Server Error
+	503, // Service Unavailable
+	504, // Gateway Time-out
+}
+
+func shouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
+	if resp == nil {
+		return false
+	}
+	for _, e := range retryErrorCodes {
+		if resp.StatusCode == e {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+
+	if resp != nil && (resp.StatusCode == 429 || resp.StatusCode == 503) {
+		var retryAfter = 1
+		retryAfterString := resp.Header.Get("X-RateLimit-Reset")
+		if retryAfterString != "" {
+			var err error
+			retryAfter, err = strconv.Atoi(retryAfterString)
+			if err != nil {
+				fs.Errorf(f, "Malformed %s header %q: %v", "X-RateLimit-Reset", retryAfterString, err)
+			}
+		}
+
+		return true, pacer.RetryAfterError(err, time.Duration(retryAfter)*time.Millisecond)
+	}
+
+	return fserrors.ShouldRetry(err) || shouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// EncodePath encapsulates the logic for encoding a path
+func (f *Fs) EncodePath(str string) string {
+	return f.opt.Enc.FromStandardPath(str)
+}
+
+// DecodePath encapsulates the logic for decoding a path
+func (f *Fs) DecodePath(str string) string {
+	return f.opt.Enc.ToStandardPath(str)
+}
+
+// EncodeFileName encapsulates the logic for encoding a file name
+func (f *Fs) EncodeFileName(str string) string {
+	return f.opt.Enc.FromStandardName(str)
+}
+
+// DecodeFileName encapsulates the logic for decoding a file name
+func (f *Fs) DecodeFileName(str string) string {
+	return f.opt.Enc.ToStandardName(str)
+}