fs: Remove the charset from the Mimetype -FIXME EXPERIMENT

.circleci/config.yml

@@ -13,10 +13,10 @@ jobs:
     - run:
         name: Cross-compile rclone
         command: |
-            docker pull rclone/xgo-cgofuse
+            docker pull billziss/xgo-cgofuse
             go get -v github.com/karalabe/xgo
             xgo \
-                --image=rclone/xgo-cgofuse \
+                --image=billziss/xgo-cgofuse \
                 --targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
                 -tags cmount \
                 .
@@ -29,19 +29,6 @@ jobs:
         command: |
             mkdir -p /tmp/rclone.dist
             cp -R rclone-* /tmp/rclone.dist
-            mkdir build
-            cp -R rclone-* build/
-
-    - run:
-        name: Build rclone
-        command: |
-            go version
-            go build
-
-    - run:
-        name: Upload artifacts
-        command: |
-            make circleci_upload
 
     - store_artifacts:
         path: /tmp/rclone.dist

CONTRIBUTING.md

@@ -351,12 +351,6 @@ Unit tests
 Integration tests
 
   * Add your backend to `fstest/test_all/config.yaml`
-      * Once you've done that then you can use the integration test framework from the project root:
-      * go install ./...
-      * test_all -backend remote
-
-Or if you want to run the integration tests manually:
-
   * Make sure integration tests pass with
       * `cd fs/operations`
       * `go test -v -remote TestRemote:`
@@ -378,3 +372,4 @@ Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](
   * `docs/content/about.md` - front page of rclone.org
   * `docs/layouts/chrome/navbar.html` - add it to the website navigation
   * `bin/make_manual.py` - add the page to the `docs` constant
+  * `cmd/cmd.go` - the main help for rclone

Makefile

@@ -185,13 +185,6 @@ ifndef BRANCH_PATH
 endif
 	@echo Beta release ready at $(BETA_URL)
 
-circleci_upload:
-	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
-ifndef BRANCH_PATH
-	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
-endif
-	@echo Beta release ready at $(BETA_URL)/testbuilds
-
 BUILD_FLAGS := -exclude "^(windows|darwin)/"
 ifeq ($(TRAVIS_OS_NAME),osx)
 	BUILD_FLAGS := -include "^darwin/" -cgo

README.md

@@ -20,7 +20,6 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
 
 ## Storage providers
 
-  * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
   * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)

backend/drive/drive.go

@@ -718,16 +718,12 @@ func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, er
 
 // Figure out if the user wants to use a team drive
 func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
-	// Stop if we are running non-interactive config
-	if fs.Config.AutoConfirm {
-		return nil
-	}
 	if opt.TeamDriveID == "" {
 		fmt.Printf("Configure this as a team drive?\n")
 	} else {
 		fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID)
 	}
-	if !config.Confirm() {
+	if !config.ConfirmWithDefault(false) {
 		return nil
 	}
 	client, err := createOAuthClient(opt, name, m)

backend/ftp/ftp.go

@@ -646,21 +646,7 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 
 // Close the FTP reader and return the connection to the pool
 func (f *ftpReadCloser) Close() error {
-	var err error
-	errchan := make(chan error, 1)
-	go func() {
-		errchan <- f.rc.Close()
-	}()
-	// Wait for Close for up to 60 seconds
-	timer := time.NewTimer(60 * time.Second)
-	select {
-	case err = <-errchan:
-		timer.Stop()
-	case <-timer.C:
-		// if timer fired assume no error but connection dead
-		fs.Errorf(f.f, "Timeout when waiting for connection Close")
-		return nil
-	}
+	err := f.rc.Close()
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()

backend/local/local.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -22,7 +21,6 @@ import (
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/fserrors"
 	"github.com/ncw/rclone/fs/hash"
-	"github.com/ncw/rclone/lib/file"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
@@ -653,7 +651,7 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 	o.fs.objectHashesMu.Unlock()
 
 	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
-		in, err := file.Open(o.path)
+		in, err := os.Open(o.path)
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -743,7 +741,6 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
 	}
 
 	n, err = file.in.Read(p)
-	log.Printf("*** Read result n=%d, err=%v, len(p)=%d", n, err, len(p))
 	if n > 0 {
 		// Hash routines never return an error
 		_, _ = file.hash.Write(p[:n])
@@ -783,7 +780,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		}
 	}
 
-	fd, err := file.Open(o.path)
+	fd, err := os.Open(o.path)
 	if err != nil {
 		return
 	}
@@ -829,7 +826,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	out, err := os.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
 	if err != nil {
 		return err
 	}

backend/local/local_internal_test.go

@@ -1,13 +1,13 @@
 package local
 
 import (
+	"os"
 	"path"
 	"testing"
 	"time"
 
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fstest"
-	"github.com/ncw/rclone/lib/file"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -38,7 +38,7 @@ func TestUpdatingCheck(t *testing.T) {
 	filePath := "sub dir/local test"
 	r.WriteFile(filePath, "content", time.Now())
 
-	fd, err := file.Open(path.Join(r.LocalName, filePath))
+	fd, err := os.Open(path.Join(r.LocalName, filePath))
 	if err != nil {
 		t.Fatalf("failed opening file %q: %v", filePath, err)
 	}

backend/onedrive/api/types.go

@@ -285,7 +285,6 @@ type AsyncOperationStatus struct {
 
 // GetID returns a normalized ID of the item
 // If DriveID is known it will be prefixed to the ID with # seperator
-// Can be parsed using onedrive.parseNormalizedID(normalizedID)
 func (i *Item) GetID() string {
 	if i.IsRemote() && i.RemoteItem.ID != "" {
 		return i.RemoteItem.ParentReference.DriveID + "#" + i.RemoteItem.ID

backend/onedrive/onedrive.go

@@ -75,8 +75,9 @@ func init() {
 				return
 			}
 
-			// Stop if we are running non-interactive config
-			if fs.Config.AutoConfirm {
+			// Are we running headless?
+			if automatic, _ := m.Get(config.ConfigAutomatic); automatic != "" {
+				// Yes, okay we are done
 				return
 			}
 
@@ -198,7 +199,7 @@ func init() {
 
 			fmt.Printf("Found drive '%s' of type '%s', URL: %s\nIs that okay?\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL)
 			// This does not work, YET :)
-			if !config.ConfirmWithConfig(m, "config_drive_ok", true) {
+			if !config.Confirm() {
 				log.Fatalf("Cancelled by user")
 			}
 
@@ -333,10 +334,20 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
-// if `relPath` == "", it reads the metadata for the item with that ID.
-func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
-	opts := newOptsCall(normalizedID, "GET", ":/"+rest.URLPathEscape(replaceReservedChars(relPath)))
+// readMetaDataForPath reads the metadata from the path
+func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Response, err error) {
+	var opts rest.Opts
+	if len(path) == 0 {
+		opts = rest.Opts{
+			Method: "GET",
+			Path:   "/root",
+		}
+	} else {
+		opts = rest.Opts{
+			Method: "GET",
+			Path:   "/root:/" + rest.URLPathEscape(replaceReservedChars(path)),
+		}
+	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(&opts, nil, &info)
 		return shouldRetry(resp, err)
@@ -345,72 +356,6 @@ func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string
 	return info, resp, err
 }
 
-// readMetaDataForPath reads the metadata from the path (relative to the absolute root)
-func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Response, err error) {
-	firstSlashIndex := strings.IndexRune(path, '/')
-
-	if f.driveType != driveTypePersonal || firstSlashIndex == -1 {
-		var opts rest.Opts
-		if len(path) == 0 {
-			opts = rest.Opts{
-				Method: "GET",
-				Path:   "/root",
-			}
-		} else {
-			opts = rest.Opts{
-				Method: "GET",
-				Path:   "/root:/" + rest.URLPathEscape(replaceReservedChars(path)),
-			}
-		}
-		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(&opts, nil, &info)
-			return shouldRetry(resp, err)
-		})
-		return info, resp, err
-	}
-
-	// The following branch handles the case when we're using OneDrive Personal and the path is in a folder.
-	// For OneDrive Personal, we need to consider the "shared with me" folders.
-	// An item in such a folder can only be addressed by its ID relative to the sharer's driveID or
-	// by its path relative to the folder's ID relative to the sharer's driveID.
-	// Note: A "shared with me" folder can only be placed in the sharee's absolute root.
-	// So we read metadata relative to a suitable folder's normalized ID.
-	var dirCacheFoundRoot bool
-	var rootNormalizedID string
-	if f.dirCache != nil {
-		var ok bool
-		if rootNormalizedID, ok = f.dirCache.Get(""); ok {
-			dirCacheFoundRoot = true
-		}
-	}
-
-	relPath, insideRoot := getRelativePathInsideBase(f.root, path)
-	var firstDir, baseNormalizedID string
-	if !insideRoot || !dirCacheFoundRoot {
-		// We do not have the normalized ID in dirCache for our query to base on. Query it manually.
-		firstDir, relPath = path[:firstSlashIndex], path[firstSlashIndex+1:]
-		info, resp, err := f.readMetaDataForPath(firstDir)
-		if err != nil {
-			return info, resp, err
-		}
-		baseNormalizedID = info.GetID()
-	} else {
-		if f.root != "" {
-			// Read metadata based on root
-			baseNormalizedID = rootNormalizedID
-		} else {
-			// Read metadata based on firstDir
-			firstDir, relPath = path[:firstSlashIndex], path[firstSlashIndex+1:]
-			baseNormalizedID, err = f.dirCache.FindDir(firstDir, false)
-			if err != nil {
-				return nil, nil, err
-			}
-		}
-	}
-
-	return f.readMetaDataForPathRelativeToID(baseNormalizedID, relPath)
-}
-
 // errorHandler parses a non 2xx error response into an error
 func errorHandler(resp *http.Response) error {
 	// Decode error response
@@ -569,11 +514,18 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 // FindLeaf finds a directory of name leaf in the folder with ID pathID
 func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
 	// fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf)
-	_, ok := f.dirCache.GetInv(pathID)
+	parent, ok := f.dirCache.GetInv(pathID)
 	if !ok {
 		return "", false, errors.New("couldn't find parent ID")
 	}
-	info, resp, err := f.readMetaDataForPathRelativeToID(pathID, leaf)
+	path := leaf
+	if parent != "" {
+		path = parent + "/" + path
+	}
+	if f.dirCache.FoundRoot() {
+		path = f.rootSlash() + path
+	}
+	info, resp, err := f.readMetaDataForPath(path)
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
 			return "", false, nil
@@ -915,13 +867,13 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	opts.ExtraHeaders = map[string]string{"Prefer": "respond-async"}
 	opts.NoResponse = true
 
-	id, dstDriveID, _ := parseNormalizedID(directoryID)
+	id, _, _ := parseDirID(directoryID)
 
 	replacedLeaf := replaceReservedChars(leaf)
 	copyReq := api.CopyItemRequest{
 		Name: &replacedLeaf,
 		ParentReference: api.ItemReference{
-			DriveID: dstDriveID,
+			DriveID: f.driveID,
 			ID:      id,
 		},
 	}
@@ -988,23 +940,15 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		return nil, err
 	}
 
-	id, dstDriveID, _ := parseNormalizedID(directoryID)
-	_, srcObjDriveID, _ := parseNormalizedID(srcObj.id)
-
-	if dstDriveID != srcObjDriveID {
-		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
-		// "Items cannot be moved between Drives using this request."
-		return nil, fs.ErrorCantMove
-	}
-
 	// Move the object
 	opts := newOptsCall(srcObj.id, "PATCH", "")
 
+	id, _, _ := parseDirID(directoryID)
+
 	move := api.MoveItemRequest{
 		Name: replaceReservedChars(leaf),
 		ParentReference: &api.ItemReference{
-			DriveID: dstDriveID,
-			ID:      id,
+			ID: id,
 		},
 		// We set the mod time too as it gets reset otherwise
 		FileSystemInfo: &api.FileSystemInfoFacet{
@@ -1080,20 +1024,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	if err != nil {
 		return err
 	}
-	parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(srcRemote, false)
-	if err != nil {
-		return err
-	}
-	_, srcDriveID, _ := parseNormalizedID(srcID)
-
-	if dstDriveID != srcDriveID {
-		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
-		// "Items cannot be moved between Drives using this request."
-		return fs.ErrorCantDirMove
-	}
+	parsedDstDirID, _, _ := parseDirID(dstDirectoryID)
 
 	// Check destination does not exist
 	if dstRemote != "" {
@@ -1107,8 +1038,14 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 	}
 
+	// Find ID of src
+	srcID, err := srcFs.dirCache.FindDir(srcRemote, false)
+	if err != nil {
+		return err
+	}
+
 	// Get timestamps of src so they can be preserved
-	srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(srcID, "")
+	srcInfo, _, err := srcFs.readMetaDataForPath(srcPath)
 	if err != nil {
 		return err
 	}
@@ -1118,8 +1055,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	move := api.MoveItemRequest{
 		Name: replaceReservedChars(leaf),
 		ParentReference: &api.ItemReference{
-			DriveID: dstDriveID,
-			ID:      parsedDstDirID,
+			ID: parsedDstDirID,
 		},
 		// We set the mod time too as it gets reset otherwise
 		FileSystemInfo: &api.FileSystemInfoFacet{
@@ -1186,7 +1122,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 	if err != nil {
 		return "", err
 	}
-	opts := newOptsCall(info.GetID(), "POST", "/createLink")
+	opts := newOptsCall(info.ID, "POST", "/createLink")
 
 	share := api.CreateShareLinkRequest{
 		Type:  "view",
@@ -1334,13 +1270,13 @@ func (o *Object) ModTime() time.Time {
 // setModTime sets the modification time of the local fs object
 func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
 	var opts rest.Opts
-	leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
-	trueDirID, drive, rootURL := parseNormalizedID(directoryID)
+	_, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
+	_, drive, rootURL := parseDirID(directoryID)
 	if drive != "" {
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf),
+			Path:    "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()),
 		}
 	} else {
 		opts = rest.Opts{
@@ -1408,7 +1344,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 // createUploadSession creates an upload session for the object
 func (o *Object) createUploadSession(modTime time.Time) (response *api.CreateUploadResponse, err error) {
 	leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
-	id, drive, rootURL := parseNormalizedID(directoryID)
+	id, drive, rootURL := parseDirID(directoryID)
 	var opts rest.Opts
 	if drive != "" {
 		opts = rest.Opts{
@@ -1541,13 +1477,13 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
 	fs.Debugf(o, "Starting singlepart upload")
 	var resp *http.Response
 	var opts rest.Opts
-	leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
-	trueDirID, drive, rootURL := parseNormalizedID(directoryID)
+	_, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
+	_, drive, rootURL := parseDirID(directoryID)
 	if drive != "" {
 		opts = rest.Opts{
 			Method:        "PUT",
 			RootURL:       rootURL,
-			Path:          "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf) + ":/content",
+			Path:          "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
 			ContentLength: &size,
 			Body:          in,
 		}
@@ -1630,8 +1566,8 @@ func (o *Object) ID() string {
 	return o.id
 }
 
-func newOptsCall(normalizedID string, method string, route string) (opts rest.Opts) {
-	id, drive, rootURL := parseNormalizedID(normalizedID)
+func newOptsCall(id string, method string, route string) (opts rest.Opts) {
+	id, drive, rootURL := parseDirID(id)
 
 	if drive != "" {
 		return rest.Opts{
@@ -1646,10 +1582,7 @@ func newOptsCall(normalizedID string, method string, route string) (opts rest.Op
 	}
 }
 
-// parseNormalizedID parses a normalized ID (may be in the form `driveID#itemID` or just `itemID`)
-// and returns itemID, driveID, rootURL.
-// Such a normalized ID can come from (*Item).GetID()
-func parseNormalizedID(ID string) (string, string, string) {
+func parseDirID(ID string) (string, string, string) {
 	if strings.Index(ID, "#") >= 0 {
 		s := strings.Split(ID, "#")
 		return s[1], s[0], graphURL + "/drives"
@@ -1657,21 +1590,6 @@ func parseNormalizedID(ID string) (string, string, string) {
 	return ID, "", ""
 }
 
-// getRelativePathInsideBase checks if `target` is inside `base`. If so, it
-// returns a relative path for `target` based on `base` and a boolean `true`.
-// Otherwise returns "", false.
-func getRelativePathInsideBase(base, target string) (string, bool) {
-	if base == "" {
-		return target, true
-	}
-
-	baseSlash := base + "/"
-	if strings.HasPrefix(target+"/", baseSlash) {
-		return target[len(baseSlash):], true
-	}
-	return "", false
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/qingstor/qingstor.go

@@ -72,51 +72,14 @@ func init() {
 			Help:     "Number of connection retries.",
 			Default:  3,
 			Advanced: true,
-		}, {
-			Name: "upload_cutoff",
-			Help: `Cutoff for switching to chunked upload
-
-Any files larger than this will be uploaded in chunks of chunk_size.
-The minimum is 0 and the maximum is 5GB.`,
-			Default:  defaultUploadCutoff,
-			Advanced: true,
-		}, {
-			Name: "chunk_size",
-			Help: `Chunk size to use for uploading.
-
-When uploading files larger than upload_cutoff they will be uploaded
-as multipart uploads using this chunk size.
-
-Note that "--qingstor-upload-concurrency" chunks of this size are buffered
-in memory per transfer.
-
-If you are transferring large files over high speed links and you have
-enough memory, then increasing this will speed up the transfers.`,
-			Default:  minChunkSize,
-			Advanced: true,
-		}, {
-			Name: "upload_concurrency",
-			Help: `Concurrency for multipart uploads.
-
-This is the number of chunks of the same file that are uploaded
-concurrently.
-
-If you are uploading small numbers of large file over high speed link
-and these uploads do not fully utilize your bandwidth, then increasing
-this may help to speed up the transfers.`,
-			Default:  4,
-			Advanced: true,
 		}},
 	})
 }
 
 // Constants
 const (
-	listLimitSize       = 1000                   // Number of items to read at once
-	maxSizeForCopy      = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY
-	minChunkSize        = fs.SizeSuffix(minMultiPartSize)
-	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
-	maxUploadCutoff     = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
+	listLimitSize  = 1000                   // Number of items to read at once
+	maxSizeForCopy = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY
 )
 
 // Globals
@@ -129,15 +92,12 @@ func timestampToTime(tp int64) time.Time {
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth           bool          `config:"env_auth"`
-	AccessKeyID       string        `config:"access_key_id"`
-	SecretAccessKey   string        `config:"secret_access_key"`
-	Endpoint          string        `config:"endpoint"`
-	Zone              string        `config:"zone"`
-	ConnectionRetries int           `config:"connection_retries"`
-	UploadCutoff      fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize         fs.SizeSuffix `config:"chunk_size"`
-	UploadConcurrency int           `config:"upload_concurrency"`
+	EnvAuth           bool   `config:"env_auth"`
+	AccessKeyID       string `config:"access_key_id"`
+	SecretAccessKey   string `config:"secret_access_key"`
+	Endpoint          string `config:"endpoint"`
+	Zone              string `config:"zone"`
+	ConnectionRetries int    `config:"connection_retries"`
 }
 
 // Fs represents a remote qingstor server
@@ -267,36 +227,6 @@ func qsServiceConnection(opt *Options) (*qs.Service, error) {
 	return qs.Init(cf)
 }
 
-func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	if cs < minChunkSize {
-		return errors.Errorf("%s is less than %s", cs, minChunkSize)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadChunkSize(cs)
-	if err == nil {
-		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-	}
-	return
-}
-
-func checkUploadCutoff(cs fs.SizeSuffix) error {
-	if cs > maxUploadCutoff {
-		return errors.Errorf("%s is greater than %s", cs, maxUploadCutoff)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadCutoff(cs)
-	if err == nil {
-		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
-	}
-	return
-}
-
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -305,14 +235,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = checkUploadChunkSize(opt.ChunkSize)
-	if err != nil {
-		return nil, errors.Wrap(err, "qingstor: chunk size")
-	}
-	err = checkUploadCutoff(opt.UploadCutoff)
-	if err != nil {
-		return nil, errors.Wrap(err, "qingstor: upload cutoff")
-	}
 	bucket, key, err := qsParsePath(root)
 	if err != nil {
 		return nil, err
@@ -991,24 +913,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	mimeType := fs.MimeType(src)
 
 	req := uploadInput{
-		body:        in,
-		qsSvc:       o.fs.svc,
-		bucket:      o.fs.bucket,
-		zone:        o.fs.zone,
-		key:         key,
-		mimeType:    mimeType,
-		partSize:    int64(o.fs.opt.ChunkSize),
-		concurrency: o.fs.opt.UploadConcurrency,
+		body:     in,
+		qsSvc:    o.fs.svc,
+		bucket:   o.fs.bucket,
+		zone:     o.fs.zone,
+		key:      key,
+		mimeType: mimeType,
 	}
 	uploader := newUploader(&req)
 
-	size := src.Size()
-	multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
-	if multipart {
-		err = uploader.upload()
-	} else {
-		err = uploader.singlePartUpload(in, size)
-	}
+	err = uploader.upload()
 	if err != nil {
 		return err
 	}

backend/qingstor/qingstor_test.go

@@ -2,12 +2,12 @@
 
 // +build !plan9
 
-package qingstor
+package qingstor_test
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/backend/qingstor"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
@@ -15,19 +15,6 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestQingStor:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: minChunkSize,
-		},
+		NilObject:  (*qingstor.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/qingstor/upload.go

@@ -152,11 +152,11 @@ func (u *uploader) init() {
 }
 
 // singlePartUpload upload a single object that contentLength less than "defaultUploadPartSize"
-func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
+func (u *uploader) singlePartUpload(buf io.ReadSeeker) error {
 	bucketInit, _ := u.bucketInit()
 
 	req := qs.PutObjectInput{
-		ContentLength: &size,
+		ContentLength: &u.readerPos,
 		ContentType:   &u.cfg.mimeType,
 		Body:          buf,
 	}
@@ -180,7 +180,7 @@ func (u *uploader) upload() error {
 	reader, _, err := u.nextReader()
 	if err == io.EOF { // single part
 		fs.Debugf(u, "Uploading as single part object to QingStor")
-		return u.singlePartUpload(reader, u.readerPos)
+		return u.singlePartUpload(reader)
 	} else if err != nil {
 		return errors.Errorf("read upload data failed: %s", err)
 	}
@@ -392,14 +392,6 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) error {
 		var nextChunkLen int
 		reader, nextChunkLen, err = mu.nextReader()
 		if err != nil && err != io.EOF {
-			// empty ch
-			go func() {
-				for range ch {
-				}
-			}()
-			// Wait for all goroutines finish
-			close(ch)
-			mu.wg.Wait()
 			return err
 		}
 		if nextChunkLen == 0 && partNumber > 0 {

backend/s3/s3.go

@@ -53,7 +53,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
+		Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name: fs.ConfigProvider,
@@ -61,9 +61,6 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "AWS",
 				Help:  "Amazon Web Services (AWS) S3",
-			}, {
-				Value: "Alibaba",
-				Help:  "Alibaba Cloud Object Storage System (OSS) formerly Aliyun",
 			}, {
 				Value: "Ceph",
 				Help:  "Ceph Object Storage",
@@ -79,9 +76,6 @@ func init() {
 			}, {
 				Value: "Minio",
 				Help:  "Minio Object Storage",
-			}, {
-				Value: "Netease",
-				Help:  "Netease Object Storage (NOS)",
 			}, {
 				Value: "Wasabi",
 				Help:  "Wasabi Object Storage",
@@ -156,7 +150,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba",
+			Provider: "!AWS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure. Will use v4 signatures and an empty region.",
@@ -275,73 +269,10 @@ func init() {
 				Value: "s3.tor01.objectstorage.service.networklayer.com",
 				Help:  "Toronto Single Site Private Endpoint",
 			}},
-		}, {
-			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
-			Name:     "endpoint",
-			Help:     "Endpoint for OSS API.",
-			Provider: "Alibaba",
-			Examples: []fs.OptionExample{{
-				Value: "oss-cn-hangzhou.aliyuncs.com",
-				Help:  "East China 1 (Hangzhou)",
-			}, {
-				Value: "oss-cn-shanghai.aliyuncs.com",
-				Help:  "East China 2 (Shanghai)",
-			}, {
-				Value: "oss-cn-qingdao.aliyuncs.com",
-				Help:  "North China 1 (Qingdao)",
-			}, {
-				Value: "oss-cn-beijing.aliyuncs.com",
-				Help:  "North China 2 (Beijing)",
-			}, {
-				Value: "oss-cn-zhangjiakou.aliyuncs.com",
-				Help:  "North China 3 (Zhangjiakou)",
-			}, {
-				Value: "oss-cn-huhehaote.aliyuncs.com",
-				Help:  "North China 5 (Huhehaote)",
-			}, {
-				Value: "oss-cn-shenzhen.aliyuncs.com",
-				Help:  "South China 1 (Shenzhen)",
-			}, {
-				Value: "oss-cn-hongkong.aliyuncs.com",
-				Help:  "Hong Kong (Hong Kong)",
-			}, {
-				Value: "oss-us-west-1.aliyuncs.com",
-				Help:  "US West 1 (Silicon Valley)",
-			}, {
-				Value: "oss-us-east-1.aliyuncs.com",
-				Help:  "US East 1 (Virginia)",
-			}, {
-				Value: "oss-ap-southeast-1.aliyuncs.com",
-				Help:  "Southeast Asia Southeast 1 (Singapore)",
-			}, {
-				Value: "oss-ap-southeast-2.aliyuncs.com",
-				Help:  "Asia Pacific Southeast 2 (Sydney)",
-			}, {
-				Value: "oss-ap-southeast-3.aliyuncs.com",
-				Help:  "Southeast Asia Southeast 3 (Kuala Lumpur)",
-			}, {
-				Value: "oss-ap-southeast-5.aliyuncs.com",
-				Help:  "Asia Pacific Southeast 5 (Jakarta)",
-			}, {
-				Value: "oss-ap-northeast-1.aliyuncs.com",
-				Help:  "Asia Pacific Northeast 1 (Japan)",
-			}, {
-				Value: "oss-ap-south-1.aliyuncs.com",
-				Help:  "Asia Pacific South 1 (Mumbai)",
-			}, {
-				Value: "oss-eu-central-1.aliyuncs.com",
-				Help:  "Central Europe 1 (Frankfurt)",
-			}, {
-				Value: "oss-eu-west-1.aliyuncs.com",
-				Help:  "West Europe (London)",
-			}, {
-				Value: "oss-me-east-1.aliyuncs.com",
-				Help:  "Middle East 1 (Dubai)",
-			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,Alibaba",
+			Provider: "!AWS,IBMCOS",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-west-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -518,13 +449,11 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba",
+			Provider: "!AWS,IBMCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
 
-This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
-
 For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Note that this ACL is applied when server side copying objects as S3
@@ -570,28 +499,6 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
 				Help:     "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS",
 				Provider: "IBMCOS",
 			}},
-		}, {
-			Name: "bucket_acl",
-			Help: `Canned ACL used when creating buckets.
-
-For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
-
-Note that this ACL is applied when only when creating buckets.  If it
-isn't set then "acl" is used instead.`,
-			Advanced: true,
-			Examples: []fs.OptionExample{{
-				Value: "private",
-				Help:  "Owner gets FULL_CONTROL. No one else has access rights (default).",
-			}, {
-				Value: "public-read",
-				Help:  "Owner gets FULL_CONTROL. The AllUsers group gets READ access.",
-			}, {
-				Value: "public-read-write",
-				Help:  "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.",
-			}, {
-				Value: "authenticated-read",
-				Help:  "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.",
-			}},
 		}, {
 			Name:     "server_side_encryption",
 			Help:     "The server-side encryption algorithm used when storing this object in S3.",
@@ -640,24 +547,6 @@ isn't set then "acl" is used instead.`,
 				Value: "GLACIER",
 				Help:  "Glacier storage class",
 			}},
-		}, {
-			// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
-			Name:     "storage_class",
-			Help:     "The storage class to use when storing new objects in OSS.",
-			Provider: "Alibaba",
-			Examples: []fs.OptionExample{{
-				Value: "",
-				Help:  "Default",
-			}, {
-				Value: "STANDARD",
-				Help:  "Standard storage class",
-			}, {
-				Value: "GLACIER",
-				Help:  "Archive storage mode.",
-			}, {
-				Value: "STANDARD_IA",
-				Help:  "Infrequent access storage mode.",
-			}},
 		}, {
 			Name: "upload_cutoff",
 			Help: `Cutoff for switching to chunked upload
@@ -751,7 +640,6 @@ type Options struct {
 	Endpoint             string        `config:"endpoint"`
 	LocationConstraint   string        `config:"location_constraint"`
 	ACL                  string        `config:"acl"`
-	BucketACL            string        `config:"bucket_acl"`
 	ServerSideEncryption string        `config:"server_side_encryption"`
 	SSEKMSKeyID          string        `config:"sse_kms_key_id"`
 	StorageClass         string        `config:"storage_class"`
@@ -826,7 +714,7 @@ func (f *Fs) Features() *fs.Features {
 // retryErrorCodes is a slice of error codes that we will retry
 // See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 var retryErrorCodes = []int{
-	// 409, // Conflict - various states that could be resolved on a retry
+	409, // Conflict - various states that could be resolved on a retry
 	503, // Service Unavailable/Slow Down - "Reduce your request rate"
 }
 
@@ -834,13 +722,14 @@ var retryErrorCodes = []int{
 // as it should notice closed connections and timeouts which are the most likely
 // sort of failure modes
 func shouldRetry(err error) (bool, error) {
+
 	// If this is an awserr object, try and extract more useful information to determine if we should retry
 	if awsError, ok := err.(awserr.Error); ok {
 		// Simple case, check the original embedded error in case it's generically retriable
 		if fserrors.ShouldRetry(awsError.OrigErr()) {
 			return true, err
 		}
-		// Failing that, if it's a RequestFailure it's probably got an http status code we can check
+		//Failing that, if it's a RequestFailure it's probably got an http status code we can check
 		if reqErr, ok := err.(awserr.RequestFailure); ok {
 			for _, e := range retryErrorCodes {
 				if reqErr.StatusCode() == e {
@@ -849,7 +738,7 @@ func shouldRetry(err error) (bool, error) {
 			}
 		}
 	}
-	// Ok, not an awserr, check for generic failure conditions
+	//Ok, not an awserr, check for generic failure conditions
 	return fserrors.ShouldRetry(err), err
 }
 
@@ -926,9 +815,6 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
-		opt.ForcePathStyle = false
-	}
 	awsConfig := aws.NewConfig().
 		WithRegion(opt.Region).
 		WithMaxRetries(maxRetries).
@@ -1018,12 +904,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	if opt.ACL == "" {
-		opt.ACL = "private"
-	}
-	if opt.BucketACL == "" {
-		opt.BucketACL = opt.ACL
-	}
 	c, ses, err := s3Connection(opt)
 	if err != nil {
 		return nil, err
@@ -1382,7 +1262,7 @@ func (f *Fs) Mkdir(dir string) error {
 	}
 	req := s3.CreateBucketInput{
 		Bucket: &f.bucket,
-		ACL:    &f.opt.BucketACL,
+		ACL:    &f.opt.ACL,
 	}
 	if f.opt.LocationConstraint != "" {
 		req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
@@ -1401,7 +1281,6 @@ func (f *Fs) Mkdir(dir string) error {
 	if err == nil {
 		f.bucketOK = true
 		f.bucketDeleted = false
-		fs.Infof(f, "Bucket created with ACL %q", *req.ACL)
 	}
 	return err
 }
@@ -1425,7 +1304,6 @@ func (f *Fs) Rmdir(dir string) error {
 	if err == nil {
 		f.bucketOK = false
 		f.bucketDeleted = true
-		fs.Infof(f, "Bucket deleted")
 	}
 	return err
 }

backend/sftp/sftp.go

@@ -66,22 +66,7 @@ func init() {
 			IsPassword: true,
 		}, {
 			Name: "key_file",
-			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
-		}, {
-			Name: "key_file_pass",
-			Help: `The passphrase to decrypt the PEM-encoded private key file.
-
-Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
-in the new OpenSSH format can't be used.`,
-			IsPassword: true,
-		}, {
-			Name: "key_use_agent",
-			Help: `When set forces the usage of the ssh-agent.
-
-When key-file is also set, the ".pub" file of the specified key-file is read and only the associated key is
-requested from the ssh-agent. This allows to avoid ` + "`Too many authentication failures for *username*`" + ` errors
-when the ssh-agent contains many keys.`,
-			Default: false,
+			Help: "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.",
 		}, {
 			Name:    "use_insecure_cipher",
 			Help:    "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
@@ -137,8 +122,6 @@ type Options struct {
 	Port              string `config:"port"`
 	Pass              string `config:"pass"`
 	KeyFile           string `config:"key_file"`
-	KeyFilePass       string `config:"key_file_pass"`
-	KeyUseAgent       bool   `config:"key_use_agent"`
 	UseInsecureCipher bool   `config:"use_insecure_cipher"`
 	DisableHashCheck  bool   `config:"disable_hashcheck"`
 	AskPassword       bool   `config:"ask_password"`
@@ -315,18 +298,6 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 	f.poolMu.Unlock()
 }
 
-// shellExpand replaces a leading "~" with "${HOME}" and expands all environment
-// variables afterwards.
-func shellExpand(s string) string {
-	if s != "" {
-		if s[0] == '~' {
-			s = "${HOME}" + s[1:]
-		}
-		s = os.ExpandEnv(s)
-	}
-	return s
-}
-
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -354,9 +325,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
 	}
 
-	keyFile := shellExpand(opt.KeyFile)
 	// Add ssh agent-auth if no password or file specified
-	if (opt.Pass == "" && keyFile == "") || opt.KeyUseAgent {
+	if opt.Pass == "" && opt.KeyFile == "" {
 		sshAgentClient, _, err := sshagent.New()
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
@@ -365,46 +335,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't read ssh agent signers")
 		}
-		if keyFile != "" {
-			pubBytes, err := ioutil.ReadFile(keyFile + ".pub")
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to read public key file")
-			}
-			pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to parse public key file")
-			}
-			pubM := pub.Marshal()
-			found := false
-			for _, s := range signers {
-				if bytes.Equal(pubM, s.PublicKey().Marshal()) {
-					sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(s))
-					found = true
-					break
-				}
-			}
-			if !found {
-				return nil, errors.New("private key not found in the ssh-agent")
-			}
-		} else {
-			sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
-		}
+		sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
 	}
 
 	// Load key file if specified
-	if keyFile != "" {
-		key, err := ioutil.ReadFile(keyFile)
+	if opt.KeyFile != "" {
+		key, err := ioutil.ReadFile(opt.KeyFile)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read private key file")
 		}
-		clearpass := ""
-		if opt.KeyFilePass != "" {
-			clearpass, err = obscure.Reveal(opt.KeyFilePass)
-			if err != nil {
-				return nil, err
-			}
-		}
-		signer, err := ssh.ParsePrivateKeyWithPassphrase(key, []byte(clearpass))
+		signer, err := ssh.ParsePrivateKey(key)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to parse private key file")
 		}

bin/build-xgo-cgofuse.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-docker build -t rclone/xgo-cgofuse https://github.com/billziss-gh/cgofuse.git
-docker images
-docker push rclone/xgo-cgofuse

cmd/config/config.go

@@ -93,15 +93,6 @@ For example to make a swift remote of name myremote using auto config
 you would do:
 
     rclone config create myremote swift env_auth true
-
-Note that if the config process would normally ask a question the
-default is taken.  Each time that happens rclone will print a message
-saying how to affect the value taken.
-
-So for example if you wanted to configure a Google Drive remote but
-using remote authorization you would do this:
-
-    rclone config create mydrive drive config_is_local false
 `,
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 256, command, args)
@@ -128,11 +119,6 @@ in pairs of <key> <value>.
 For example to update the env_auth field of a remote of name myremote you would do:
 
     rclone config update myremote swift env_auth true
-
-If the remote uses oauth the token will be updated, if you don't
-require this add an extra parameter thus:
-
-    rclone config update myremote swift env_auth true config_refresh_token false
 `,
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(3, 256, command, args)

cmd/mountlib/mount.go

@@ -4,7 +4,6 @@ import (
 	"io"
 	"log"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
@@ -63,28 +62,6 @@ func checkMountEmpty(mountpoint string) error {
 	return nil
 }
 
-// Check the root doesn't overlap the mountpoint
-func checkMountpointOverlap(root, mountpoint string) error {
-	abs := func(x string) string {
-		if absX, err := filepath.EvalSymlinks(x); err == nil {
-			x = absX
-		}
-		if absX, err := filepath.Abs(x); err == nil {
-			x = absX
-		}
-		x = filepath.ToSlash(x)
-		if !strings.HasSuffix(x, "/") {
-			x += "/"
-		}
-		return x
-	}
-	rootAbs, mountpointAbs := abs(root), abs(mountpoint)
-	if strings.HasPrefix(rootAbs, mountpointAbs) || strings.HasPrefix(mountpointAbs, rootAbs) {
-		return errors.Errorf("mount point %q and directory to be mounted %q mustn't overlap", mountpoint, root)
-	}
-	return nil
-}
-
 // NewMountCommand makes a mount command with the given name and Mount function
 func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
 	var commandDefintion = &cobra.Command{
@@ -243,14 +220,7 @@ be copied to the vfs cache before opening with --vfs-cache-mode full.
 				config.PassConfigKeyForDaemonization = true
 			}
 
-			mountpoint := args[1]
 			fdst := cmd.NewFsDir(args)
-			if fdst.Name() == "" || fdst.Name() == "local" {
-				err := checkMountpointOverlap(fdst.Root(), mountpoint)
-				if err != nil {
-					log.Fatalf("Fatal error: %v", err)
-				}
-			}
 
 			// Show stats if the user has specifically requested them
 			if cmd.ShowStats() {
@@ -260,7 +230,7 @@ be copied to the vfs cache before opening with --vfs-cache-mode full.
 			// Skip checkMountEmpty if --allow-non-empty flag is used or if
 			// the Operating System is Windows
 			if !AllowNonEmpty && runtime.GOOS != "windows" {
-				err := checkMountEmpty(mountpoint)
+				err := checkMountEmpty(args[1])
 				if err != nil {
 					log.Fatalf("Fatal error: %v", err)
 				}
@@ -283,7 +253,7 @@ be copied to the vfs cache before opening with --vfs-cache-mode full.
 				}
 			}
 
-			err := Mount(fdst, mountpoint)
+			err := Mount(fdst, args[1])
 			if err != nil {
 				log.Fatalf("Fatal error: %v", err)
 			}

cmd/serve/dlna/cd-service-desc.go

@@ -1,451 +0,0 @@
-package dlna
-
-const contentDirectoryServiceDescription = `<?xml version="1.0"?>
-<scpd xmlns="urn:schemas-upnp-org:service-1-0">
-  <specVersion>
-    <major>1</major>
-    <minor>0</minor>
-  </specVersion>
-  <actionList>
-    <action>
-      <name>GetSearchCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SearchCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SearchCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSortCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SortCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SortCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSortExtensionCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SortExtensionCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SortExtensionCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetFeatureList</name>
-      <argumentList>
-        <argument>
-          <name>FeatureList</name>
-          <direction>out</direction>
-          <relatedStateVariable>FeatureList</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSystemUpdateID</name>
-      <argumentList>
-        <argument>
-          <name>Id</name>
-          <direction>out</direction>
-          <relatedStateVariable>SystemUpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>Browse</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>BrowseFlag</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_BrowseFlag</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Filter</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>StartingIndex</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>RequestedCount</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SortCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NumberReturned</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TotalMatches</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>UpdateID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>Search</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SearchCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SearchCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Filter</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>StartingIndex</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>RequestedCount</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SortCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NumberReturned</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TotalMatches</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>UpdateID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>CreateObject</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Elements</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>ObjectID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>DestroyObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>UpdateObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>CurrentTagValue</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewTagValue</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>MoveObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewParentID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewObjectID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>ImportResource</name>
-      <argumentList>
-        <argument>
-          <name>SourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>DestinationURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>ExportResource</name>
-      <argumentList>
-        <argument>
-          <name>SourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>DestinationURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>StopTransferResource</name>
-      <argumentList>
-        <argument>
-          <name>TransferID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>DeleteResource</name>
-      <argumentList>
-        <argument>
-          <name>ResourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetTransferProgress</name>
-      <argumentList>
-        <argument>
-          <name>TransferID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferStatus</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferStatus</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferLength</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferLength</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferTotal</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferTotal</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>CreateReference</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-  </actionList>
-  <serviceStateTable>
-    <stateVariable sendEvents="no">
-      <name>SearchCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>SortCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>SortExtensionCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>SystemUpdateID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>ContainerUpdateIDs</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>TransferIDs</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>FeatureList</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_ObjectID</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Result</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_SearchCriteria</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_BrowseFlag</name>
-      <dataType>string</dataType>
-      <allowedValueList>
-        <allowedValue>BrowseMetadata</allowedValue>
-        <allowedValue>BrowseDirectChildren</allowedValue>
-      </allowedValueList>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Filter</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_SortCriteria</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Index</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Count</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_UpdateID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferStatus</name>
-      <dataType>string</dataType>
-      <allowedValueList>
-        <allowedValue>COMPLETED</allowedValue>
-        <allowedValue>ERROR</allowedValue>
-        <allowedValue>IN_PROGRESS</allowedValue>
-        <allowedValue>STOPPED</allowedValue>
-      </allowedValueList>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferLength</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferTotal</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TagValueList</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_URI</name>
-      <dataType>uri</dataType>
-    </stateVariable>
-  </serviceStateTable>
-</scpd>`

cmd/serve/dlna/cds.go

@@ -1,240 +0,0 @@
-package dlna
-
-import (
-	"encoding/xml"
-	"fmt"
-	"log"
-	"net/http"
-	"net/url"
-	"os"
-	"path"
-	"path/filepath"
-	"sort"
-
-	"github.com/anacrolix/dms/dlna"
-	"github.com/anacrolix/dms/upnp"
-	"github.com/anacrolix/dms/upnpav"
-	"github.com/ncw/rclone/vfs"
-	"github.com/pkg/errors"
-)
-
-type contentDirectoryService struct {
-	*server
-	upnp.Eventing
-}
-
-func (cds *contentDirectoryService) updateIDString() string {
-	return fmt.Sprintf("%d", uint32(os.Getpid()))
-}
-
-// Turns the given entry and DMS host into a UPnP object. A nil object is
-// returned if the entry is not of interest.
-func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fileInfo os.FileInfo, host string) (ret interface{}, err error) {
-	obj := upnpav.Object{
-		ID:         cdsObject.ID(),
-		Restricted: 1,
-		ParentID:   cdsObject.ParentID(),
-	}
-
-	if fileInfo.IsDir() {
-		obj.Class = "object.container.storageFolder"
-		obj.Title = fileInfo.Name()
-		ret = upnpav.Container{Object: obj}
-		return
-	}
-
-	if !fileInfo.Mode().IsRegular() {
-		return
-	}
-
-	// Hardcode "videoItem" so that files show up in VLC.
-	obj.Class = "object.item.videoItem"
-	obj.Title = fileInfo.Name()
-
-	item := upnpav.Item{
-		Object: obj,
-		Res:    make([]upnpav.Resource, 0, 1),
-	}
-
-	item.Res = append(item.Res, upnpav.Resource{
-		URL: (&url.URL{
-			Scheme: "http",
-			Host:   host,
-			Path:   resPath,
-			RawQuery: url.Values{
-				"path": {cdsObject.Path},
-			}.Encode(),
-		}).String(),
-		// Hardcode "video/x-matroska" so that files show up in VLC.
-		ProtocolInfo: fmt.Sprintf("http-get:*:video/x-matroska:%s", dlna.ContentFeatures{
-			SupportRange: true,
-		}.String()),
-		Bitrate:    0,
-		Duration:   "",
-		Size:       uint64(fileInfo.Size()),
-		Resolution: "",
-	})
-
-	ret = item
-	return
-}
-
-// Returns all the upnpav objects in a directory.
-func (cds *contentDirectoryService) readContainer(o object, host string) (ret []interface{}, err error) {
-	node, err := cds.vfs.Stat(o.Path)
-	if err != nil {
-		return
-	}
-
-	if !node.IsDir() {
-		err = errors.New("not a directory")
-		return
-	}
-
-	dir := node.(*vfs.Dir)
-	dirEntries, err := dir.ReadDirAll()
-	if err != nil {
-		err = errors.New("failed to list directory")
-		return
-	}
-
-	sort.Sort(dirEntries)
-
-	for _, de := range dirEntries {
-		child := object{
-			path.Join(o.Path, de.Name()),
-		}
-		obj, err := cds.cdsObjectToUpnpavObject(child, de, host)
-		if err != nil {
-			log.Printf("error with %s: %s", child.FilePath(), err)
-			continue
-		}
-		if obj != nil {
-			ret = append(ret, obj)
-		} else {
-			log.Printf("bad %s", de)
-		}
-	}
-
-	return
-}
-
-type browse struct {
-	ObjectID       string
-	BrowseFlag     string
-	Filter         string
-	StartingIndex  int
-	RequestedCount int
-}
-
-// ContentDirectory object from ObjectID.
-func (cds *contentDirectoryService) objectFromID(id string) (o object, err error) {
-	o.Path, err = url.QueryUnescape(id)
-	if err != nil {
-		return
-	}
-	if o.Path == "0" {
-		o.Path = "/"
-	}
-	o.Path = path.Clean(o.Path)
-	if !path.IsAbs(o.Path) {
-		err = fmt.Errorf("bad ObjectID %v", o.Path)
-		return
-	}
-	return
-}
-
-func (cds *contentDirectoryService) Handle(action string, argsXML []byte, r *http.Request) (map[string]string, error) {
-	host := r.Host
-
-	switch action {
-	case "GetSystemUpdateID":
-		return map[string]string{
-			"Id": cds.updateIDString(),
-		}, nil
-	case "GetSortCapabilities":
-		return map[string]string{
-			"SortCaps": "dc:title",
-		}, nil
-	case "Browse":
-		var browse browse
-		if err := xml.Unmarshal([]byte(argsXML), &browse); err != nil {
-			return nil, err
-		}
-		obj, err := cds.objectFromID(browse.ObjectID)
-		if err != nil {
-			return nil, upnp.Errorf(upnpav.NoSuchObjectErrorCode, err.Error())
-		}
-		switch browse.BrowseFlag {
-		case "BrowseDirectChildren":
-			objs, err := cds.readContainer(obj, host)
-			if err != nil {
-				return nil, upnp.Errorf(upnpav.NoSuchObjectErrorCode, err.Error())
-			}
-			totalMatches := len(objs)
-			objs = objs[func() (low int) {
-				low = browse.StartingIndex
-				if low > len(objs) {
-					low = len(objs)
-				}
-				return
-			}():]
-			if browse.RequestedCount != 0 && int(browse.RequestedCount) < len(objs) {
-				objs = objs[:browse.RequestedCount]
-			}
-			result, err := xml.Marshal(objs)
-			if err != nil {
-				return nil, err
-			}
-			return map[string]string{
-				"TotalMatches":   fmt.Sprint(totalMatches),
-				"NumberReturned": fmt.Sprint(len(objs)),
-				"Result":         didlLite(string(result)),
-				"UpdateID":       cds.updateIDString(),
-			}, nil
-		default:
-			return nil, upnp.Errorf(upnp.ArgumentValueInvalidErrorCode, "unhandled browse flag: %v", browse.BrowseFlag)
-		}
-	case "GetSearchCapabilities":
-		return map[string]string{
-			"SearchCaps": "",
-		}, nil
-	default:
-		return nil, upnp.InvalidActionError
-	}
-}
-
-// Represents a ContentDirectory object.
-type object struct {
-	Path string // The cleaned, absolute path for the object relative to the server.
-}
-
-// Returns the actual local filesystem path for the object.
-func (o *object) FilePath() string {
-	return filepath.FromSlash(o.Path)
-}
-
-// Returns the ObjectID for the object. This is used in various ContentDirectory actions.
-func (o object) ID() string {
-	if !path.IsAbs(o.Path) {
-		log.Panicf("Relative object path: %s", o.Path)
-	}
-	if len(o.Path) == 1 {
-		return "0"
-	}
-	return url.QueryEscape(o.Path)
-}
-
-func (o *object) IsRoot() bool {
-	return o.Path == "/"
-}
-
-// Returns the object's parent ObjectID. Fortunately it can be deduced from the
-// ObjectID (for now).
-func (o object) ParentID() string {
-	if o.IsRoot() {
-		return "-1"
-	}
-	o.Path = path.Dir(o.Path)
-	return o.ID()
-}

cmd/serve/dlna/dlna.go

@@ -1,440 +0,0 @@
-package dlna
-
-import (
-	"bytes"
-	"encoding/xml"
-	"fmt"
-	"log"
-	"net"
-	"net/http"
-	"net/url"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/anacrolix/dms/soap"
-	"github.com/anacrolix/dms/ssdp"
-	"github.com/anacrolix/dms/upnp"
-	"github.com/ncw/rclone/cmd"
-	"github.com/ncw/rclone/cmd/serve/dlna/dlnaflags"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/vfs"
-	"github.com/ncw/rclone/vfs/vfsflags"
-	"github.com/spf13/cobra"
-)
-
-func init() {
-	dlnaflags.AddFlags(Command.Flags())
-	vfsflags.AddFlags(Command.Flags())
-}
-
-// Command definition for cobra.
-var Command = &cobra.Command{
-	Use:   "dlna remote:path",
-	Short: `Serve remote:path over DLNA`,
-	Long: `rclone serve dlna is a DLNA media server for media stored in a rclone remote. Many
-devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN
-and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast
-packets (SSDP) and will thus only work on LANs.
-
-Rclone will list all files present in the remote, without filtering based on media formats or
-file extensions. Additionally, there is no media transcoding support. This means that some
-players might show files that they are not able to play back correctly.
-
-` + dlnaflags.Help + vfs.Help,
-	Run: func(command *cobra.Command, args []string) {
-		cmd.CheckArgs(1, 1, command, args)
-		f := cmd.NewFsSrc(args)
-
-		cmd.Run(false, false, command, func() error {
-			s := newServer(f, &dlnaflags.Opt)
-			if err := s.Serve(); err != nil {
-				log.Fatal(err)
-			}
-			s.Wait()
-			return nil
-		})
-	},
-}
-
-const (
-	serverField         = "Linux/3.4 DLNADOC/1.50 UPnP/1.0 DMS/1.0"
-	rootDeviceType      = "urn:schemas-upnp-org:device:MediaServer:1"
-	rootDeviceModelName = "rclone"
-	resPath             = "/res"
-	rootDescPath        = "/rootDesc.xml"
-	serviceControlURL   = "/ctl"
-)
-
-// Groups the service definition with its XML description.
-type service struct {
-	upnp.Service
-	SCPD string
-}
-
-// Exposed UPnP AV services.
-var services = []*service{
-	{
-		Service: upnp.Service{
-			ServiceType: "urn:schemas-upnp-org:service:ContentDirectory:1",
-			ServiceId:   "urn:upnp-org:serviceId:ContentDirectory",
-			ControlURL:  serviceControlURL,
-		},
-		SCPD: contentDirectoryServiceDescription,
-	},
-}
-
-func devices() []string {
-	return []string{
-		"urn:schemas-upnp-org:device:MediaServer:1",
-	}
-}
-
-func serviceTypes() (ret []string) {
-	for _, s := range services {
-		ret = append(ret, s.ServiceType)
-	}
-	return
-}
-
-type server struct {
-	// The service SOAP handler keyed by service URN.
-	services map[string]UPnPService
-
-	Interfaces []net.Interface
-
-	HTTPConn       net.Listener
-	httpListenAddr string
-	httpServeMux   *http.ServeMux
-
-	rootDeviceUUID string
-	rootDescXML    []byte
-
-	FriendlyName string
-
-	// For waiting on the listener to close
-	waitChan chan struct{}
-
-	// Time interval between SSPD announces
-	AnnounceInterval time.Duration
-
-	f   fs.Fs
-	vfs *vfs.VFS
-}
-
-func newServer(f fs.Fs, opt *dlnaflags.Options) *server {
-	hostName, err := os.Hostname()
-	if err != nil {
-		hostName = ""
-	} else {
-		hostName = " (" + hostName + ")"
-	}
-
-	s := &server{
-		AnnounceInterval: 10 * time.Second,
-		FriendlyName:     "rclone" + hostName,
-
-		httpListenAddr: opt.ListenAddr,
-
-		f:   f,
-		vfs: vfs.New(f, &vfsflags.Opt),
-	}
-
-	s.initServicesMap()
-	s.listInterfaces()
-
-	s.httpServeMux = http.NewServeMux()
-	s.rootDeviceUUID = makeDeviceUUID(s.FriendlyName)
-	s.rootDescXML, err = xml.MarshalIndent(
-		upnp.DeviceDesc{
-			SpecVersion: upnp.SpecVersion{Major: 1, Minor: 0},
-			Device: upnp.Device{
-				DeviceType:   rootDeviceType,
-				FriendlyName: s.FriendlyName,
-				Manufacturer: "rclone (rclone.org)",
-				ModelName:    rootDeviceModelName,
-				UDN:          s.rootDeviceUUID,
-				ServiceList: func() (ss []upnp.Service) {
-					for _, s := range services {
-						ss = append(ss, s.Service)
-					}
-					return
-				}(),
-			},
-		},
-		" ", "  ")
-	if err != nil {
-		// Contents are hardcoded, so this will never happen in production.
-		log.Panicf("Marshal root descriptor XML: %v", err)
-	}
-	s.rootDescXML = append([]byte(`<?xml version="1.0"?>`), s.rootDescXML...)
-	s.initMux(s.httpServeMux)
-
-	return s
-}
-
-// UPnPService is the interface for the SOAP service.
-type UPnPService interface {
-	Handle(action string, argsXML []byte, r *http.Request) (respArgs map[string]string, err error)
-	Subscribe(callback []*url.URL, timeoutSeconds int) (sid string, actualTimeout int, err error)
-	Unsubscribe(sid string) error
-}
-
-// initServicesMap is called during initialization of the server to prepare some internal datastructures.
-func (s *server) initServicesMap() {
-	urn, err := upnp.ParseServiceType(services[0].ServiceType)
-	if err != nil {
-		// The service type is hardcoded, so this error should never happen.
-		log.Panicf("ParseServiceType: %v", err)
-	}
-	s.services = map[string]UPnPService{
-		urn.Type: &contentDirectoryService{
-			server: s,
-		},
-	}
-	return
-}
-
-// listInterfaces is called during initialization of the server to list the network interfaces
-// on the machine.
-func (s *server) listInterfaces() {
-	ifs, err := net.Interfaces()
-	if err != nil {
-		fs.Errorf(s.f, "list network interfaces: %v", err)
-		return
-	}
-
-	var tmp []net.Interface
-	for _, intf := range ifs {
-		if intf.Flags&net.FlagUp == 0 || intf.MTU <= 0 {
-			continue
-		}
-		s.Interfaces = append(s.Interfaces, intf)
-		tmp = append(tmp, intf)
-	}
-}
-
-func (s *server) initMux(mux *http.ServeMux) {
-	mux.HandleFunc(resPath, func(w http.ResponseWriter, r *http.Request) {
-		remotePath := r.URL.Query().Get("path")
-		node, err := s.vfs.Stat(remotePath)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
-
-		file := node.(*vfs.File)
-		in, err := file.Open(os.O_RDONLY)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-		}
-		defer fs.CheckClose(in, &err)
-
-		http.ServeContent(w, r, remotePath, node.ModTime(), in)
-		return
-	})
-
-	mux.HandleFunc(rootDescPath, func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("content-type", `text/xml; charset="utf-8"`)
-		w.Header().Set("content-length", fmt.Sprint(len(s.rootDescXML)))
-		w.Header().Set("server", serverField)
-		_, err := w.Write(s.rootDescXML)
-		if err != nil {
-			fs.Errorf(s, "Failed to serve root descriptor XML: %v", err)
-		}
-	})
-
-	// Install handlers to serve SCPD for each UPnP service.
-	for _, s := range services {
-		p := path.Join("/scpd", s.ServiceId)
-		s.SCPDURL = p
-
-		mux.HandleFunc(s.SCPDURL, func(serviceDesc string) http.HandlerFunc {
-			return func(w http.ResponseWriter, r *http.Request) {
-				w.Header().Set("content-type", `text/xml; charset="utf-8"`)
-				http.ServeContent(w, r, ".xml", time.Time{}, bytes.NewReader([]byte(serviceDesc)))
-			}
-		}(s.SCPD))
-	}
-
-	mux.HandleFunc(serviceControlURL, s.serviceControlHandler)
-}
-
-// Handle a service control HTTP request.
-func (s *server) serviceControlHandler(w http.ResponseWriter, r *http.Request) {
-	soapActionString := r.Header.Get("SOAPACTION")
-	soapAction, err := upnp.ParseActionHTTPHeader(soapActionString)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-	var env soap.Envelope
-	if err := xml.NewDecoder(r.Body).Decode(&env); err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	w.Header().Set("Content-Type", `text/xml; charset="utf-8"`)
-	w.Header().Set("Ext", "")
-	w.Header().Set("server", serverField)
-	soapRespXML, code := func() ([]byte, int) {
-		respArgs, err := s.soapActionResponse(soapAction, env.Body.Action, r)
-		if err != nil {
-			upnpErr := upnp.ConvertError(err)
-			return mustMarshalXML(soap.NewFault("UPnPError", upnpErr)), 500
-		}
-		return marshalSOAPResponse(soapAction, respArgs), 200
-	}()
-	bodyStr := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" standalone="yes"?><s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"><s:Body>%s</s:Body></s:Envelope>`, soapRespXML)
-	w.WriteHeader(code)
-	if _, err := w.Write([]byte(bodyStr)); err != nil {
-		log.Print(err)
-	}
-}
-
-// Handle a SOAP request and return the response arguments or UPnP error.
-func (s *server) soapActionResponse(sa upnp.SoapAction, actionRequestXML []byte, r *http.Request) (map[string]string, error) {
-	service, ok := s.services[sa.Type]
-	if !ok {
-		// TODO: What's the invalid service error?
-		return nil, upnp.Errorf(upnp.InvalidActionErrorCode, "Invalid service: %s", sa.Type)
-	}
-	return service.Handle(sa.Action, actionRequestXML, r)
-}
-
-// Serve runs the server - returns the error only if
-// the listener was not started; does not block, so
-// use s.Wait() to block on the listener indefinitely.
-func (s *server) Serve() (err error) {
-	if s.HTTPConn == nil {
-		s.HTTPConn, err = net.Listen("tcp", s.httpListenAddr)
-		if err != nil {
-			return
-		}
-	}
-
-	go func() {
-		s.startSSDP()
-	}()
-
-	go func() {
-		fs.Logf(s.f, "Serving HTTP on %s", s.HTTPConn.Addr().String())
-
-		err = s.serveHTTP()
-		if err != nil {
-			fs.Logf(s.f, "Error on serving HTTP server: %v", err)
-		}
-	}()
-
-	return nil
-}
-
-// Wait blocks while the listener is open.
-func (s *server) Wait() {
-	<-s.waitChan
-}
-
-func (s *server) Close() {
-	err := s.HTTPConn.Close()
-	if err != nil {
-		fs.Errorf(s.f, "Error closing HTTP server: %v", err)
-		return
-	}
-	close(s.waitChan)
-}
-
-// Run SSDP (multicast for server discovery) on all interfaces.
-func (s *server) startSSDP() {
-	active := 0
-	stopped := make(chan struct{})
-	for _, intf := range s.Interfaces {
-		active++
-		go func(intf2 net.Interface) {
-			defer func() {
-				stopped <- struct{}{}
-			}()
-			s.ssdpInterface(intf2)
-		}(intf)
-	}
-	for active > 0 {
-		<-stopped
-		active--
-	}
-}
-
-// Run SSDP server on an interface.
-func (s *server) ssdpInterface(intf net.Interface) {
-	// Figure out which HTTP location to advertise based on the interface IP.
-	advertiseLocationFn := func(ip net.IP) string {
-		url := url.URL{
-			Scheme: "http",
-			Host: (&net.TCPAddr{
-				IP:   ip,
-				Port: s.HTTPConn.Addr().(*net.TCPAddr).Port,
-			}).String(),
-			Path: rootDescPath,
-		}
-		return url.String()
-	}
-
-	ssdpServer := ssdp.Server{
-		Interface:      intf,
-		Devices:        devices(),
-		Services:       serviceTypes(),
-		Location:       advertiseLocationFn,
-		Server:         serverField,
-		UUID:           s.rootDeviceUUID,
-		NotifyInterval: s.AnnounceInterval,
-	}
-
-	// An interface with these flags should be valid for SSDP.
-	const ssdpInterfaceFlags = net.FlagUp | net.FlagMulticast
-
-	if err := ssdpServer.Init(); err != nil {
-		if intf.Flags&ssdpInterfaceFlags != ssdpInterfaceFlags {
-			// Didn't expect it to work anyway.
-			return
-		}
-		if strings.Contains(err.Error(), "listen") {
-			// OSX has a lot of dud interfaces. Failure to create a socket on
-			// the interface are what we're expecting if the interface is no
-			// good.
-			return
-		}
-		log.Printf("Error creating ssdp server on %s: %s", intf.Name, err)
-		return
-	}
-	defer ssdpServer.Close()
-	log.Println("Started SSDP on", intf.Name)
-	stopped := make(chan struct{})
-	go func() {
-		defer close(stopped)
-		if err := ssdpServer.Serve(); err != nil {
-			log.Printf("%q: %q\n", intf.Name, err)
-		}
-	}()
-	select {
-	case <-s.waitChan:
-		// Returning will close the server.
-	case <-stopped:
-	}
-}
-
-func (s *server) serveHTTP() error {
-	srv := &http.Server{
-		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			s.httpServeMux.ServeHTTP(w, r)
-		}),
-	}
-	err := srv.Serve(s.HTTPConn)
-	select {
-	case <-s.waitChan:
-		return nil
-	default:
-		return err
-	}
-}

cmd/serve/dlna/dlna_test.go

@@ -1,88 +0,0 @@
-// +build go1.8
-
-package dlna
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"os"
-	"testing"
-
-	"github.com/ncw/rclone/vfs"
-
-	_ "github.com/ncw/rclone/backend/local"
-	"github.com/ncw/rclone/cmd/serve/dlna/dlnaflags"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-var (
-	dlnaServer *server
-)
-
-const (
-	testBindAddress = "localhost:51777"
-	testURL         = "http://" + testBindAddress + "/"
-)
-
-func startServer(t *testing.T, f fs.Fs) {
-	opt := dlnaflags.DefaultOpt
-	opt.ListenAddr = testBindAddress
-	dlnaServer = newServer(f, &opt)
-	assert.NoError(t, dlnaServer.Serve())
-}
-
-func TestInit(t *testing.T) {
-	config.LoadConfig()
-
-	f, err := fs.NewFs("testdata/files")
-	l, _ := f.List("")
-	fmt.Println(l)
-	require.NoError(t, err)
-
-	startServer(t, f)
-}
-
-// Make sure that it serves rootDesc.xml (SCPD in uPnP parlance).
-func TestRootSCPD(t *testing.T) {
-	req, err := http.NewRequest("GET", testURL+"rootDesc.xml", nil)
-	require.NoError(t, err)
-	resp, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
-	require.NoError(t, err)
-	// Make sure that the SCPD contains a CDS service.
-	require.Contains(t, string(body),
-		"<serviceType>urn:schemas-upnp-org:service:ContentDirectory:1</serviceType>")
-}
-
-// Make sure that it serves content from the remote.
-func TestServeContent(t *testing.T) {
-	itemPath := "/small_jpeg.jpg"
-	pathQuery := url.QueryEscape(itemPath)
-	req, err := http.NewRequest("GET", testURL+"res?path="+pathQuery, nil)
-	require.NoError(t, err)
-	resp, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-	defer fs.CheckClose(resp.Body, &err)
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	actualContents, err := ioutil.ReadAll(resp.Body)
-	assert.NoError(t, err)
-
-	// Now compare the contents with the golden file.
-	node, err := dlnaServer.vfs.Stat(itemPath)
-	assert.NoError(t, err)
-	goldenFile := node.(*vfs.File)
-	goldenReader, err := goldenFile.Open(os.O_RDONLY)
-	assert.NoError(t, err)
-	defer fs.CheckClose(goldenReader, &err)
-	goldenContents, err := ioutil.ReadAll(goldenReader)
-	assert.NoError(t, err)
-
-	require.Equal(t, goldenContents, actualContents)
-}

cmd/serve/dlna/dlna_util.go

@@ -1,52 +0,0 @@
-package dlna
-
-import (
-	"crypto/md5"
-	"encoding/xml"
-	"fmt"
-	"io"
-	"log"
-
-	"github.com/anacrolix/dms/soap"
-	"github.com/anacrolix/dms/upnp"
-)
-
-func makeDeviceUUID(unique string) string {
-	h := md5.New()
-	if _, err := io.WriteString(h, unique); err != nil {
-		log.Panicf("makeDeviceUUID write failed: %s", err)
-	}
-	buf := h.Sum(nil)
-	return upnp.FormatUUID(buf)
-}
-
-func didlLite(chardata string) string {
-	return `<DIDL-Lite` +
-		` xmlns:dc="http://purl.org/dc/elements/1.1/"` +
-		` xmlns:upnp="urn:schemas-upnp-org:metadata-1-0/upnp/"` +
-		` xmlns="urn:schemas-upnp-org:metadata-1-0/DIDL-Lite/"` +
-		` xmlns:dlna="urn:schemas-dlna-org:metadata-1-0/">` +
-		chardata +
-		`</DIDL-Lite>`
-}
-
-func mustMarshalXML(value interface{}) []byte {
-	ret, err := xml.MarshalIndent(value, "", "  ")
-	if err != nil {
-		log.Panicf("mustMarshalXML failed to marshal %v: %s", value, err)
-	}
-	return ret
-}
-
-// Marshal SOAP response arguments into a response XML snippet.
-func marshalSOAPResponse(sa upnp.SoapAction, args map[string]string) []byte {
-	soapArgs := make([]soap.Arg, 0, len(args))
-	for argName, value := range args {
-		soapArgs = append(soapArgs, soap.Arg{
-			XMLName: xml.Name{Local: argName},
-			Value:   value,
-		})
-	}
-	return []byte(fmt.Sprintf(`<u:%[1]sResponse xmlns:u="%[2]s">%[3]s</u:%[1]sResponse>`,
-		sa.Action, sa.ServiceURN.String(), mustMarshalXML(soapArgs)))
-}

cmd/serve/dlna/dlnaflags/dlnaflags.go

@@ -1,42 +0,0 @@
-package dlnaflags
-
-import (
-	"github.com/ncw/rclone/fs/config/flags"
-	"github.com/ncw/rclone/fs/rc"
-	"github.com/spf13/pflag"
-)
-
-// Help contains the text for the command line help and manual.
-var Help = `
-### Server options
-
-Use --addr to specify which IP address and port the server should
-listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
-IPs.
-
-`
-
-// Options is the type for DLNA serving options.
-type Options struct {
-	ListenAddr string
-}
-
-// DefaultOpt contains the defaults options for DLNA serving.
-var DefaultOpt = Options{
-	ListenAddr: ":7879",
-}
-
-// Opt contains the options for DLNA serving.
-var (
-	Opt = DefaultOpt
-)
-
-func addFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) {
-	rc.AddOption("dlna", &Opt)
-	flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "ip:port or :port to bind the DLNA http server to.")
-}
-
-// AddFlags add the command line flags for DLNA serving.
-func AddFlags(flagSet *pflag.FlagSet) {
-	addFlagsPrefix(flagSet, "", &Opt)
-}

cmd/serve/serve.go

@@ -3,8 +3,6 @@ package serve
 import (
 	"errors"
 
-	"github.com/ncw/rclone/cmd/serve/dlna"
-
 	"github.com/ncw/rclone/cmd"
 	"github.com/ncw/rclone/cmd/serve/ftp"
 	"github.com/ncw/rclone/cmd/serve/http"
@@ -21,9 +19,6 @@ func init() {
 	if restic.Command != nil {
 		Command.AddCommand(restic.Command)
 	}
-	if dlna.Command != nil {
-		Command.AddCommand(dlna.Command)
-	}
 	if ftp.Command != nil {
 		Command.AddCommand(ftp.Command)
 	}

docs/content/about.md

@@ -13,7 +13,6 @@ Rclone
 
 Rclone is a command line program to sync files and directories to and from:
 
-* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
 * {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}} ([See note](/amazonclouddrive/#status))
 * {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
 * {{< provider name="Backblaze B2" home="https://www.backblaze.com/b2/cloud-storage.html" config="/b2/" >}}

docs/content/authors.md

@@ -225,6 +225,3 @@ Contributors
   * Arkadius Stefanski <arkste@gmail.com>
   * Jay <dev@jaygoel.com>
   * andrea rota <a@xelera.eu>
-  * nicolov <nicolov@users.noreply.github.com>
-  * Dario Guzik <dario@guzik.com.ar>
-  * qip <qip@users.noreply.github.com>

docs/content/docs.md

@@ -437,8 +437,8 @@ Normally the config file is in your home directory as a file called
 older version). If `$XDG_CONFIG_HOME` is set it will be at
 `$XDG_CONFIG_HOME/rclone/rclone.conf`
 
-If you run `rclone config file` you will see where the default
-location is for you.
+If you run `rclone -h` and look at the help for the `--config` option
+you will see where the default location is for you.
 
 Use this flag to override the config location, eg `rclone
 --config=".myconfig" .config`.

docs/content/faq.md

@@ -15,8 +15,8 @@ work on all the remote storage systems.
 ### Can I copy the config from one machine to another ###
 
 Sure!  Rclone stores all of its config in a single file.  If you want
-to find this file, run `rclone config file` which will tell you where
-it is.
+to find this file, the simplest way is to run `rclone -h` and look at
+the help for the `--config` flag which will tell you where it is.
 
 See the [remote setup docs](/remote_setup/) for more info.
 

docs/content/qingstor.md

@@ -234,50 +234,4 @@ Number of connection retries.
 - Type:        int
 - Default:     3
 
-#### --qingstor-upload-cutoff
-
-Cutoff for switching to chunked upload
-
-Any files larger than this will be uploaded in chunks of chunk_size.
-The minimum is 0 and the maximum is 5GB.
-
-- Config:      upload_cutoff
-- Env Var:     RCLONE_QINGSTOR_UPLOAD_CUTOFF
-- Type:        SizeSuffix
-- Default:     200M
-
-#### --qingstor-chunk-size
-
-Chunk size to use for uploading.
-
-When uploading files larger than upload_cutoff they will be uploaded
-as multipart uploads using this chunk size.
-
-Note that "--qingstor-upload-concurrency" chunks of this size are buffered
-in memory per transfer.
-
-If you are transferring large files over high speed links and you have
-enough memory, then increasing this will speed up the transfers.
-
-- Config:      chunk_size
-- Env Var:     RCLONE_QINGSTOR_CHUNK_SIZE
-- Type:        SizeSuffix
-- Default:     4M
-
-#### --qingstor-upload-concurrency
-
-Concurrency for multipart uploads.
-
-This is the number of chunks of the same file that are uploaded
-concurrently.
-
-If you are uploading small numbers of large file over high speed link
-and these uploads do not fully utilize your bandwidth, then increasing
-this may help to speed up the transfers.
-
-- Config:      upload_concurrency
-- Env Var:     RCLONE_QINGSTOR_UPLOAD_CONCURRENCY
-- Type:        int
-- Default:     4
-
 <!--- autogenerated options stop -->

docs/content/remote_setup.md

@@ -74,14 +74,15 @@ So first configure rclone on your desktop machine
 
 to set up the config file.
 
-Find the config file by running `rclone config file`, for example
+Find the config file by running `rclone -h` and looking for the help for the `--config` option
 
 ```
-$ rclone config file
-Configuration file is stored at:
-/home/user/.rclone.conf
+$ rclone -h
+[snip]
+      --config="/home/user/.rclone.conf": Config file.
+[snip]
 ```
 
 Now transfer it to the remote box (scp, cut paste, ftp, sftp etc) and
-place it in the correct place (use `rclone config file` on the remote
-box to find out where).
+place it in the correct place (use `rclone -h` on the remote box to
+find out where).

docs/content/s3.md

@@ -10,7 +10,6 @@ date: "2016-07-11"
 The S3 backend can be used with a number of different providers:
 
 * {{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/#amazon-s3" >}}
-* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
 * {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
 * {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 * {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@@ -401,7 +400,7 @@ the object(s) in question before using rclone.
 <!--- autogenerated options start - DO NOT EDIT, instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs -->
 ### Standard Options
 
-Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
+Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)).
 
 #### --s3-provider
 
@@ -414,8 +413,6 @@ Choose your S3 provider.
 - Examples:
     - "AWS"
         - Amazon Web Services (AWS) S3
-    - "Alibaba"
-        - Alibaba Cloud Object Storage System (OSS) formerly Aliyun
     - "Ceph"
         - Ceph Object Storage
     - "DigitalOcean"
@@ -426,8 +423,6 @@ Choose your S3 provider.
         - IBM COS S3
     - "Minio"
         - Minio Object Storage
-    - "Netease"
-        - Netease Object Storage (NOS)
     - "Wasabi"
         - Wasabi Object Storage
     - "Other"
@@ -627,54 +622,6 @@ Specify if using an IBM COS On Premise.
 
 #### --s3-endpoint
 
-Endpoint for OSS API.
-
-- Config:      endpoint
-- Env Var:     RCLONE_S3_ENDPOINT
-- Type:        string
-- Default:     ""
-- Examples:
-    - "oss-cn-hangzhou.aliyuncs.com"
-        - East China 1 (Hangzhou)
-    - "oss-cn-shanghai.aliyuncs.com"
-        - East China 2 (Shanghai)
-    - "oss-cn-qingdao.aliyuncs.com"
-        - North China 1 (Qingdao)
-    - "oss-cn-beijing.aliyuncs.com"
-        - North China 2 (Beijing)
-    - "oss-cn-zhangjiakou.aliyuncs.com"
-        - North China 3 (Zhangjiakou)
-    - "oss-cn-huhehaote.aliyuncs.com"
-        - North China 5 (Huhehaote)
-    - "oss-cn-shenzhen.aliyuncs.com"
-        - South China 1 (Shenzhen)
-    - "oss-cn-hongkong.aliyuncs.com"
-        - Hong Kong (Hong Kong)
-    - "oss-us-west-1.aliyuncs.com"
-        - US West 1 (Silicon Valley)
-    - "oss-us-east-1.aliyuncs.com"
-        - US East 1 (Virginia)
-    - "oss-ap-southeast-1.aliyuncs.com"
-        - Southeast Asia Southeast 1 (Singapore)
-    - "oss-ap-southeast-2.aliyuncs.com"
-        - Asia Pacific Southeast 2 (Sydney)
-    - "oss-ap-southeast-3.aliyuncs.com"
-        - Southeast Asia Southeast 3 (Kuala Lumpur)
-    - "oss-ap-southeast-5.aliyuncs.com"
-        - Asia Pacific Southeast 5 (Jakarta)
-    - "oss-ap-northeast-1.aliyuncs.com"
-        - Asia Pacific Northeast 1 (Japan)
-    - "oss-ap-south-1.aliyuncs.com"
-        - Asia Pacific South 1 (Mumbai)
-    - "oss-eu-central-1.aliyuncs.com"
-        - Central Europe 1 (Frankfurt)
-    - "oss-eu-west-1.aliyuncs.com"
-        - West Europe (London)
-    - "oss-me-east-1.aliyuncs.com"
-        - Middle East 1 (Dubai)
-
-#### --s3-endpoint
-
 Endpoint for S3 API.
 Required when using an S3 clone.
 
@@ -908,27 +855,11 @@ The storage class to use when storing new objects in S3.
     - "ONEZONE_IA"
         - One Zone Infrequent Access storage class
     - "GLACIER"
-        - Glacier storage class
-
-#### --s3-storage-class
-
-The storage class to use when storing new objects in OSS.
-
-- Config:      storage_class
-- Env Var:     RCLONE_S3_STORAGE_CLASS
-- Type:        string
-- Default:     ""
-- Examples:
-    - "Standard"
-        - Standard storage class
-    - "Archive"
-        - Archive storage mode.
-    - "IA"
-        - Infrequent access storage mode.
+        - GLACIER storage class
 
 ### Advanced Options
 
-Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
+Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)).
 
 #### --s3-upload-cutoff
 
@@ -1527,41 +1458,30 @@ server_side_encryption =
 storage_class =
 ```
 
-### Alibaba OSS {#alibaba-oss}
+### Aliyun OSS / Netease NOS  ###
 
-Here is an example of making an [Alibaba Cloud (Aliyun) OSS](https://www.alibabacloud.com/product/oss/)
-configuration.  First run:
+This describes how to set up Aliyun OSS - Netease NOS is the same
+except for different endpoints.
 
-    rclone config
-
-This will guide you through an interactive setup process.
+Note this is a pretty standard S3 setup, except for the setting of
+`force_path_style = false` in the advanced config.
 
 ```
-No remotes found - make a new one
-n) New remote
-s) Set configuration password
-q) Quit config
-n/s/q> n
+# rclone config
+e/n/d/r/c/s/q> n
 name> oss
 Type of storage to configure.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
-[snip]
- 4 / Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)
+ 3 / Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)
    \ "s3"
-[snip]
 Storage> s3
 Choose your S3 provider.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
- 1 / Amazon Web Services (AWS) S3
-   \ "AWS"
- 2 / Alibaba Cloud Object Storage System (OSS) formerly Aliyun
-   \ "Alibaba"
- 3 / Ceph Object Storage
-   \ "Ceph"
-[snip]
-provider> Alibaba
+ 8 / Any other S3 compatible provider
+   \ "Other"
+provider> other
 Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
 Only applies if access_key_id and secret_access_key is blank.
 Enter a boolean value (true or false). Press Enter for the default ("false").
@@ -1574,71 +1494,70 @@ env_auth> 1
 AWS Access Key ID.
 Leave blank for anonymous access or runtime credentials.
 Enter a string value. Press Enter for the default ("").
-access_key_id> accesskeyid
+access_key_id> xxxxxxxxxxxx
 AWS Secret Access Key (password)
 Leave blank for anonymous access or runtime credentials.
 Enter a string value. Press Enter for the default ("").
-secret_access_key> secretaccesskey
-Endpoint for OSS API.
+secret_access_key> xxxxxxxxxxxxxxxxx
+Region to connect to.
+Leave blank if you are using an S3 clone and you don't have a region.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
- 1 / East China 1 (Hangzhou)
-   \ "oss-cn-hangzhou.aliyuncs.com"
- 2 / East China 2 (Shanghai)
-   \ "oss-cn-shanghai.aliyuncs.com"
- 3 / North China 1 (Qingdao)
-   \ "oss-cn-qingdao.aliyuncs.com"
-[snip]
-endpoint> 1
-Canned ACL used when creating buckets and storing or copying objects.
-
-Note that this ACL is applied when server side copying objects as S3
-doesn't copy the ACL from the source but rather writes a fresh one.
+ 1 / Use this if unsure. Will use v4 signatures and an empty region.
+   \ ""
+ 2 / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.
+   \ "other-v2-signature"
+region> 1
+Endpoint for S3 API.
+Required when using an S3 clone.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+endpoint> oss-cn-shenzhen.aliyuncs.com
+Location constraint - must be set to match the Region.
+Leave blank if not sure. Used when creating buckets only.
+Enter a string value. Press Enter for the default ("").
+location_constraint>
+Canned ACL used when creating buckets and/or storing objects in S3.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
  1 / Owner gets FULL_CONTROL. No one else has access rights (default).
    \ "private"
- 2 / Owner gets FULL_CONTROL. The AllUsers group gets READ access.
-   \ "public-read"
-   / Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.
-[snip]
 acl> 1
-The storage class to use when storing new objects in OSS.
-Enter a string value. Press Enter for the default ("").
-Choose a number from below, or type in your own value
- 1 / Default
-   \ ""
- 2 / Standard storage class
-   \ "STANDARD"
- 3 / Archive storage mode.
-   \ "GLACIER"
- 4 / Infrequent access storage mode.
-   \ "STANDARD_IA"
-storage_class> 1
 Edit advanced config? (y/n)
 y) Yes
 n) No
-y/n> n
+y/n> y
+Chunk size to use for uploading
+Enter a size with suffix k,M,G,T. Press Enter for the default ("5M").
+chunk_size>
+Don't store MD5 checksum with object metadata
+Enter a boolean value (true or false). Press Enter for the default ("false").
+disable_checksum>
+An AWS session token
+Enter a string value. Press Enter for the default ("").
+session_token>
+Concurrency for multipart uploads.
+Enter a signed integer. Press Enter for the default ("2").
+upload_concurrency>
+If true use path style access if false use virtual hosted style.
+Some providers (eg Aliyun OSS or Netease COS) require this.
+Enter a boolean value (true or false). Press Enter for the default ("true").
+force_path_style> false
 Remote config
 --------------------
 [oss]
 type = s3
-provider = Alibaba
+provider = Other
 env_auth = false
-access_key_id = accesskeyid
-secret_access_key = secretaccesskey
-endpoint = oss-cn-hangzhou.aliyuncs.com
+access_key_id = xxxxxxxxx
+secret_access_key = xxxxxxxxxxxxx
+endpoint = oss-cn-shenzhen.aliyuncs.com
 acl = private
-storage_class = Standard
+force_path_style = false
 --------------------
 y) Yes this is OK
 e) Edit this remote
 d) Delete this remote
 y/e/d> y
 ```
-
-### Netease NOS  ###
-
-For Netease NOS configure as per the configurator `rclone config`
-setting the provider `Netease`.  This will automatically set
-`force_path_style = false` which is necessary for it to run properly.

docs/content/sftp.md

@@ -124,15 +124,11 @@ The SFTP remote supports three authentication methods:
   * Key file
   * ssh-agent
 
-Key files should be PEM-encoded private key files. For instance `/home/$USER/.ssh/id_rsa`.
-Only unencrypted OpenSSH or PEM encrypted files are supported.
+Key files should be unencrypted PEM-encoded private key files.  For
+instance `/home/$USER/.ssh/id_rsa`.
 
-If you don't specify `pass` or `key_file` then rclone will attempt to contact an ssh-agent.
-
-You can also specify `key_use_agent` to force the usage of an ssh-agent. In this case
-`key_file` can also be specified to force the usage of a specific key in the ssh-agent.
-
-Using an ssh-agent is the only way to load encrypted OpenSSH keys at the moment.
+If you don't specify `pass` or `key_file` then rclone will attempt to
+contact an ssh-agent.
 
 If you set the `--sftp-ask-password` option, rclone will prompt for a
 password when needed and no password has been configured.
@@ -208,38 +204,13 @@ SSH password, leave blank to use ssh-agent.
 
 #### --sftp-key-file
 
-Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.
+Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.
 
 - Config:      key_file
 - Env Var:     RCLONE_SFTP_KEY_FILE
 - Type:        string
 - Default:     ""
 
-#### --sftp-key-file-pass
-
-The passphrase to decrypt the PEM-encoded private key file.
-
-Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
-in the new OpenSSH format can't be used.
-
-- Config:      key_file_pass
-- Env Var:     RCLONE_SFTP_KEY_FILE_PASS
-- Type:        string
-- Default:     ""
-
-#### --sftp-key-use-agent
-
-When set forces the usage of the ssh-agent.
-
-When key-file is also set, the ".pub" file of the specified key-file is read and only the associated key is
-requested from the ssh-agent. This allows to avoid `Too many authentication failures for *username*` errors
-when the ssh-agent contains many keys.
-
-- Config:      key_use_agent
-- Env Var:     RCLONE_SFTP_KEY_USE_AGENT
-- Type:        bool
-- Default:     false
-
 #### --sftp-use-insecure-cipher
 
 Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.

docs/content/webdav.md

@@ -253,7 +253,7 @@ pass = encryptedpassword
 
 ### dCache ###
 
-[dCache](https://www.dcache.org/) is a storage system with WebDAV doors that support, beside basic and x509, 
+dCache is a storage system with WebDAV doors that support, beside basic and x509, 
 authentication with [Macaroons](https://www.dcache.org/manuals/workshop-2017-05-29-Umea/000-Final/anupam_macaroons_v02.pdf) (bearer tokens).
 
 Configure as normal using the `other` type.  Don't enter a username or
@@ -271,5 +271,5 @@ pass =
 bearer_token = your-macaroon
 ```
 
-There is a [script](https://github.com/sara-nl/GridScripts/blob/master/get-macaroon) that
+There is a [script](https://github.com/onnozweers/dcache-scripts/blob/master/get-share-link) that
 obtains a Macaroon from a dCache WebDAV endpoint, and creates an rclone config file.

fs/accounting/accounting.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"sync"
 	"time"
-	"unicode/utf8"
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/asyncreader"
@@ -244,24 +243,6 @@ func (acc *Account) eta() (etaDuration time.Duration, ok bool) {
 	return eta(acc.bytes, acc.size, acc.avg)
 }
 
-// shortenName shortens in to size runes long
-// If size <= 0 then in is left untouched
-func shortenName(in string, size int) string {
-	if size <= 0 {
-		return in
-	}
-	if utf8.RuneCountInString(in) <= size {
-		return in
-	}
-	name := []rune(in)
-	size-- // don't count elipsis rune
-	suffixLength := size / 2
-	prefixLength := size - suffixLength
-	suffixStart := len(name) - suffixLength
-	name = append(append(name[:prefixLength], '…'), name[suffixStart:]...)
-	return string(name)
-}
-
 // String produces stats for this file
 func (acc *Account) String() string {
 	a, b := acc.progress()
@@ -276,6 +257,16 @@ func (acc *Account) String() string {
 		}
 	}
 
+	name := []rune(acc.name)
+	if fs.Config.StatsFileNameLength > 0 {
+		if len(name) > fs.Config.StatsFileNameLength {
+			suffixLength := fs.Config.StatsFileNameLength / 2
+			prefixLength := fs.Config.StatsFileNameLength - suffixLength
+			suffixStart := len(name) - suffixLength
+			name = append(append(name[:prefixLength], '…'), name[suffixStart:]...)
+		}
+	}
+
 	if fs.Config.DataRateUnit == "bits" {
 		cur = cur * 8
 	}
@@ -285,11 +276,11 @@ func (acc *Account) String() string {
 		percentageDone = int(100 * float64(a) / float64(b))
 	}
 
-	return fmt.Sprintf("%*s:%3d%% /%s, %s/s, %s",
-		fs.Config.StatsFileNameLength,
-		shortenName(acc.name, fs.Config.StatsFileNameLength),
-		percentageDone,
-		fs.SizeSuffix(b),
+	done := fmt.Sprintf("%2d%% /%s", percentageDone, fs.SizeSuffix(b))
+
+	return fmt.Sprintf("%45s: %s, %s/s, %s",
+		string(name),
+		done,
 		fs.SizeSuffix(cur),
 		etas,
 	)

fs/accounting/accounting_test.go

@@ -2,12 +2,10 @@ package accounting
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"strings"
 	"testing"
-	"unicode/utf8"
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/asyncreader"
@@ -210,46 +208,3 @@ func TestAccountMaxTransfer(t *testing.T) {
 	assert.Equal(t, ErrorMaxTransferLimitReached, err)
 	assert.True(t, fserrors.IsFatalError(err))
 }
-
-func TestShortenName(t *testing.T) {
-	for _, test := range []struct {
-		in   string
-		size int
-		want string
-	}{
-		{"", 0, ""},
-		{"abcde", 10, "abcde"},
-		{"abcde", 0, "abcde"},
-		{"abcde", -1, "abcde"},
-		{"abcde", 5, "abcde"},
-		{"abcde", 4, "ab…e"},
-		{"abcde", 3, "a…e"},
-		{"abcde", 2, "a…"},
-		{"abcde", 1, "…"},
-		{"abcdef", 6, "abcdef"},
-		{"abcdef", 5, "ab…ef"},
-		{"abcdef", 4, "ab…f"},
-		{"abcdef", 3, "a…f"},
-		{"abcdef", 2, "a…"},
-		{"áßcdèf", 1, "…"},
-		{"áßcdè", 5, "áßcdè"},
-		{"áßcdè", 4, "áß…è"},
-		{"áßcdè", 3, "á…è"},
-		{"áßcdè", 2, "á…"},
-		{"áßcdè", 1, "…"},
-		{"áßcdèł", 6, "áßcdèł"},
-		{"áßcdèł", 5, "áß…èł"},
-		{"áßcdèł", 4, "áß…ł"},
-		{"áßcdèł", 3, "á…ł"},
-		{"áßcdèł", 2, "á…"},
-		{"áßcdèł", 1, "…"},
-	} {
-		t.Run(fmt.Sprintf("in=%q, size=%d", test.in, test.size), func(t *testing.T) {
-			got := shortenName(test.in, test.size)
-			assert.Equal(t, test.want, got)
-			if test.size > 0 {
-				assert.True(t, utf8.RuneCountInString(got) <= test.size, "too big")
-			}
-		})
-	}
-}

fs/accounting/stats.go

@@ -92,8 +92,8 @@ type StatsInfo struct {
 // NewStats cretates an initialised StatsInfo
 func NewStats() *StatsInfo {
 	return &StatsInfo{
-		checking:     newStringSet(fs.Config.Checkers, "checking"),
-		transferring: newStringSet(fs.Config.Transfers, "transferring"),
+		checking:     newStringSet(fs.Config.Checkers),
+		transferring: newStringSet(fs.Config.Transfers),
 		start:        time.Now(),
 		inProgress:   newInProgress(),
 	}
@@ -320,13 +320,6 @@ func (s *StatsInfo) GetLastError() error {
 	return s.lastError
 }
 
-// GetChecks returns the number of checks
-func (s *StatsInfo) GetChecks() int64 {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.checks
-}
-
 // FatalError sets the fatalError flag
 func (s *StatsInfo) FatalError() {
 	s.mu.Lock()

fs/accounting/stringset.go

@@ -1,26 +1,21 @@
 package accounting
 
 import (
-	"fmt"
 	"sort"
 	"strings"
 	"sync"
-
-	"github.com/ncw/rclone/fs"
 )
 
 // stringSet holds a set of strings
 type stringSet struct {
 	mu    sync.RWMutex
 	items map[string]struct{}
-	name  string
 }
 
 // newStringSet creates a new empty string set of capacity size
-func newStringSet(size int, name string) *stringSet {
+func newStringSet(size int) *stringSet {
 	return &stringSet{
 		items: make(map[string]struct{}, size),
-		name:  name,
 	}
 }
 
@@ -62,11 +57,7 @@ func (ss *stringSet) Strings() []string {
 		if acc := Stats.inProgress.get(name); acc != nil {
 			out = acc.String()
 		} else {
-			out = fmt.Sprintf("%*s: %s",
-				fs.Config.StatsFileNameLength,
-				shortenName(name, fs.Config.StatsFileNameLength),
-				ss.name,
-			)
+			out = name
 		}
 		strings = append(strings, " * "+out)
 	}

fs/config.go

@@ -85,7 +85,6 @@ type ConfigInfo struct {
 	MaxBacklog            int
 	StatsOneLine          bool
 	Progress              bool
-	Cookie                bool
 }
 
 // NewConfig creates a new config with everything set to the default
@@ -110,7 +109,7 @@ func NewConfig() *ConfigInfo {
 	c.BufferSize = SizeSuffix(16 << 20)
 	c.UserAgent = "rclone/" + Version
 	c.StreamingUploadCutoff = SizeSuffix(100 * 1024)
-	c.StatsFileNameLength = 45
+	c.StatsFileNameLength = 40
 	c.AskPassword = true
 	c.TPSLimitBurst = 1
 	c.MaxTransfer = -1

fs/config/config.go

@@ -27,7 +27,6 @@ import (
 	"github.com/Unknwon/goconfig"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
-	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/driveletter"
@@ -58,8 +57,8 @@ const (
 	// ConfigTokenURL is the config key used to store the token server endpoint
 	ConfigTokenURL = "token_url"
 
-	// ConfigAuthorize indicates that we just want "rclone authorize"
-	ConfigAuthorize = "config_authorize"
+	// ConfigAutomatic indicates that we want non-interactive configuration
+	ConfigAutomatic = "config_automatic"
 )
 
 // Global
@@ -640,38 +639,21 @@ func Command(commands []string) byte {
 	}
 }
 
+// ConfirmWithDefault asks the user for Yes or No and returns true or false.
+//
+// If AutoConfirm is set, it will return the Default value passed in
+func ConfirmWithDefault(Default bool) bool {
+	if fs.Config.AutoConfirm {
+		return Default
+	}
+	return Command([]string{"yYes", "nNo"}) == 'y'
+}
+
 // Confirm asks the user for Yes or No and returns true or false
 //
 // If AutoConfirm is set, it will return true
 func Confirm() bool {
-	return Command([]string{"yYes", "nNo"}) == 'y'
-}
-
-// ConfirmWithConfig asks the user for Yes or No and returns true or
-// false.
-//
-// If AutoConfirm is set, it will look up the value in m and return
-// that, but if it isn't set then it will return the Default value
-// passed in
-func ConfirmWithConfig(m configmap.Getter, configName string, Default bool) bool {
-	if fs.Config.AutoConfirm {
-		configString, ok := m.Get(configName)
-		if ok {
-			configValue, err := strconv.ParseBool(configString)
-			if err != nil {
-				fs.Errorf(nil, "Failed to parse config parameter %s=%q as boolean - using default %v: %v", configName, configString, Default, err)
-			} else {
-				Default = configValue
-			}
-		}
-		answer := "No"
-		if Default {
-			answer = "Yes"
-		}
-		fmt.Printf("Auto confirm is set: answering %s, override by setting config parameter %s=%v\n", answer, configName, !Default)
-		return Default
-	}
-	return Confirm()
+	return ConfirmWithDefault(true)
 }
 
 // Choose one of the defaults or type a new string if newOk is set
@@ -961,6 +943,8 @@ func CreateRemote(name string, provider string, keyValues rc.Params) error {
 	getConfigData().DeleteSection(name)
 	// Set the type
 	getConfigData().SetValue(name, "type", provider)
+	// Show this is automatically configured
+	getConfigData().SetValue(name, ConfigAutomatic, "yes")
 	// Set the remaining values
 	return UpdateRemote(name, keyValues)
 }
@@ -1243,7 +1227,6 @@ func SetPassword() {
 //   rclone authorize "fs name"
 //   rclone authorize "fs name" "client id" "client secret"
 func Authorize(args []string) {
-	defer suppressConfirm()()
 	switch len(args) {
 	case 1, 3:
 	default:
@@ -1260,8 +1243,8 @@ func Authorize(args []string) {
 	// Make sure we delete it
 	defer DeleteRemote(name)
 
-	// Indicate that we are running rclone authorize
-	getConfigData().SetValue(name, ConfigAuthorize, "true")
+	// Indicate that we want fully automatic configuration.
+	getConfigData().SetValue(name, ConfigAutomatic, "yes")
 	if len(args) == 3 {
 		getConfigData().SetValue(name, ConfigClientID, args[1])
 		getConfigData().SetValue(name, ConfigClientSecret, args[2])

fs/config/configflags/configflags.go

@@ -40,7 +40,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.IntVarP(flagSet, &fs.Config.Transfers, "transfers", "", fs.Config.Transfers, "Number of file transfers to run in parallel.")
 	flags.StringVarP(flagSet, &config.ConfigPath, "config", "", config.ConfigPath, "Config file.")
 	flags.StringVarP(flagSet, &config.CacheDir, "cache-dir", "", config.CacheDir, "Directory rclone will use for caching.")
-	flags.BoolVarP(flagSet, &fs.Config.CheckSum, "checksum", "c", fs.Config.CheckSum, "Skip based on checksum (if available) & size, not mod-time & size")
+	flags.BoolVarP(flagSet, &fs.Config.CheckSum, "checksum", "c", fs.Config.CheckSum, "Skip based on checksum & size, not mod-time & size")
 	flags.BoolVarP(flagSet, &fs.Config.SizeOnly, "size-only", "", fs.Config.SizeOnly, "Skip based on size only, not mod-time or checksum")
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreTimes, "ignore-times", "I", fs.Config.IgnoreTimes, "Don't skip files that match size and time - transfer all files")
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreExisting, "ignore-existing", "", fs.Config.IgnoreExisting, "Skip all files that exist on destination")
@@ -87,7 +87,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.")
 	flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.")
 	flags.BoolVarP(flagSet, &fs.Config.Progress, "progress", "P", fs.Config.Progress, "Show progress during transfer.")
-	flags.BoolVarP(flagSet, &fs.Config.Cookie, "use-cookies", "", fs.Config.Cookie, "Enable session cookiejar.")
 }
 
 // SetFlags converts any flags into config which weren't straight foward

fs/fshttp/http.go

@@ -7,14 +7,12 @@ import (
 	"crypto/tls"
 	"net"
 	"net/http"
-	"net/http/cookiejar"
 	"net/http/httputil"
 	"reflect"
 	"sync"
 	"time"
 
 	"github.com/ncw/rclone/fs"
-	"golang.org/x/net/publicsuffix"
 	"golang.org/x/time/rate"
 )
 
@@ -24,10 +22,9 @@ const (
 )
 
 var (
-	transport    http.RoundTripper
-	noTransport  sync.Once
-	tpsBucket    *rate.Limiter // for limiting number of http transactions per second
-	cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
+	transport   http.RoundTripper
+	noTransport sync.Once
+	tpsBucket   *rate.Limiter // for limiting number of http transactions per second
 )
 
 // StartHTTPTokenBucket starts the token bucket if necessary
@@ -145,13 +142,9 @@ func NewTransport(ci *fs.ConfigInfo) http.RoundTripper {
 
 // NewClient returns an http.Client with the correct timeouts
 func NewClient(ci *fs.ConfigInfo) *http.Client {
-	transport := &http.Client{
+	return &http.Client{
 		Transport: NewTransport(ci),
 	}
-	if ci.Cookie {
-		transport.Jar = cookieJar
-	}
-	return transport
 }
 
 // Transport is a our http Transport which wraps an http.Transport

fs/mimetype.go

@@ -12,6 +12,7 @@ func MimeTypeFromName(remote string) (mimeType string) {
 	if !strings.ContainsRune(mimeType, '/') {
 		mimeType = "application/octet-stream"
 	}
+	mimeType = strings.Replace(mimeType, "; charset=utf-8", "", -1) // Remove charset
 	return mimeType
 }
 

fs/operations/operations.go

@@ -104,8 +104,6 @@ func sizeDiffers(src, dst fs.ObjectInfo) bool {
 	return src.Size() != dst.Size()
 }
 
-var checksumWarning sync.Once
-
 func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
 	if sizeDiffers(src, dst) {
 		fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
@@ -127,9 +125,6 @@ func equal(src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
 			return false
 		}
 		if ht == hash.None {
-			checksumWarning.Do(func() {
-				fs.Logf(dst.Fs(), "--checksum is in use but the source and destination have no hashes in common; falling back to --size-only")
-			})
 			fs.Debugf(src, "Size of src and dst objects identical")
 		} else {
 			fs.Debugf(src, "Size and %v of src and dst objects identical", ht)
@@ -577,7 +572,6 @@ type checkMarch struct {
 	noHashes        int32
 	srcFilesMissing int32
 	dstFilesMissing int32
-	matches         int32
 }
 
 // DstOnly have an object which is in the destination only
@@ -645,7 +639,6 @@ func (c *checkMarch) Match(dst, src fs.DirEntry) (recurse bool) {
 			if differ {
 				atomic.AddInt32(&c.differences, 1)
 			} else {
-				atomic.AddInt32(&c.matches, 1)
 				fs.Debugf(dstX, "OK")
 			}
 			if noHash {
@@ -713,9 +706,6 @@ func CheckFn(fdst, fsrc fs.Fs, check checkFn, oneway bool) error {
 	if c.noHashes > 0 {
 		fs.Logf(fdst, "%d hashes could not be checked", c.noHashes)
 	}
-	if c.matches > 0 {
-		fs.Logf(fdst, "%d matching files", c.matches)
-	}
 	if c.differences > 0 {
 		return errors.Errorf("%d differences found", c.differences)
 	}

fs/operations/operations_test.go

@@ -25,10 +25,8 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -273,17 +271,11 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs, oneway bool) e
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 
-	check := func(i int, wantErrors int64, wantChecks int64, oneway bool) {
+	check := func(i int, wantErrors int64, oneway bool) {
 		fs.Debugf(r.Fremote, "%d: Starting check test", i)
-		accounting.Stats.ResetCounters()
-		var buf bytes.Buffer
-		log.SetOutput(&buf)
-		defer func() {
-			log.SetOutput(os.Stderr)
-		}()
+		oldErrors := accounting.Stats.GetErrors()
 		err := checkFunction(r.Fremote, r.Flocal, oneway)
-		gotErrors := accounting.Stats.GetErrors()
-		gotChecks := accounting.Stats.GetChecks()
+		gotErrors := accounting.Stats.GetErrors() - oldErrors
 		if wantErrors == 0 && err != nil {
 			t.Errorf("%d: Got error when not expecting one: %v", i, err)
 		}
@@ -293,27 +285,21 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs, oneway bool) e
 		if wantErrors != gotErrors {
 			t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors)
 		}
-		if gotChecks > 0 && !strings.Contains(buf.String(), "matching files") {
-			t.Errorf("%d: Total files matching line missing", i)
-		}
-		if wantChecks != gotChecks {
-			t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks)
-		}
 		fs.Debugf(r.Fremote, "%d: Ending check test", i)
 	}
 
 	file1 := r.WriteBoth("rutabaga", "is tasty", t3)
 	fstest.CheckItems(t, r.Fremote, file1)
 	fstest.CheckItems(t, r.Flocal, file1)
-	check(1, 0, 1, false)
+	check(1, 0, false)
 
 	file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
 	fstest.CheckItems(t, r.Flocal, file1, file2)
-	check(2, 1, 1, false)
+	check(2, 1, false)
 
 	file3 := r.WriteObject("empty space", "", t2)
 	fstest.CheckItems(t, r.Fremote, file1, file3)
-	check(3, 2, 1, false)
+	check(3, 2, false)
 
 	file2r := file2
 	if fs.Config.SizeOnly {
@@ -322,16 +308,16 @@ func testCheck(t *testing.T, checkFunction func(fdst, fsrc fs.Fs, oneway bool) e
 		r.WriteObject("potato2", "------------------------------------------------------------", t1)
 	}
 	fstest.CheckItems(t, r.Fremote, file1, file2r, file3)
-	check(4, 1, 2, false)
+	check(4, 1, false)
 
 	r.WriteFile("empty space", "", t2)
 	fstest.CheckItems(t, r.Flocal, file1, file2, file3)
-	check(5, 0, 3, false)
+	check(5, 0, false)
 
 	file4 := r.WriteObject("remotepotato", "------------------------------------------------------------", t1)
 	fstest.CheckItems(t, r.Fremote, file1, file2r, file3, file4)
-	check(6, 1, 3, false)
-	check(7, 0, 3, true)
+	check(6, 1, false)
+	check(7, 0, true)
 }
 
 func TestCheck(t *testing.T) {

fstest/test_all/config.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"path"
 
-	"github.com/ncw/rclone/fs"
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
 )
@@ -28,12 +27,11 @@ type Test struct {
 //
 // FIXME make bucket based remotes set sub-dir automatically???
 type Backend struct {
-	Backend  string   // name of the backend directory
-	Remote   string   // name of the test remote
-	SubDir   bool     // set to test with -sub-dir
-	FastList bool     // set to test with -fast-list
-	OneOnly  bool     // set to run only one backend test at once
-	Ignore   []string // test names to ignore the failure of
+	Backend  string // name of the backend directory
+	Remote   string // name of the test remote
+	SubDir   bool   // set to test with -sub-dir
+	FastList bool   // set to test with -fast-list
+	OneOnly  bool   // set to run only one backend test at once
 }
 
 // MakeRuns creates Run objects the Backend and Test
@@ -49,10 +47,6 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
 	if b.FastList && t.FastList {
 		fastlists = append(fastlists, true)
 	}
-	ignore := make(map[string]struct{}, len(b.Ignore))
-	for _, item := range b.Ignore {
-		ignore[item] = struct{}{}
-	}
 	for _, subdir := range subdirs {
 		for _, fastlist := range fastlists {
 			run := &Run{
@@ -64,7 +58,6 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
 				NoRetries: t.NoRetries,
 				OneOnly:   b.OneOnly,
 				NoBinary:  t.NoBinary,
-				Ignore:    ignore,
 			}
 			if t.AddBackend {
 				run.Path = path.Join(run.Path, b.Backend)
@@ -126,12 +119,7 @@ func (c *Config) filterBackendsByRemotes(remotes []string) {
 		}
 		if !found {
 			log.Printf("Remote %q not found - inserting with default flags", name)
-			// Lookup which backend
-			fsInfo, _, _, _, err := fs.ConfigFs(name)
-			if err != nil {
-				log.Fatalf("couldn't find remote %q: %v", name, err)
-			}
-			newBackends = append(newBackends, Backend{Backend: fsInfo.FileName(), Remote: name})
+			newBackends = append(newBackends, Backend{Remote: name})
 		}
 	}
 	c.Backends = newBackends

fstest/test_all/config.yaml

@@ -53,30 +53,6 @@ backends:
    remote:   "TestS3:"
    subdir:   true
    fastlist: true
- - backend:  "s3"
-   remote:   "TestS3Minio:"
-   subdir:   true
-   fastlist: true
- - backend:  "s3"
-   remote:   "TestS3Wasabi:"
-   subdir:   true
-   fastlist: true
- - backend:  "s3"
-   remote:   "TestS3DigitalOcean:"
-   subdir:   true
-   fastlist: true
-   ignore:
-     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
- - backend:  "s3"
-   remote:   "TestS3Ceph:"
-   subdir:   true
-   fastlist: true
-   ignore:
-     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
- - backend:  "s3"
-   remote:   "TestS3Alibaba:"
-   subdir:   true
-   fastlist: true
  - backend:  "sftp"
    remote:   "TestSftp:"
    subdir:   false
@@ -85,12 +61,6 @@ backends:
    remote:   "TestSwift:"
    subdir:   true
    fastlist: true
- - backend:  "swift"
-   remote:   "TestSwiftCeph:"
-   subdir:   true
-   fastlist: true
-   ignore:
-     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "yandex"
    remote:   "TestYandex:"
    subdir:   false
@@ -128,8 +98,6 @@ backends:
    remote:   "TestMega:"
    subdir:   false
    fastlist: false
-   ignore:
-     - TestIntegration/FsMkdir/FsPutFiles/PublicLink
  - backend:  "opendrive"
    remote:   "TestOpenDrive:"
    subdir:   false

fstest/test_all/run.go

@@ -45,7 +45,6 @@ type Run struct {
 	NoRetries bool   // don't retry if set
 	OneOnly   bool   // only run test for this backend at once
 	NoBinary  bool   // set to not build a binary
-	Ignore    map[string]struct{}
 	// Internals
 	cmdLine     []string
 	cmdString   string
@@ -139,15 +138,9 @@ func (r *Run) findFailures() {
 	oldFailedTests := r.failedTests
 	r.failedTests = nil
 	excludeParents := map[string]struct{}{}
-	ignored := 0
 	for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
 		failedTest := string(matches[1])
-		// Skip any ignored failures
-		if _, found := r.Ignore[failedTest]; found {
-			ignored++
-		} else {
-			r.failedTests = append(r.failedTests, failedTest)
-		}
+		r.failedTests = append(r.failedTests, failedTest)
 		// Find all the parents of this test
 		parts := strings.Split(failedTest, "/")
 		for i := len(parts) - 1; i >= 1; i-- {
@@ -162,12 +155,6 @@ func (r *Run) findFailures() {
 		}
 	}
 	r.failedTests = newTests
-	if len(r.failedTests) == 0 && ignored > 0 {
-		log.Printf("%q - Found %d ignored errors only - marking as good", r.cmdString, ignored)
-		r.err = nil
-		r.dumpOutput()
-		return
-	}
 	if len(r.failedTests) != 0 {
 		r.runFlag = testsToRegexp(r.failedTests)
 	} else {

go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/Unknwon/goconfig v0.0.0-20181105214110-56bd8ab18619
 	github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6
 	github.com/abbot/go-http-auth v0.4.0
-	github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5
 	github.com/aws/aws-sdk-go v1.15.81
 	github.com/billziss-gh/cgofuse v1.1.0
 	github.com/coreos/bbolt v0.0.0-20180318001526-af9db2027c98
@@ -18,7 +17,7 @@ require (
 	github.com/djherbis/times v1.1.0
 	github.com/dropbox/dropbox-sdk-go-unofficial v5.4.0+incompatible
 	github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9 // indirect
-	github.com/goftp/server v0.0.0-20190111142836-88de73f463af
+	github.com/goftp/server v0.0.0-20180914132916-1fd52c8552f1
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect

go.sum

@@ -14,8 +14,6 @@ github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6 h1:UCQe3W9LxwL2ff5r0PqQfS
 github.com/a8m/tree v0.0.0-20180321023834-3cf936ce15d6/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg=
 github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
 github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
-github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5 h1:lmyFvZXNGOmsKCYXNwzDLWafnxeewxsFwdsvTvSC1sg=
-github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE=
 github.com/aws/aws-sdk-go v1.15.81 h1:va7uoFaV9uKAtZ6BTmp1u7paoMsizYRRLvRuoC07nQ8=
 github.com/aws/aws-sdk-go v1.15.81/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
 github.com/billziss-gh/cgofuse v1.1.0 h1:tATn9ZDvuPcOVlvR4tJitGHgAqy1y18+4mKmRfdfjec=
@@ -39,8 +37,6 @@ github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9 h1:cC0Hbb+18DJ4i
 github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9/go.mod h1:GpOj6zuVBG3Inr9qjEnuVTgBlk2lZ1S9DcoFiXWyKss=
 github.com/goftp/server v0.0.0-20180914132916-1fd52c8552f1 h1:WjgeEHEDLGx56ndxS6FYi6qFjZGajSVHPuEPdpJ60cI=
 github.com/goftp/server v0.0.0-20180914132916-1fd52c8552f1/go.mod h1:k/SS6VWkxY7dHPhoMQ8IdRu8L4lQtmGbhyXGg+vCnXE=
-github.com/goftp/server v0.0.0-20190111142836-88de73f463af h1:PJxb1aA1z+Ohy2j28L92+ng9phXpZVFRFbPkfmJcRGo=
-github.com/goftp/server v0.0.0-20190111142836-88de73f463af/go.mod h1:k/SS6VWkxY7dHPhoMQ8IdRu8L4lQtmGbhyXGg+vCnXE=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=

lib/file/file.go

@@ -1,22 +0,0 @@
-// Package file provides a version of os.OpenFile, the handles of
-// which can be renamed and deleted under Windows.
-package file
-
-import "os"
-
-// Open opens the named file for reading. If successful, methods on
-// the returned file can be used for reading; the associated file
-// descriptor has mode O_RDONLY.
-// If there is an error, it will be of type *PathError.
-func Open(name string) (*os.File, error) {
-	return OpenFile(name, os.O_RDONLY, 0)
-}
-
-// Create creates the named file with mode 0666 (before umask), truncating
-// it if it already exists. If successful, methods on the returned
-// File can be used for I/O; the associated file descriptor has mode
-// O_RDWR.
-// If there is an error, it will be of type *PathError.
-func Create(name string) (*os.File, error) {
-	return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
-}

lib/file/file_other.go

@@ -1,15 +0,0 @@
-//+build !windows
-
-package file
-
-import "os"
-
-// OpenFile is the generalized open call; most users will use Open or Create
-// instead. It opens the named file with specified flag (O_RDONLY etc.) and
-// perm (before umask), if applicable. If successful, methods on the returned
-// File can be used for I/O. If there is an error, it will be of type
-// *PathError.
-//
-// Under both Unix and Windows this will allow open files to be
-// renamed and or deleted.
-var OpenFile = os.OpenFile

lib/file/file_test.go

@@ -1,154 +0,0 @@
-package file
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// Create a test directory then tidy up
-func testDir(t *testing.T) (string, func()) {
-	dir, err := ioutil.TempDir("", "rclone-test")
-	require.NoError(t, err)
-	return dir, func() {
-		assert.NoError(t, os.RemoveAll(dir))
-	}
-}
-
-// This lists dir and checks the listing is as expected without checking the size
-func checkListingNoSize(t *testing.T, dir string, want []string) {
-	var got []string
-	nodes, err := ioutil.ReadDir(dir)
-	require.NoError(t, err)
-	for _, node := range nodes {
-		got = append(got, fmt.Sprintf("%s,%v", node.Name(), node.IsDir()))
-	}
-	assert.Equal(t, want, got)
-}
-
-// This lists dir and checks the listing is as expected
-func checkListing(t *testing.T, dir string, want []string) {
-	var got []string
-	nodes, err := ioutil.ReadDir(dir)
-	require.NoError(t, err)
-	for _, node := range nodes {
-		got = append(got, fmt.Sprintf("%s,%d,%v", node.Name(), node.Size(), node.IsDir()))
-	}
-	assert.Equal(t, want, got)
-}
-
-// Test we can rename an open file
-func TestOpenFileRename(t *testing.T) {
-	dir, tidy := testDir(t)
-	defer tidy()
-
-	filepath := path.Join(dir, "file1")
-	f, err := Create(filepath)
-	require.NoError(t, err)
-
-	_, err = f.Write([]byte("hello"))
-	assert.NoError(t, err)
-
-	checkListingNoSize(t, dir, []string{
-		"file1,false",
-	})
-
-	// Delete the file first
-	assert.NoError(t, os.Remove(filepath))
-
-	// .. then close it
-	assert.NoError(t, f.Close())
-
-	checkListing(t, dir, nil)
-}
-
-// Test we can delete an open file
-func TestOpenFileDelete(t *testing.T) {
-	dir, tidy := testDir(t)
-	defer tidy()
-
-	filepath := path.Join(dir, "file1")
-	f, err := Create(filepath)
-	require.NoError(t, err)
-
-	_, err = f.Write([]byte("hello"))
-	assert.NoError(t, err)
-
-	checkListingNoSize(t, dir, []string{
-		"file1,false",
-	})
-
-	// Rename the file while open
-	filepath2 := path.Join(dir, "file2")
-	assert.NoError(t, os.Rename(filepath, filepath2))
-
-	checkListingNoSize(t, dir, []string{
-		"file2,false",
-	})
-
-	// .. then close it
-	assert.NoError(t, f.Close())
-
-	checkListing(t, dir, []string{
-		"file2,5,false",
-	})
-}
-
-// Smoke test the Open, OpenFile and Create functions
-func TestOpenFileOperations(t *testing.T) {
-	dir, tidy := testDir(t)
-	defer tidy()
-
-	filepath := path.Join(dir, "file1")
-
-	// Create the file
-
-	f, err := Create(filepath)
-	require.NoError(t, err)
-
-	_, err = f.Write([]byte("hello"))
-	assert.NoError(t, err)
-
-	assert.NoError(t, f.Close())
-
-	checkListing(t, dir, []string{
-		"file1,5,false",
-	})
-
-	// Append onto the file
-
-	f, err = OpenFile(filepath, os.O_RDWR|os.O_APPEND, 0666)
-	require.NoError(t, err)
-
-	_, err = f.Write([]byte("HI"))
-	assert.NoError(t, err)
-
-	assert.NoError(t, f.Close())
-
-	checkListing(t, dir, []string{
-		"file1,7,false",
-	})
-
-	// Read it back in
-
-	f, err = Open(filepath)
-	require.NoError(t, err)
-	var b = make([]byte, 10)
-	n, err := f.Read(b)
-	assert.True(t, err == io.EOF || err == nil)
-	assert.Equal(t, 7, n)
-	assert.Equal(t, "helloHI", string(b[:n]))
-
-	assert.NoError(t, f.Close())
-
-	checkListing(t, dir, []string{
-		"file1,7,false",
-	})
-
-}

lib/file/file_windows.go

@@ -1,66 +0,0 @@
-//+build windows
-
-package file
-
-import (
-	"os"
-	"syscall"
-)
-
-// OpenFile is the generalized open call; most users will use Open or Create
-// instead. It opens the named file with specified flag (O_RDONLY etc.) and
-// perm (before umask), if applicable. If successful, methods on the returned
-// File can be used for I/O. If there is an error, it will be of type
-// *PathError.
-//
-// Under both Unix and Windows this will allow open files to be
-// renamed and or deleted.
-func OpenFile(path string, mode int, perm os.FileMode) (*os.File, error) {
-	// This code copied from syscall_windows.go in the go source and then
-	// modified to support renaming and deleting open files by adding
-	// FILE_SHARE_DELETE.
-	//
-	// https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-createfilea#file_share_delete
-	if len(path) == 0 {
-		return nil, syscall.ERROR_FILE_NOT_FOUND
-	}
-	pathp, err := syscall.UTF16PtrFromString(path)
-	if err != nil {
-		return nil, err
-	}
-	var access uint32
-	switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) {
-	case syscall.O_RDONLY:
-		access = syscall.GENERIC_READ
-	case syscall.O_WRONLY:
-		access = syscall.GENERIC_WRITE
-	case syscall.O_RDWR:
-		access = syscall.GENERIC_READ | syscall.GENERIC_WRITE
-	}
-	if mode&syscall.O_CREAT != 0 {
-		access |= syscall.GENERIC_WRITE
-	}
-	if mode&syscall.O_APPEND != 0 {
-		access &^= syscall.GENERIC_WRITE
-		access |= syscall.FILE_APPEND_DATA
-	}
-	sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
-	var createmode uint32
-	switch {
-	case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL):
-		createmode = syscall.CREATE_NEW
-	case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC):
-		createmode = syscall.CREATE_ALWAYS
-	case mode&syscall.O_CREAT == syscall.O_CREAT:
-		createmode = syscall.OPEN_ALWAYS
-	case mode&syscall.O_TRUNC == syscall.O_TRUNC:
-		createmode = syscall.TRUNCATE_EXISTING
-	default:
-		createmode = syscall.OPEN_EXISTING
-	}
-	h, e := syscall.CreateFile(pathp, access, sharemode, nil, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0)
-	if e != nil {
-		return nil, e
-	}
-	return os.NewFile(uintptr(h), path), nil
-}

lib/oauthutil/oauthutil.go

@@ -358,26 +358,18 @@ func ConfigErrorCheck(id, name string, m configmap.Mapper, errorHandler func(*ht
 
 func doConfig(id, name string, m configmap.Mapper, errorHandler func(*http.Request) AuthError, oauthConfig *oauth2.Config, offline bool, opts []oauth2.AuthCodeOption) error {
 	oauthConfig, changed := overrideCredentials(name, m, oauthConfig)
-	authorizeOnlyValue, ok := m.Get(config.ConfigAuthorize)
-	authorizeOnly := ok && authorizeOnlyValue != "" // set if being run by "rclone authorize"
+	auto, ok := m.Get(config.ConfigAutomatic)
+	automatic := ok && auto != ""
 
 	// See if already have a token
 	tokenString, ok := m.Get("token")
 	if ok && tokenString != "" {
 		fmt.Printf("Already have a token - refresh?\n")
-		if !config.ConfirmWithConfig(m, "config_refresh_token", true) {
+		if !config.Confirm() {
 			return nil
 		}
 	}
 
-	// Ask the user whether they are using a local machine
-	isLocal := func() bool {
-		fmt.Printf("Use auto config?\n")
-		fmt.Printf(" * Say Y if not sure\n")
-		fmt.Printf(" * Say N if you are working on a remote or headless machine\n")
-		return config.ConfirmWithConfig(m, "config_is_local", true)
-	}
-
 	// Detect whether we should use internal web server
 	useWebServer := false
 	switch oauthConfig.RedirectURL {
@@ -386,10 +378,14 @@ func doConfig(id, name string, m configmap.Mapper, errorHandler func(*http.Reque
 			fmt.Printf("Make sure your Redirect URL is set to %q in your custom config.\n", oauthConfig.RedirectURL)
 		}
 		useWebServer = true
-		if authorizeOnly {
+		if automatic {
 			break
 		}
-		if !isLocal() {
+		fmt.Printf("Use auto config?\n")
+		fmt.Printf(" * Say Y if not sure\n")
+		fmt.Printf(" * Say N if you are working on a remote or headless machine\n")
+		auto := config.Confirm()
+		if !auto {
 			fmt.Printf("For this to work, you will need rclone available on a machine that has a web browser available.\n")
 			fmt.Printf("Execute the following on your machine:\n")
 			if changed {
@@ -411,9 +407,12 @@ func doConfig(id, name string, m configmap.Mapper, errorHandler func(*http.Reque
 			return PutToken(name, m, token, true)
 		}
 	case TitleBarRedirectURL:
-		useWebServer = authorizeOnly
-		if !authorizeOnly {
-			useWebServer = isLocal()
+		useWebServer = automatic
+		if !automatic {
+			fmt.Printf("Use auto config?\n")
+			fmt.Printf(" * Say Y if not sure\n")
+			fmt.Printf(" * Say N if you are working on a remote or headless machine or Y didn't work\n")
+			useWebServer = config.Confirm()
 		}
 		if useWebServer {
 			// copy the config and set to use the internal webserver
@@ -480,12 +479,12 @@ func doConfig(id, name string, m configmap.Mapper, errorHandler func(*http.Reque
 	}
 
 	// Print code if we do automatic retrieval
-	if authorizeOnly {
+	if automatic {
 		result, err := json.Marshal(token)
 		if err != nil {
 			return errors.Wrap(err, "failed to marshal token")
 		}
-		fmt.Printf("Paste the following into your remote machine --->\n%s\n<---End paste\n", result)
+		fmt.Printf("Paste the following into your remote machine --->\n%s\n<---End paste", result)
 	}
 	return PutToken(name, m, token, true)
 }

vendor/github.com/anacrolix/dms/LICENSE

@@ -1,24 +0,0 @@
-Copyright (c) 2012, Matt Joiner <anacrolix@gmail.com>.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-    * Neither the name of the <organization> nor the
-      names of its contributors may be used to endorse or promote products
-      derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/anacrolix/dms/dlna/dlna.go

@@ -1,102 +0,0 @@
-package dlna
-
-import (
-	"fmt"
-	"strings"
-	"time"
-)
-
-const (
-	TimeSeekRangeDomain   = "TimeSeekRange.dlna.org"
-	ContentFeaturesDomain = "contentFeatures.dlna.org"
-	TransferModeDomain    = "transferMode.dlna.org"
-)
-
-type ContentFeatures struct {
-	ProfileName     string
-	SupportTimeSeek bool
-	SupportRange    bool
-	// Play speeds, DLNA.ORG_PS would go here if supported.
-	Transcoded bool
-}
-
-func BinaryInt(b bool) uint {
-	if b {
-		return 1
-	} else {
-		return 0
-	}
-}
-
-// flags are in hex. trailing 24 zeroes, 26 are after the space
-// "DLNA.ORG_OP=" time-seek-range-supp bytes-range-header-supp
-func (cf ContentFeatures) String() (ret string) {
-	//DLNA.ORG_PN=[a-zA-Z0-9_]*
-	params := make([]string, 0, 2)
-	if cf.ProfileName != "" {
-		params = append(params, "DLNA.ORG_PN="+cf.ProfileName)
-	}
-	params = append(params, fmt.Sprintf(
-		"DLNA.ORG_OP=%b%b;DLNA.ORG_CI=%b",
-		BinaryInt(cf.SupportTimeSeek),
-		BinaryInt(cf.SupportRange),
-		BinaryInt(cf.Transcoded)))
-	return strings.Join(params, ";")
-}
-
-func ParseNPTTime(s string) (time.Duration, error) {
-	var h, m, sec, ms time.Duration
-	n, err := fmt.Sscanf(s, "%d:%2d:%2d.%3d", &h, &m, &sec, &ms)
-	if err != nil {
-		return -1, err
-	}
-	if n < 3 {
-		return -1, fmt.Errorf("invalid npt time: %s", s)
-	}
-	ret := time.Duration(h) * time.Hour
-	ret += time.Duration(m) * time.Minute
-	ret += sec * time.Second
-	ret += ms * time.Millisecond
-	return ret, nil
-}
-
-func FormatNPTTime(npt time.Duration) string {
-	npt /= time.Millisecond
-	ms := npt % 1000
-	npt /= 1000
-	s := npt % 60
-	npt /= 60
-	m := npt % 60
-	npt /= 60
-	h := npt
-	return fmt.Sprintf("%02d:%02d:%02d.%03d", h, m, s, ms)
-}
-
-type NPTRange struct {
-	Start, End time.Duration
-}
-
-func ParseNPTRange(s string) (ret NPTRange, err error) {
-	ss := strings.SplitN(s, "-", 2)
-	if ss[0] != "" {
-		ret.Start, err = ParseNPTTime(ss[0])
-		if err != nil {
-			return
-		}
-	}
-	if ss[1] != "" {
-		ret.End, err = ParseNPTTime(ss[1])
-		if err != nil {
-			return
-		}
-	}
-	return
-}
-
-func (me NPTRange) String() (ret string) {
-	ret = me.Start.String() + "-"
-	if me.End >= 0 {
-		ret += me.End.String()
-	}
-	return
-}

vendor/github.com/anacrolix/dms/soap/soap.go

@@ -1,68 +0,0 @@
-package soap
-
-import (
-	"encoding/xml"
-)
-
-const (
-	EncodingStyle = "http://schemas.xmlsoap.org/soap/encoding/"
-	EnvelopeNS    = "http://schemas.xmlsoap.org/soap/envelope/"
-)
-
-type Arg struct {
-	XMLName xml.Name
-	Value   string `xml:",chardata"`
-}
-
-type Action struct {
-	XMLName xml.Name
-	Args    []Arg
-}
-
-type Body struct {
-	Action []byte `xml:",innerxml"`
-}
-
-type UPnPError struct {
-	XMLName xml.Name `xml:"urn:schemas-upnp-org:control-1-0 UPnPError"`
-	Code    uint     `xml:"errorCode"`
-	Desc    string   `xml:"errorDescription"`
-}
-
-type FaultDetail struct {
-	XMLName xml.Name `xml:"detail"`
-	Data    interface{}
-}
-
-type Fault struct {
-	XMLName     xml.Name    `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault"`
-	FaultCode   string      `xml:"faultcode"`
-	FaultString string      `xml:"faultstring"`
-	Detail      FaultDetail `xml:"detail"`
-}
-
-func NewFault(s string, detail interface{}) *Fault {
-	return &Fault{
-		FaultCode:   EnvelopeNS + ":Client",
-		FaultString: s,
-		Detail: FaultDetail{
-			Data: detail,
-		},
-	}
-}
-
-type Envelope struct {
-	XMLName       xml.Name `xml:"http://schemas.xmlsoap.org/soap/envelope/ Envelope"`
-	EncodingStyle string   `xml:"encodingStyle,attr"`
-	Body          Body     `xml:"http://schemas.xmlsoap.org/soap/envelope/ Body"`
-}
-
-/* XML marshalling of nested namespaces is broken.
-
-func NewEnvelope(action []byte) Envelope {
-	return Envelope{
-		EncodingStyle: EncodingStyle,
-		Body:          Body{action},
-	}
-}
-*/

vendor/github.com/anacrolix/dms/ssdp/ssdp.go

@@ -1,330 +0,0 @@
-package ssdp
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"io"
-	"log"
-	"math/rand"
-	"net"
-	"net/http"
-	"net/textproto"
-	"strconv"
-	"strings"
-	"time"
-
-	"golang.org/x/net/ipv4"
-)
-
-const (
-	AddrString = "239.255.255.250:1900"
-	rootDevice = "upnp:rootdevice"
-	aliveNTS   = "ssdp:alive"
-	byebyeNTS  = "ssdp:byebye"
-)
-
-var (
-	NetAddr *net.UDPAddr
-)
-
-func init() {
-	var err error
-	NetAddr, err = net.ResolveUDPAddr("udp4", AddrString)
-	if err != nil {
-		log.Panicf("Could not resolve %s: %s", AddrString, err)
-	}
-}
-
-type badStringError struct {
-	what string
-	str  string
-}
-
-func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) }
-
-func ReadRequest(b *bufio.Reader) (req *http.Request, err error) {
-	tp := textproto.NewReader(b)
-	var s string
-	if s, err = tp.ReadLine(); err != nil {
-		return nil, err
-	}
-	defer func() {
-		if err == io.EOF {
-			err = io.ErrUnexpectedEOF
-		}
-	}()
-
-	var f []string
-	// TODO a split that only allows N values?
-	if f = strings.SplitN(s, " ", 3); len(f) < 3 {
-		return nil, &badStringError{"malformed request line", s}
-	}
-	if f[1] != "*" {
-		return nil, &badStringError{"bad URL request", f[1]}
-	}
-	req = &http.Request{
-		Method: f[0],
-	}
-	var ok bool
-	if req.ProtoMajor, req.ProtoMinor, ok = http.ParseHTTPVersion(strings.TrimSpace(f[2])); !ok {
-		return nil, &badStringError{"malformed HTTP version", f[2]}
-	}
-
-	mimeHeader, err := tp.ReadMIMEHeader()
-	if err != nil {
-		return nil, err
-	}
-	req.Header = http.Header(mimeHeader)
-	return
-}
-
-type Server struct {
-	conn           *net.UDPConn
-	Interface      net.Interface
-	Server         string
-	Services       []string
-	Devices        []string
-	Location       func(net.IP) string
-	UUID           string
-	NotifyInterval time.Duration
-	closed         chan struct{}
-}
-
-func makeConn(ifi net.Interface) (ret *net.UDPConn, err error) {
-	ret, err = net.ListenMulticastUDP("udp", &ifi, NetAddr)
-	if err != nil {
-		return
-	}
-	p := ipv4.NewPacketConn(ret)
-	if err := p.SetMulticastTTL(2); err != nil {
-		log.Println(err)
-	}
-	if err := p.SetMulticastLoopback(true); err != nil {
-		log.Println(err)
-	}
-	return
-}
-
-func (me *Server) serve() {
-	for {
-		b := make([]byte, me.Interface.MTU)
-		n, addr, err := me.conn.ReadFromUDP(b)
-		select {
-		case <-me.closed:
-			return
-		default:
-		}
-		if err != nil {
-			log.Printf("error reading from UDP socket: %s", err)
-			break
-		}
-		go me.handle(b[:n], addr)
-	}
-}
-
-func (me *Server) Init() (err error) {
-	me.closed = make(chan struct{})
-	me.conn, err = makeConn(me.Interface)
-	return
-}
-
-func (me *Server) Close() {
-	close(me.closed)
-	me.sendByeBye()
-	me.conn.Close()
-}
-
-func (me *Server) Serve() (err error) {
-	go me.serve()
-	for {
-		addrs, err := me.Interface.Addrs()
-		if err != nil {
-			return err
-		}
-		for _, addr := range addrs {
-			ip := func() net.IP {
-				switch val := addr.(type) {
-				case *net.IPNet:
-					return val.IP
-				case *net.IPAddr:
-					return val.IP
-				}
-				panic(fmt.Sprint("unexpected addr type:", addr))
-			}()
-			extraHdrs := [][2]string{
-				{"CACHE-CONTROL", fmt.Sprintf("max-age=%d", 5*me.NotifyInterval/2/time.Second)},
-				{"LOCATION", me.Location(ip)},
-			}
-			me.notifyAll(aliveNTS, extraHdrs)
-		}
-		time.Sleep(me.NotifyInterval)
-	}
-}
-
-func (me *Server) usnFromTarget(target string) string {
-	if target == me.UUID {
-		return target
-	}
-	return me.UUID + "::" + target
-}
-
-func (me *Server) makeNotifyMessage(target, nts string, extraHdrs [][2]string) []byte {
-	lines := [...][2]string{
-		{"HOST", AddrString},
-		{"NT", target},
-		{"NTS", nts},
-		{"SERVER", me.Server},
-		{"USN", me.usnFromTarget(target)},
-	}
-	buf := &bytes.Buffer{}
-	fmt.Fprint(buf, "NOTIFY * HTTP/1.1\r\n")
-	writeHdr := func(keyValue [2]string) {
-		fmt.Fprintf(buf, "%s: %s\r\n", keyValue[0], keyValue[1])
-	}
-	for _, pair := range lines {
-		writeHdr(pair)
-	}
-	for _, pair := range extraHdrs {
-		writeHdr(pair)
-	}
-	fmt.Fprint(buf, "\r\n")
-	return buf.Bytes()
-}
-
-func (me *Server) send(buf []byte, addr *net.UDPAddr) {
-	if n, err := me.conn.WriteToUDP(buf, addr); err != nil {
-		log.Printf("error writing to UDP socket: %s", err)
-	} else if n != len(buf) {
-		log.Printf("short write: %d/%d bytes", n, len(buf))
-	}
-}
-
-func (me *Server) delayedSend(delay time.Duration, buf []byte, addr *net.UDPAddr) {
-	go func() {
-		select {
-		case <-time.After(delay):
-			me.send(buf, addr)
-		case <-me.closed:
-		}
-	}()
-}
-
-func (me *Server) log(args ...interface{}) {
-	args = append([]interface{}{me.Interface.Name + ":"}, args...)
-	log.Print(args...)
-}
-
-func (me *Server) sendByeBye() {
-	for _, type_ := range me.allTypes() {
-		buf := me.makeNotifyMessage(type_, byebyeNTS, nil)
-		me.send(buf, NetAddr)
-	}
-}
-
-func (me *Server) notifyAll(nts string, extraHdrs [][2]string) {
-	for _, type_ := range me.allTypes() {
-		buf := me.makeNotifyMessage(type_, nts, extraHdrs)
-		delay := time.Duration(rand.Int63n(int64(100 * time.Millisecond)))
-		me.delayedSend(delay, buf, NetAddr)
-	}
-}
-
-func (me *Server) allTypes() (ret []string) {
-	for _, a := range [][]string{
-		{rootDevice, me.UUID},
-		me.Devices,
-		me.Services,
-	} {
-		ret = append(ret, a...)
-	}
-	return
-}
-
-func (me *Server) handle(buf []byte, sender *net.UDPAddr) {
-	req, err := ReadRequest(bufio.NewReader(bytes.NewReader(buf)))
-	if err != nil {
-		log.Println(err)
-		return
-	}
-	if req.Method != "M-SEARCH" || req.Header.Get("man") != `"ssdp:discover"` {
-		return
-	}
-	var mx uint
-	if req.Header.Get("Host") == AddrString {
-		mxHeader := req.Header.Get("mx")
-		i, err := strconv.ParseUint(mxHeader, 0, 0)
-		if err != nil {
-			log.Printf("Invalid mx header %q: %s", mxHeader, err)
-			return
-		}
-		mx = uint(i)
-	} else {
-		mx = 1
-	}
-	types := func(st string) []string {
-		if st == "ssdp:all" {
-			return me.allTypes()
-		}
-		for _, t := range me.allTypes() {
-			if t == st {
-				return []string{t}
-			}
-		}
-		return nil
-	}(req.Header.Get("st"))
-	for _, ip := range func() (ret []net.IP) {
-		addrs, err := me.Interface.Addrs()
-		if err != nil {
-			panic(err)
-		}
-		for _, addr := range addrs {
-			if ip, ok := func() (net.IP, bool) {
-				switch data := addr.(type) {
-				case *net.IPNet:
-					if data.Contains(sender.IP) {
-						return data.IP, true
-					}
-					return nil, false
-				case *net.IPAddr:
-					return data.IP, true
-				}
-				panic(addr)
-			}(); ok {
-				ret = append(ret, ip)
-			}
-		}
-		return
-	}() {
-		for _, type_ := range types {
-			resp := me.makeResponse(ip, type_, req)
-			delay := time.Duration(rand.Int63n(int64(time.Second) * int64(mx)))
-			me.delayedSend(delay, resp, sender)
-		}
-	}
-}
-
-func (me *Server) makeResponse(ip net.IP, targ string, req *http.Request) (ret []byte) {
-	resp := &http.Response{
-		StatusCode: 200,
-		ProtoMajor: 1,
-		ProtoMinor: 1,
-		Header:     make(http.Header),
-		Request:    req,
-	}
-	for _, pair := range [...][2]string{
-		{"CACHE-CONTROL", fmt.Sprintf("max-age=%d", 5*me.NotifyInterval/2/time.Second)},
-		{"EXT", ""},
-		{"LOCATION", me.Location(ip)},
-		{"SERVER", me.Server},
-		{"ST", targ},
-		{"USN", me.usnFromTarget(targ)},
-	} {
-		resp.Header.Set(pair[0], pair[1])
-	}
-	buf := &bytes.Buffer{}
-	if err := resp.Write(buf); err != nil {
-		panic(err)
-	}
-	return buf.Bytes()
-}

vendor/github.com/anacrolix/dms/upnp/eventing.go

@@ -1,91 +0,0 @@
-package upnp
-
-import (
-	"crypto/rand"
-	"encoding/xml"
-	"fmt"
-	"io"
-	"log"
-	"net/url"
-	"regexp"
-	"time"
-)
-
-// TODO: Why use namespace prefixes in PropertySet et al? Because the spec
-// uses them, and I believe the Golang standard library XML spec implementers
-// incorrectly assume that you can get away with just xmlns="".
-
-// propertyset is the root element sent in an event callback.
-type PropertySet struct {
-	XMLName    struct{} `xml:"e:propertyset"`
-	Properties []Property
-	// This should be set to `"urn:schemas-upnp-org:event-1-0"`.
-	Space string `xml:"xmlns:e,attr"`
-}
-
-// propertys provide namespacing to the contained variables.
-type Property struct {
-	XMLName  struct{} `xml:"e:property"`
-	Variable Variable
-}
-
-// Represents an evented state variable that has sendEvents="yes" in its
-// service spec.
-type Variable struct {
-	XMLName xml.Name
-	Value   string `xml:",chardata"`
-}
-
-type subscriber struct {
-	sid     string
-	nextSeq uint32 // 0 for initial event, wraps from Uint32Max to 1.
-	urls    []*url.URL
-	expiry  time.Time
-}
-
-// Intended to eventually be an embeddable implementation for managing
-// eventing for a service. Not complete.
-type Eventing struct {
-	subscribers map[string]*subscriber
-}
-
-func (me *Eventing) Subscribe(callback []*url.URL, timeoutSeconds int) (sid string, actualTimeout int, err error) {
-	var uuid [16]byte
-	io.ReadFull(rand.Reader, uuid[:])
-	sid = FormatUUID(uuid[:])
-	if _, ok := me.subscribers[sid]; ok {
-		err = fmt.Errorf("already subscribed: %s", sid)
-		return
-	}
-	ssr := &subscriber{
-		sid:    sid,
-		urls:   callback,
-		expiry: time.Now().Add(time.Duration(timeoutSeconds) * time.Second),
-	}
-	if me.subscribers == nil {
-		me.subscribers = make(map[string]*subscriber)
-	}
-	me.subscribers[sid] = ssr
-	actualTimeout = int(ssr.expiry.Sub(time.Now()) / time.Second)
-	return
-}
-
-func (me *Eventing) Unsubscribe(sid string) error {
-	return nil
-}
-
-var callbackURLRegexp = regexp.MustCompile("<(.*?)>")
-
-// Parse the CALLBACK HTTP header in an event subscription request. See UPnP
-// Device Architecture 4.1.2.
-func ParseCallbackURLs(callback string) (ret []*url.URL) {
-	for _, match := range callbackURLRegexp.FindAllStringSubmatch(callback, -1) {
-		_url, err := url.Parse(match[1])
-		if err != nil {
-			log.Printf("bad callback url: %q", match[1])
-			continue
-		}
-		ret = append(ret, _url)
-	}
-	return
-}

vendor/github.com/anacrolix/dms/upnp/upnp.go

@@ -1,159 +0,0 @@
-package upnp
-
-import (
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"log"
-	"regexp"
-	"strconv"
-	"strings"
-)
-
-var serviceURNRegexp *regexp.Regexp = regexp.MustCompile(`^urn:schemas-upnp-org:service:(\w+):(\d+)$`)
-
-type ServiceURN struct {
-	Type    string
-	Version uint64
-}
-
-func (me ServiceURN) String() string {
-	return fmt.Sprintf("urn:schemas-upnp-org:service:%s:%d", me.Type, me.Version)
-}
-
-func ParseServiceType(s string) (ret ServiceURN, err error) {
-	matches := serviceURNRegexp.FindStringSubmatch(s)
-	if matches == nil {
-		err = errors.New(s)
-		return
-	}
-	if len(matches) != 3 {
-		log.Panicf("Invalid serviceURNRegexp ?")
-	}
-	ret.Type = matches[1]
-	ret.Version, err = strconv.ParseUint(matches[2], 0, 0)
-	return
-}
-
-type SoapAction struct {
-	ServiceURN
-	Action string
-}
-
-func ParseActionHTTPHeader(s string) (ret SoapAction, err error) {
-	if s[0] != '"' || s[len(s)-1] != '"' {
-		return
-	}
-	s = s[1 : len(s)-1]
-	hashIndex := strings.LastIndex(s, "#")
-	if hashIndex == -1 {
-		return
-	}
-	ret.Action = s[hashIndex+1:]
-	ret.ServiceURN, err = ParseServiceType(s[:hashIndex])
-	return
-}
-
-type SpecVersion struct {
-	Major int `xml:"major"`
-	Minor int `xml:"minor"`
-}
-
-type Icon struct {
-	Mimetype string `xml:"mimetype"`
-	Width    int    `xml:"width"`
-	Height   int    `xml:"height"`
-	Depth    int    `xml:"depth"`
-	URL      string `xml:"url"`
-}
-
-type Service struct {
-	XMLName     xml.Name `xml:"service"`
-	ServiceType string   `xml:"serviceType"`
-	ServiceId   string   `xml:"serviceId"`
-	SCPDURL     string
-	ControlURL  string `xml:"controlURL"`
-	EventSubURL string `xml:"eventSubURL"`
-}
-
-type Device struct {
-	DeviceType   string `xml:"deviceType"`
-	FriendlyName string `xml:"friendlyName"`
-	Manufacturer string `xml:"manufacturer"`
-	ModelName    string `xml:"modelName"`
-	UDN          string
-	IconList     []Icon    `xml:"iconList>icon"`
-	ServiceList  []Service `xml:"serviceList>service"`
-}
-
-type DeviceDesc struct {
-	XMLName     xml.Name    `xml:"urn:schemas-upnp-org:device-1-0 root"`
-	SpecVersion SpecVersion `xml:"specVersion"`
-	Device      Device      `xml:"device"`
-}
-
-type Error struct {
-	XMLName xml.Name `xml:"urn:schemas-upnp-org:control-1-0 UPnPError"`
-	Code    uint     `xml:"errorCode"`
-	Desc    string   `xml:"errorDescription"`
-}
-
-func (e *Error) Error() string {
-	return fmt.Sprintf("%d %s", e.Code, e.Desc)
-}
-
-const (
-	InvalidActionErrorCode        = 401
-	ActionFailedErrorCode         = 501
-	ArgumentValueInvalidErrorCode = 600
-)
-
-var (
-	InvalidActionError        = Errorf(401, "Invalid Action")
-	ArgumentValueInvalidError = Errorf(600, "The argument value is invalid")
-)
-
-// Errorf creates an UPNP error from the given code and description
-func Errorf(code uint, tpl string, args ...interface{}) *Error {
-	return &Error{Code: code, Desc: fmt.Sprintf(tpl, args...)}
-}
-
-// ConvertError converts any error to an UPNP error
-func ConvertError(err error) *Error {
-	if err == nil {
-		return nil
-	}
-	if e, ok := err.(*Error); ok {
-		return e
-	}
-	return Errorf(ActionFailedErrorCode, err.Error())
-}
-
-type Action struct {
-	Name      string
-	Arguments []Argument
-}
-
-type Argument struct {
-	Name            string
-	Direction       string
-	RelatedStateVar string
-}
-
-type SCPD struct {
-	XMLName           xml.Name        `xml:"urn:schemas-upnp-org:service-1-0 scpd"`
-	SpecVersion       SpecVersion     `xml:"specVersion"`
-	ActionList        []Action        `xml:"actionList>action"`
-	ServiceStateTable []StateVariable `xml:"serviceStateTable>stateVariable"`
-}
-
-type StateVariable struct {
-	SendEvents    string    `xml:"sendEvents,attr"`
-	Name          string    `xml:"name"`
-	DataType      string    `xml:"dataType"`
-	AllowedValues *[]string `xml:"allowedValueList>allowedValue,omitempty"`
-}
-
-func FormatUUID(buf []byte) string {
-	return fmt.Sprintf("uuid:%x-%x-%x-%x-%x", buf[:4], buf[4:6], buf[6:8], buf[8:10], buf[10:16])
-}

vendor/github.com/anacrolix/dms/upnpav/upnpav.go

@@ -1,45 +0,0 @@
-package upnpav
-
-import (
-	"encoding/xml"
-)
-
-const (
-	NoSuchObjectErrorCode = 701
-)
-
-type Resource struct {
-	XMLName      xml.Name `xml:"res"`
-	ProtocolInfo string   `xml:"protocolInfo,attr"`
-	URL          string   `xml:",chardata"`
-	Size         uint64   `xml:"size,attr,omitempty"`
-	Bitrate      uint     `xml:"bitrate,attr,omitempty"`
-	Duration     string   `xml:"duration,attr,omitempty"`
-	Resolution   string   `xml:"resolution,attr,omitempty"`
-}
-
-type Container struct {
-	Object
-	XMLName    xml.Name `xml:"container"`
-	ChildCount int      `xml:"childCount,attr"`
-}
-
-type Item struct {
-	Object
-	XMLName xml.Name `xml:"item"`
-	Res     []Resource
-}
-
-type Object struct {
-	ID          string `xml:"id,attr"`
-	ParentID    string `xml:"parentID,attr"`
-	Restricted  int    `xml:"restricted,attr"` // indicates whether the object is modifiable
-	Class       string `xml:"upnp:class"`
-	Icon        string `xml:"upnp:icon,omitempty"`
-	Title       string `xml:"dc:title"`
-	Artist      string `xml:"upnp:artist,omitempty"`
-	Album       string `xml:"upnp:album,omitempty"`
-	Genre       string `xml:"upnp:genre,omitempty"`
-	AlbumArtURI string `xml:"upnp:albumArtURI,omitempty"`
-	Searchable  int    `xml:"searchable,attr"`
-}

vendor/github.com/goftp/server/socket.go

@@ -14,7 +14,6 @@ import (
 	"strings"
 	"sync"
 	"syscall"
-	"time"
 )
 
 // DataSocket describes a data socket is used to send non-control data between the client and
@@ -209,23 +208,13 @@ func (socket *ftpPassiveSocket) GoListenAndServe(sessionID string) (err error) {
 		return
 	}
 
-	var tcplistener *net.TCPListener
-	tcplistener, err = net.ListenTCP("tcp", laddr)
+	var listener net.Listener
+	listener, err = net.ListenTCP("tcp", laddr)
 	if err != nil {
 		socket.logger.Print(sessionID, err)
 		return
 	}
 
-	// The timeout, for a remote client to establish connection
-	// with a PASV style data connection.
-	const acceptTimeout = 60 * time.Second
-	err = tcplistener.SetDeadline(time.Now().Add(acceptTimeout))
-	if err != nil {
-		socket.logger.Print(sessionID, err)
-		return
-	}
-
-	var listener net.Listener = tcplistener
 	add := listener.Addr()
 	parts := strings.Split(add.String(), ":")
 	port, err := strconv.Atoi(parts[len(parts)-1])

vendor/golang.org/x/net/bpf/asm.go

@@ -1,41 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bpf
-
-import "fmt"
-
-// Assemble converts insts into raw instructions suitable for loading
-// into a BPF virtual machine.
-//
-// Currently, no optimization is attempted, the assembled program flow
-// is exactly as provided.
-func Assemble(insts []Instruction) ([]RawInstruction, error) {
-	ret := make([]RawInstruction, len(insts))
-	var err error
-	for i, inst := range insts {
-		ret[i], err = inst.Assemble()
-		if err != nil {
-			return nil, fmt.Errorf("assembling instruction %d: %s", i+1, err)
-		}
-	}
-	return ret, nil
-}
-
-// Disassemble attempts to parse raw back into
-// Instructions. Unrecognized RawInstructions are assumed to be an
-// extension not implemented by this package, and are passed through
-// unchanged to the output. The allDecoded value reports whether insts
-// contains no RawInstructions.
-func Disassemble(raw []RawInstruction) (insts []Instruction, allDecoded bool) {
-	insts = make([]Instruction, len(raw))
-	allDecoded = true
-	for i, r := range raw {
-		insts[i] = r.Disassemble()
-		if _, ok := insts[i].(RawInstruction); ok {
-			allDecoded = false
-		}
-	}
-	return insts, allDecoded
-}

vendor/golang.org/x/net/bpf/constants.go

@@ -1,222 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bpf
-
-// A Register is a register of the BPF virtual machine.
-type Register uint16
-
-const (
-	// RegA is the accumulator register. RegA is always the
-	// destination register of ALU operations.
-	RegA Register = iota
-	// RegX is the indirection register, used by LoadIndirect
-	// operations.
-	RegX
-)
-
-// An ALUOp is an arithmetic or logic operation.
-type ALUOp uint16
-
-// ALU binary operation types.
-const (
-	ALUOpAdd ALUOp = iota << 4
-	ALUOpSub
-	ALUOpMul
-	ALUOpDiv
-	ALUOpOr
-	ALUOpAnd
-	ALUOpShiftLeft
-	ALUOpShiftRight
-	aluOpNeg // Not exported because it's the only unary ALU operation, and gets its own instruction type.
-	ALUOpMod
-	ALUOpXor
-)
-
-// A JumpTest is a comparison operator used in conditional jumps.
-type JumpTest uint16
-
-// Supported operators for conditional jumps.
-// K can be RegX for JumpIfX
-const (
-	// K == A
-	JumpEqual JumpTest = iota
-	// K != A
-	JumpNotEqual
-	// K > A
-	JumpGreaterThan
-	// K < A
-	JumpLessThan
-	// K >= A
-	JumpGreaterOrEqual
-	// K <= A
-	JumpLessOrEqual
-	// K & A != 0
-	JumpBitsSet
-	// K & A == 0
-	JumpBitsNotSet
-)
-
-// An Extension is a function call provided by the kernel that
-// performs advanced operations that are expensive or impossible
-// within the BPF virtual machine.
-//
-// Extensions are only implemented by the Linux kernel.
-//
-// TODO: should we prune this list? Some of these extensions seem
-// either broken or near-impossible to use correctly, whereas other
-// (len, random, ifindex) are quite useful.
-type Extension int
-
-// Extension functions available in the Linux kernel.
-const (
-	// extOffset is the negative maximum number of instructions used
-	// to load instructions by overloading the K argument.
-	extOffset = -0x1000
-	// ExtLen returns the length of the packet.
-	ExtLen Extension = 1
-	// ExtProto returns the packet's L3 protocol type.
-	ExtProto Extension = 0
-	// ExtType returns the packet's type (skb->pkt_type in the kernel)
-	//
-	// TODO: better documentation. How nice an API do we want to
-	// provide for these esoteric extensions?
-	ExtType Extension = 4
-	// ExtPayloadOffset returns the offset of the packet payload, or
-	// the first protocol header that the kernel does not know how to
-	// parse.
-	ExtPayloadOffset Extension = 52
-	// ExtInterfaceIndex returns the index of the interface on which
-	// the packet was received.
-	ExtInterfaceIndex Extension = 8
-	// ExtNetlinkAttr returns the netlink attribute of type X at
-	// offset A.
-	ExtNetlinkAttr Extension = 12
-	// ExtNetlinkAttrNested returns the nested netlink attribute of
-	// type X at offset A.
-	ExtNetlinkAttrNested Extension = 16
-	// ExtMark returns the packet's mark value.
-	ExtMark Extension = 20
-	// ExtQueue returns the packet's assigned hardware queue.
-	ExtQueue Extension = 24
-	// ExtLinkLayerType returns the packet's hardware address type
-	// (e.g. Ethernet, Infiniband).
-	ExtLinkLayerType Extension = 28
-	// ExtRXHash returns the packets receive hash.
-	//
-	// TODO: figure out what this rxhash actually is.
-	ExtRXHash Extension = 32
-	// ExtCPUID returns the ID of the CPU processing the current
-	// packet.
-	ExtCPUID Extension = 36
-	// ExtVLANTag returns the packet's VLAN tag.
-	ExtVLANTag Extension = 44
-	// ExtVLANTagPresent returns non-zero if the packet has a VLAN
-	// tag.
-	//
-	// TODO: I think this might be a lie: it reads bit 0x1000 of the
-	// VLAN header, which changed meaning in recent revisions of the
-	// spec - this extension may now return meaningless information.
-	ExtVLANTagPresent Extension = 48
-	// ExtVLANProto returns 0x8100 if the frame has a VLAN header,
-	// 0x88a8 if the frame has a "Q-in-Q" double VLAN header, or some
-	// other value if no VLAN information is present.
-	ExtVLANProto Extension = 60
-	// ExtRand returns a uniformly random uint32.
-	ExtRand Extension = 56
-)
-
-// The following gives names to various bit patterns used in opcode construction.
-
-const (
-	opMaskCls uint16 = 0x7
-	// opClsLoad masks
-	opMaskLoadDest  = 0x01
-	opMaskLoadWidth = 0x18
-	opMaskLoadMode  = 0xe0
-	// opClsALU & opClsJump
-	opMaskOperand  = 0x08
-	opMaskOperator = 0xf0
-)
-
-const (
-	// +---------------+-----------------+---+---+---+
-	// | AddrMode (3b) | LoadWidth (2b)  | 0 | 0 | 0 |
-	// +---------------+-----------------+---+---+---+
-	opClsLoadA uint16 = iota
-	// +---------------+-----------------+---+---+---+
-	// | AddrMode (3b) | LoadWidth (2b)  | 0 | 0 | 1 |
-	// +---------------+-----------------+---+---+---+
-	opClsLoadX
-	// +---+---+---+---+---+---+---+---+
-	// | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 |
-	// +---+---+---+---+---+---+---+---+
-	opClsStoreA
-	// +---+---+---+---+---+---+---+---+
-	// | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 |
-	// +---+---+---+---+---+---+---+---+
-	opClsStoreX
-	// +---------------+-----------------+---+---+---+
-	// | Operator (4b) | OperandSrc (1b) | 1 | 0 | 0 |
-	// +---------------+-----------------+---+---+---+
-	opClsALU
-	// +-----------------------------+---+---+---+---+
-	// |      TestOperator (4b)      | 0 | 1 | 0 | 1 |
-	// +-----------------------------+---+---+---+---+
-	opClsJump
-	// +---+-------------------------+---+---+---+---+
-	// | 0 | 0 | 0 |   RetSrc (1b)   | 0 | 1 | 1 | 0 |
-	// +---+-------------------------+---+---+---+---+
-	opClsReturn
-	// +---+-------------------------+---+---+---+---+
-	// | 0 | 0 | 0 |  TXAorTAX (1b)  | 0 | 1 | 1 | 1 |
-	// +---+-------------------------+---+---+---+---+
-	opClsMisc
-)
-
-const (
-	opAddrModeImmediate uint16 = iota << 5
-	opAddrModeAbsolute
-	opAddrModeIndirect
-	opAddrModeScratch
-	opAddrModePacketLen // actually an extension, not an addressing mode.
-	opAddrModeMemShift
-)
-
-const (
-	opLoadWidth4 uint16 = iota << 3
-	opLoadWidth2
-	opLoadWidth1
-)
-
-// Operand for ALU and Jump instructions
-type opOperand uint16
-
-// Supported operand sources.
-const (
-	opOperandConstant opOperand = iota << 3
-	opOperandX
-)
-
-// An jumpOp is a conditional jump condition.
-type jumpOp uint16
-
-// Supported jump conditions.
-const (
-	opJumpAlways jumpOp = iota << 4
-	opJumpEqual
-	opJumpGT
-	opJumpGE
-	opJumpSet
-)
-
-const (
-	opRetSrcConstant uint16 = iota << 4
-	opRetSrcA
-)
-
-const (
-	opMiscTAX = 0x00
-	opMiscTXA = 0x80
-)

vendor/golang.org/x/net/bpf/doc.go

@@ -1,82 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-/*
-
-Package bpf implements marshaling and unmarshaling of programs for the
-Berkeley Packet Filter virtual machine, and provides a Go implementation
-of the virtual machine.
-
-BPF's main use is to specify a packet filter for network taps, so that
-the kernel doesn't have to expensively copy every packet it sees to
-userspace. However, it's been repurposed to other areas where running
-user code in-kernel is needed. For example, Linux's seccomp uses BPF
-to apply security policies to system calls. For simplicity, this
-documentation refers only to packets, but other uses of BPF have their
-own data payloads.
-
-BPF programs run in a restricted virtual machine. It has almost no
-access to kernel functions, and while conditional branches are
-allowed, they can only jump forwards, to guarantee that there are no
-infinite loops.
-
-The virtual machine
-
-The BPF VM is an accumulator machine. Its main register, called
-register A, is an implicit source and destination in all arithmetic
-and logic operations. The machine also has 16 scratch registers for
-temporary storage, and an indirection register (register X) for
-indirect memory access. All registers are 32 bits wide.
-
-Each run of a BPF program is given one packet, which is placed in the
-VM's read-only "main memory". LoadAbsolute and LoadIndirect
-instructions can fetch up to 32 bits at a time into register A for
-examination.
-
-The goal of a BPF program is to produce and return a verdict (uint32),
-which tells the kernel what to do with the packet. In the context of
-packet filtering, the returned value is the number of bytes of the
-packet to forward to userspace, or 0 to ignore the packet. Other
-contexts like seccomp define their own return values.
-
-In order to simplify programs, attempts to read past the end of the
-packet terminate the program execution with a verdict of 0 (ignore
-packet). This means that the vast majority of BPF programs don't need
-to do any explicit bounds checking.
-
-In addition to the bytes of the packet, some BPF programs have access
-to extensions, which are essentially calls to kernel utility
-functions. Currently, the only extensions supported by this package
-are the Linux packet filter extensions.
-
-Examples
-
-This packet filter selects all ARP packets.
-
-	bpf.Assemble([]bpf.Instruction{
-		// Load "EtherType" field from the ethernet header.
-		bpf.LoadAbsolute{Off: 12, Size: 2},
-		// Skip over the next instruction if EtherType is not ARP.
-		bpf.JumpIf{Cond: bpf.JumpNotEqual, Val: 0x0806, SkipTrue: 1},
-		// Verdict is "send up to 4k of the packet to userspace."
-		bpf.RetConstant{Val: 4096},
-		// Verdict is "ignore packet."
-		bpf.RetConstant{Val: 0},
-	})
-
-This packet filter captures a random 1% sample of traffic.
-
-	bpf.Assemble([]bpf.Instruction{
-		// Get a 32-bit random number from the Linux kernel.
-		bpf.LoadExtension{Num: bpf.ExtRand},
-		// 1% dice roll?
-		bpf.JumpIf{Cond: bpf.JumpLessThan, Val: 2^32/100, SkipFalse: 1},
-		// Capture.
-		bpf.RetConstant{Val: 4096},
-		// Ignore.
-		bpf.RetConstant{Val: 0},
-	})
-
-*/
-package bpf // import "golang.org/x/net/bpf"

vendor/golang.org/x/net/bpf/instructions.go

@@ -1,726 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bpf
-
-import "fmt"
-
-// An Instruction is one instruction executed by the BPF virtual
-// machine.
-type Instruction interface {
-	// Assemble assembles the Instruction into a RawInstruction.
-	Assemble() (RawInstruction, error)
-}
-
-// A RawInstruction is a raw BPF virtual machine instruction.
-type RawInstruction struct {
-	// Operation to execute.
-	Op uint16
-	// For conditional jump instructions, the number of instructions
-	// to skip if the condition is true/false.
-	Jt uint8
-	Jf uint8
-	// Constant parameter. The meaning depends on the Op.
-	K uint32
-}
-
-// Assemble implements the Instruction Assemble method.
-func (ri RawInstruction) Assemble() (RawInstruction, error) { return ri, nil }
-
-// Disassemble parses ri into an Instruction and returns it. If ri is
-// not recognized by this package, ri itself is returned.
-func (ri RawInstruction) Disassemble() Instruction {
-	switch ri.Op & opMaskCls {
-	case opClsLoadA, opClsLoadX:
-		reg := Register(ri.Op & opMaskLoadDest)
-		sz := 0
-		switch ri.Op & opMaskLoadWidth {
-		case opLoadWidth4:
-			sz = 4
-		case opLoadWidth2:
-			sz = 2
-		case opLoadWidth1:
-			sz = 1
-		default:
-			return ri
-		}
-		switch ri.Op & opMaskLoadMode {
-		case opAddrModeImmediate:
-			if sz != 4 {
-				return ri
-			}
-			return LoadConstant{Dst: reg, Val: ri.K}
-		case opAddrModeScratch:
-			if sz != 4 || ri.K > 15 {
-				return ri
-			}
-			return LoadScratch{Dst: reg, N: int(ri.K)}
-		case opAddrModeAbsolute:
-			if ri.K > extOffset+0xffffffff {
-				return LoadExtension{Num: Extension(-extOffset + ri.K)}
-			}
-			return LoadAbsolute{Size: sz, Off: ri.K}
-		case opAddrModeIndirect:
-			return LoadIndirect{Size: sz, Off: ri.K}
-		case opAddrModePacketLen:
-			if sz != 4 {
-				return ri
-			}
-			return LoadExtension{Num: ExtLen}
-		case opAddrModeMemShift:
-			return LoadMemShift{Off: ri.K}
-		default:
-			return ri
-		}
-
-	case opClsStoreA:
-		if ri.Op != opClsStoreA || ri.K > 15 {
-			return ri
-		}
-		return StoreScratch{Src: RegA, N: int(ri.K)}
-
-	case opClsStoreX:
-		if ri.Op != opClsStoreX || ri.K > 15 {
-			return ri
-		}
-		return StoreScratch{Src: RegX, N: int(ri.K)}
-
-	case opClsALU:
-		switch op := ALUOp(ri.Op & opMaskOperator); op {
-		case ALUOpAdd, ALUOpSub, ALUOpMul, ALUOpDiv, ALUOpOr, ALUOpAnd, ALUOpShiftLeft, ALUOpShiftRight, ALUOpMod, ALUOpXor:
-			switch operand := opOperand(ri.Op & opMaskOperand); operand {
-			case opOperandX:
-				return ALUOpX{Op: op}
-			case opOperandConstant:
-				return ALUOpConstant{Op: op, Val: ri.K}
-			default:
-				return ri
-			}
-		case aluOpNeg:
-			return NegateA{}
-		default:
-			return ri
-		}
-
-	case opClsJump:
-		switch op := jumpOp(ri.Op & opMaskOperator); op {
-		case opJumpAlways:
-			return Jump{Skip: ri.K}
-		case opJumpEqual, opJumpGT, opJumpGE, opJumpSet:
-			cond, skipTrue, skipFalse := jumpOpToTest(op, ri.Jt, ri.Jf)
-			switch operand := opOperand(ri.Op & opMaskOperand); operand {
-			case opOperandX:
-				return JumpIfX{Cond: cond, SkipTrue: skipTrue, SkipFalse: skipFalse}
-			case opOperandConstant:
-				return JumpIf{Cond: cond, Val: ri.K, SkipTrue: skipTrue, SkipFalse: skipFalse}
-			default:
-				return ri
-			}
-		default:
-			return ri
-		}
-
-	case opClsReturn:
-		switch ri.Op {
-		case opClsReturn | opRetSrcA:
-			return RetA{}
-		case opClsReturn | opRetSrcConstant:
-			return RetConstant{Val: ri.K}
-		default:
-			return ri
-		}
-
-	case opClsMisc:
-		switch ri.Op {
-		case opClsMisc | opMiscTAX:
-			return TAX{}
-		case opClsMisc | opMiscTXA:
-			return TXA{}
-		default:
-			return ri
-		}
-
-	default:
-		panic("unreachable") // switch is exhaustive on the bit pattern
-	}
-}
-
-func jumpOpToTest(op jumpOp, skipTrue uint8, skipFalse uint8) (JumpTest, uint8, uint8) {
-	var test JumpTest
-
-	// Decode "fake" jump conditions that don't appear in machine code
-	// Ensures the Assemble -> Disassemble stage recreates the same instructions
-	// See https://github.com/golang/go/issues/18470
-	if skipTrue == 0 {
-		switch op {
-		case opJumpEqual:
-			test = JumpNotEqual
-		case opJumpGT:
-			test = JumpLessOrEqual
-		case opJumpGE:
-			test = JumpLessThan
-		case opJumpSet:
-			test = JumpBitsNotSet
-		}
-
-		return test, skipFalse, 0
-	}
-
-	switch op {
-	case opJumpEqual:
-		test = JumpEqual
-	case opJumpGT:
-		test = JumpGreaterThan
-	case opJumpGE:
-		test = JumpGreaterOrEqual
-	case opJumpSet:
-		test = JumpBitsSet
-	}
-
-	return test, skipTrue, skipFalse
-}
-
-// LoadConstant loads Val into register Dst.
-type LoadConstant struct {
-	Dst Register
-	Val uint32
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a LoadConstant) Assemble() (RawInstruction, error) {
-	return assembleLoad(a.Dst, 4, opAddrModeImmediate, a.Val)
-}
-
-// String returns the instruction in assembler notation.
-func (a LoadConstant) String() string {
-	switch a.Dst {
-	case RegA:
-		return fmt.Sprintf("ld #%d", a.Val)
-	case RegX:
-		return fmt.Sprintf("ldx #%d", a.Val)
-	default:
-		return fmt.Sprintf("unknown instruction: %#v", a)
-	}
-}
-
-// LoadScratch loads scratch[N] into register Dst.
-type LoadScratch struct {
-	Dst Register
-	N   int // 0-15
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a LoadScratch) Assemble() (RawInstruction, error) {
-	if a.N < 0 || a.N > 15 {
-		return RawInstruction{}, fmt.Errorf("invalid scratch slot %d", a.N)
-	}
-	return assembleLoad(a.Dst, 4, opAddrModeScratch, uint32(a.N))
-}
-
-// String returns the instruction in assembler notation.
-func (a LoadScratch) String() string {
-	switch a.Dst {
-	case RegA:
-		return fmt.Sprintf("ld M[%d]", a.N)
-	case RegX:
-		return fmt.Sprintf("ldx M[%d]", a.N)
-	default:
-		return fmt.Sprintf("unknown instruction: %#v", a)
-	}
-}
-
-// LoadAbsolute loads packet[Off:Off+Size] as an integer value into
-// register A.
-type LoadAbsolute struct {
-	Off  uint32
-	Size int // 1, 2 or 4
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a LoadAbsolute) Assemble() (RawInstruction, error) {
-	return assembleLoad(RegA, a.Size, opAddrModeAbsolute, a.Off)
-}
-
-// String returns the instruction in assembler notation.
-func (a LoadAbsolute) String() string {
-	switch a.Size {
-	case 1: // byte
-		return fmt.Sprintf("ldb [%d]", a.Off)
-	case 2: // half word
-		return fmt.Sprintf("ldh [%d]", a.Off)
-	case 4: // word
-		if a.Off > extOffset+0xffffffff {
-			return LoadExtension{Num: Extension(a.Off + 0x1000)}.String()
-		}
-		return fmt.Sprintf("ld [%d]", a.Off)
-	default:
-		return fmt.Sprintf("unknown instruction: %#v", a)
-	}
-}
-
-// LoadIndirect loads packet[X+Off:X+Off+Size] as an integer value
-// into register A.
-type LoadIndirect struct {
-	Off  uint32
-	Size int // 1, 2 or 4
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a LoadIndirect) Assemble() (RawInstruction, error) {
-	return assembleLoad(RegA, a.Size, opAddrModeIndirect, a.Off)
-}
-
-// String returns the instruction in assembler notation.
-func (a LoadIndirect) String() string {
-	switch a.Size {
-	case 1: // byte
-		return fmt.Sprintf("ldb [x + %d]", a.Off)
-	case 2: // half word
-		return fmt.Sprintf("ldh [x + %d]", a.Off)
-	case 4: // word
-		return fmt.Sprintf("ld [x + %d]", a.Off)
-	default:
-		return fmt.Sprintf("unknown instruction: %#v", a)
-	}
-}
-
-// LoadMemShift multiplies the first 4 bits of the byte at packet[Off]
-// by 4 and stores the result in register X.
-//
-// This instruction is mainly useful to load into X the length of an
-// IPv4 packet header in a single instruction, rather than have to do
-// the arithmetic on the header's first byte by hand.
-type LoadMemShift struct {
-	Off uint32
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a LoadMemShift) Assemble() (RawInstruction, error) {
-	return assembleLoad(RegX, 1, opAddrModeMemShift, a.Off)
-}
-
-// String returns the instruction in assembler notation.
-func (a LoadMemShift) String() string {
-	return fmt.Sprintf("ldx 4*([%d]&0xf)", a.Off)
-}
-
-// LoadExtension invokes a linux-specific extension and stores the
-// result in register A.
-type LoadExtension struct {
-	Num Extension
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a LoadExtension) Assemble() (RawInstruction, error) {
-	if a.Num == ExtLen {
-		return assembleLoad(RegA, 4, opAddrModePacketLen, 0)
-	}
-	return assembleLoad(RegA, 4, opAddrModeAbsolute, uint32(extOffset+a.Num))
-}
-
-// String returns the instruction in assembler notation.
-func (a LoadExtension) String() string {
-	switch a.Num {
-	case ExtLen:
-		return "ld #len"
-	case ExtProto:
-		return "ld #proto"
-	case ExtType:
-		return "ld #type"
-	case ExtPayloadOffset:
-		return "ld #poff"
-	case ExtInterfaceIndex:
-		return "ld #ifidx"
-	case ExtNetlinkAttr:
-		return "ld #nla"
-	case ExtNetlinkAttrNested:
-		return "ld #nlan"
-	case ExtMark:
-		return "ld #mark"
-	case ExtQueue:
-		return "ld #queue"
-	case ExtLinkLayerType:
-		return "ld #hatype"
-	case ExtRXHash:
-		return "ld #rxhash"
-	case ExtCPUID:
-		return "ld #cpu"
-	case ExtVLANTag:
-		return "ld #vlan_tci"
-	case ExtVLANTagPresent:
-		return "ld #vlan_avail"
-	case ExtVLANProto:
-		return "ld #vlan_tpid"
-	case ExtRand:
-		return "ld #rand"
-	default:
-		return fmt.Sprintf("unknown instruction: %#v", a)
-	}
-}
-
-// StoreScratch stores register Src into scratch[N].
-type StoreScratch struct {
-	Src Register
-	N   int // 0-15
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a StoreScratch) Assemble() (RawInstruction, error) {
-	if a.N < 0 || a.N > 15 {
-		return RawInstruction{}, fmt.Errorf("invalid scratch slot %d", a.N)
-	}
-	var op uint16
-	switch a.Src {
-	case RegA:
-		op = opClsStoreA
-	case RegX:
-		op = opClsStoreX
-	default:
-		return RawInstruction{}, fmt.Errorf("invalid source register %v", a.Src)
-	}
-
-	return RawInstruction{
-		Op: op,
-		K:  uint32(a.N),
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a StoreScratch) String() string {
-	switch a.Src {
-	case RegA:
-		return fmt.Sprintf("st M[%d]", a.N)
-	case RegX:
-		return fmt.Sprintf("stx M[%d]", a.N)
-	default:
-		return fmt.Sprintf("unknown instruction: %#v", a)
-	}
-}
-
-// ALUOpConstant executes A = A <Op> Val.
-type ALUOpConstant struct {
-	Op  ALUOp
-	Val uint32
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a ALUOpConstant) Assemble() (RawInstruction, error) {
-	return RawInstruction{
-		Op: opClsALU | uint16(opOperandConstant) | uint16(a.Op),
-		K:  a.Val,
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a ALUOpConstant) String() string {
-	switch a.Op {
-	case ALUOpAdd:
-		return fmt.Sprintf("add #%d", a.Val)
-	case ALUOpSub:
-		return fmt.Sprintf("sub #%d", a.Val)
-	case ALUOpMul:
-		return fmt.Sprintf("mul #%d", a.Val)
-	case ALUOpDiv:
-		return fmt.Sprintf("div #%d", a.Val)
-	case ALUOpMod:
-		return fmt.Sprintf("mod #%d", a.Val)
-	case ALUOpAnd:
-		return fmt.Sprintf("and #%d", a.Val)
-	case ALUOpOr:
-		return fmt.Sprintf("or #%d", a.Val)
-	case ALUOpXor:
-		return fmt.Sprintf("xor #%d", a.Val)
-	case ALUOpShiftLeft:
-		return fmt.Sprintf("lsh #%d", a.Val)
-	case ALUOpShiftRight:
-		return fmt.Sprintf("rsh #%d", a.Val)
-	default:
-		return fmt.Sprintf("unknown instruction: %#v", a)
-	}
-}
-
-// ALUOpX executes A = A <Op> X
-type ALUOpX struct {
-	Op ALUOp
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a ALUOpX) Assemble() (RawInstruction, error) {
-	return RawInstruction{
-		Op: opClsALU | uint16(opOperandX) | uint16(a.Op),
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a ALUOpX) String() string {
-	switch a.Op {
-	case ALUOpAdd:
-		return "add x"
-	case ALUOpSub:
-		return "sub x"
-	case ALUOpMul:
-		return "mul x"
-	case ALUOpDiv:
-		return "div x"
-	case ALUOpMod:
-		return "mod x"
-	case ALUOpAnd:
-		return "and x"
-	case ALUOpOr:
-		return "or x"
-	case ALUOpXor:
-		return "xor x"
-	case ALUOpShiftLeft:
-		return "lsh x"
-	case ALUOpShiftRight:
-		return "rsh x"
-	default:
-		return fmt.Sprintf("unknown instruction: %#v", a)
-	}
-}
-
-// NegateA executes A = -A.
-type NegateA struct{}
-
-// Assemble implements the Instruction Assemble method.
-func (a NegateA) Assemble() (RawInstruction, error) {
-	return RawInstruction{
-		Op: opClsALU | uint16(aluOpNeg),
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a NegateA) String() string {
-	return fmt.Sprintf("neg")
-}
-
-// Jump skips the following Skip instructions in the program.
-type Jump struct {
-	Skip uint32
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a Jump) Assemble() (RawInstruction, error) {
-	return RawInstruction{
-		Op: opClsJump | uint16(opJumpAlways),
-		K:  a.Skip,
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a Jump) String() string {
-	return fmt.Sprintf("ja %d", a.Skip)
-}
-
-// JumpIf skips the following Skip instructions in the program if A
-// <Cond> Val is true.
-type JumpIf struct {
-	Cond      JumpTest
-	Val       uint32
-	SkipTrue  uint8
-	SkipFalse uint8
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a JumpIf) Assemble() (RawInstruction, error) {
-	return jumpToRaw(a.Cond, opOperandConstant, a.Val, a.SkipTrue, a.SkipFalse)
-}
-
-// String returns the instruction in assembler notation.
-func (a JumpIf) String() string {
-	return jumpToString(a.Cond, fmt.Sprintf("#%d", a.Val), a.SkipTrue, a.SkipFalse)
-}
-
-// JumpIfX skips the following Skip instructions in the program if A
-// <Cond> X is true.
-type JumpIfX struct {
-	Cond      JumpTest
-	SkipTrue  uint8
-	SkipFalse uint8
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a JumpIfX) Assemble() (RawInstruction, error) {
-	return jumpToRaw(a.Cond, opOperandX, 0, a.SkipTrue, a.SkipFalse)
-}
-
-// String returns the instruction in assembler notation.
-func (a JumpIfX) String() string {
-	return jumpToString(a.Cond, "x", a.SkipTrue, a.SkipFalse)
-}
-
-// jumpToRaw assembles a jump instruction into a RawInstruction
-func jumpToRaw(test JumpTest, operand opOperand, k uint32, skipTrue, skipFalse uint8) (RawInstruction, error) {
-	var (
-		cond jumpOp
-		flip bool
-	)
-	switch test {
-	case JumpEqual:
-		cond = opJumpEqual
-	case JumpNotEqual:
-		cond, flip = opJumpEqual, true
-	case JumpGreaterThan:
-		cond = opJumpGT
-	case JumpLessThan:
-		cond, flip = opJumpGE, true
-	case JumpGreaterOrEqual:
-		cond = opJumpGE
-	case JumpLessOrEqual:
-		cond, flip = opJumpGT, true
-	case JumpBitsSet:
-		cond = opJumpSet
-	case JumpBitsNotSet:
-		cond, flip = opJumpSet, true
-	default:
-		return RawInstruction{}, fmt.Errorf("unknown JumpTest %v", test)
-	}
-	jt, jf := skipTrue, skipFalse
-	if flip {
-		jt, jf = jf, jt
-	}
-	return RawInstruction{
-		Op: opClsJump | uint16(cond) | uint16(operand),
-		Jt: jt,
-		Jf: jf,
-		K:  k,
-	}, nil
-}
-
-// jumpToString converts a jump instruction to assembler notation
-func jumpToString(cond JumpTest, operand string, skipTrue, skipFalse uint8) string {
-	switch cond {
-	// K == A
-	case JumpEqual:
-		return conditionalJump(operand, skipTrue, skipFalse, "jeq", "jneq")
-	// K != A
-	case JumpNotEqual:
-		return fmt.Sprintf("jneq %s,%d", operand, skipTrue)
-	// K > A
-	case JumpGreaterThan:
-		return conditionalJump(operand, skipTrue, skipFalse, "jgt", "jle")
-	// K < A
-	case JumpLessThan:
-		return fmt.Sprintf("jlt %s,%d", operand, skipTrue)
-	// K >= A
-	case JumpGreaterOrEqual:
-		return conditionalJump(operand, skipTrue, skipFalse, "jge", "jlt")
-	// K <= A
-	case JumpLessOrEqual:
-		return fmt.Sprintf("jle %s,%d", operand, skipTrue)
-	// K & A != 0
-	case JumpBitsSet:
-		if skipFalse > 0 {
-			return fmt.Sprintf("jset %s,%d,%d", operand, skipTrue, skipFalse)
-		}
-		return fmt.Sprintf("jset %s,%d", operand, skipTrue)
-	// K & A == 0, there is no assembler instruction for JumpBitNotSet, use JumpBitSet and invert skips
-	case JumpBitsNotSet:
-		return jumpToString(JumpBitsSet, operand, skipFalse, skipTrue)
-	default:
-		return fmt.Sprintf("unknown JumpTest %#v", cond)
-	}
-}
-
-func conditionalJump(operand string, skipTrue, skipFalse uint8, positiveJump, negativeJump string) string {
-	if skipTrue > 0 {
-		if skipFalse > 0 {
-			return fmt.Sprintf("%s %s,%d,%d", positiveJump, operand, skipTrue, skipFalse)
-		}
-		return fmt.Sprintf("%s %s,%d", positiveJump, operand, skipTrue)
-	}
-	return fmt.Sprintf("%s %s,%d", negativeJump, operand, skipFalse)
-}
-
-// RetA exits the BPF program, returning the value of register A.
-type RetA struct{}
-
-// Assemble implements the Instruction Assemble method.
-func (a RetA) Assemble() (RawInstruction, error) {
-	return RawInstruction{
-		Op: opClsReturn | opRetSrcA,
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a RetA) String() string {
-	return fmt.Sprintf("ret a")
-}
-
-// RetConstant exits the BPF program, returning a constant value.
-type RetConstant struct {
-	Val uint32
-}
-
-// Assemble implements the Instruction Assemble method.
-func (a RetConstant) Assemble() (RawInstruction, error) {
-	return RawInstruction{
-		Op: opClsReturn | opRetSrcConstant,
-		K:  a.Val,
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a RetConstant) String() string {
-	return fmt.Sprintf("ret #%d", a.Val)
-}
-
-// TXA copies the value of register X to register A.
-type TXA struct{}
-
-// Assemble implements the Instruction Assemble method.
-func (a TXA) Assemble() (RawInstruction, error) {
-	return RawInstruction{
-		Op: opClsMisc | opMiscTXA,
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a TXA) String() string {
-	return fmt.Sprintf("txa")
-}
-
-// TAX copies the value of register A to register X.
-type TAX struct{}
-
-// Assemble implements the Instruction Assemble method.
-func (a TAX) Assemble() (RawInstruction, error) {
-	return RawInstruction{
-		Op: opClsMisc | opMiscTAX,
-	}, nil
-}
-
-// String returns the instruction in assembler notation.
-func (a TAX) String() string {
-	return fmt.Sprintf("tax")
-}
-
-func assembleLoad(dst Register, loadSize int, mode uint16, k uint32) (RawInstruction, error) {
-	var (
-		cls uint16
-		sz  uint16
-	)
-	switch dst {
-	case RegA:
-		cls = opClsLoadA
-	case RegX:
-		cls = opClsLoadX
-	default:
-		return RawInstruction{}, fmt.Errorf("invalid target register %v", dst)
-	}
-	switch loadSize {
-	case 1:
-		sz = opLoadWidth1
-	case 2:
-		sz = opLoadWidth2
-	case 4:
-		sz = opLoadWidth4
-	default:
-		return RawInstruction{}, fmt.Errorf("invalid load byte length %d", sz)
-	}
-	return RawInstruction{
-		Op: cls | sz | mode,
-		K:  k,
-	}, nil
-}

vendor/golang.org/x/net/bpf/setter.go

@@ -1,10 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bpf
-
-// A Setter is a type which can attach a compiled BPF filter to itself.
-type Setter interface {
-	SetBPF(filter []RawInstruction) error
-}

vendor/golang.org/x/net/bpf/vm.go

@@ -1,150 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bpf
-
-import (
-	"errors"
-	"fmt"
-)
-
-// A VM is an emulated BPF virtual machine.
-type VM struct {
-	filter []Instruction
-}
-
-// NewVM returns a new VM using the input BPF program.
-func NewVM(filter []Instruction) (*VM, error) {
-	if len(filter) == 0 {
-		return nil, errors.New("one or more Instructions must be specified")
-	}
-
-	for i, ins := range filter {
-		check := len(filter) - (i + 1)
-		switch ins := ins.(type) {
-		// Check for out-of-bounds jumps in instructions
-		case Jump:
-			if check <= int(ins.Skip) {
-				return nil, fmt.Errorf("cannot jump %d instructions; jumping past program bounds", ins.Skip)
-			}
-		case JumpIf:
-			if check <= int(ins.SkipTrue) {
-				return nil, fmt.Errorf("cannot jump %d instructions in true case; jumping past program bounds", ins.SkipTrue)
-			}
-			if check <= int(ins.SkipFalse) {
-				return nil, fmt.Errorf("cannot jump %d instructions in false case; jumping past program bounds", ins.SkipFalse)
-			}
-		case JumpIfX:
-			if check <= int(ins.SkipTrue) {
-				return nil, fmt.Errorf("cannot jump %d instructions in true case; jumping past program bounds", ins.SkipTrue)
-			}
-			if check <= int(ins.SkipFalse) {
-				return nil, fmt.Errorf("cannot jump %d instructions in false case; jumping past program bounds", ins.SkipFalse)
-			}
-		// Check for division or modulus by zero
-		case ALUOpConstant:
-			if ins.Val != 0 {
-				break
-			}
-
-			switch ins.Op {
-			case ALUOpDiv, ALUOpMod:
-				return nil, errors.New("cannot divide by zero using ALUOpConstant")
-			}
-		// Check for unknown extensions
-		case LoadExtension:
-			switch ins.Num {
-			case ExtLen:
-			default:
-				return nil, fmt.Errorf("extension %d not implemented", ins.Num)
-			}
-		}
-	}
-
-	// Make sure last instruction is a return instruction
-	switch filter[len(filter)-1].(type) {
-	case RetA, RetConstant:
-	default:
-		return nil, errors.New("BPF program must end with RetA or RetConstant")
-	}
-
-	// Though our VM works using disassembled instructions, we
-	// attempt to assemble the input filter anyway to ensure it is compatible
-	// with an operating system VM.
-	_, err := Assemble(filter)
-
-	return &VM{
-		filter: filter,
-	}, err
-}
-
-// Run runs the VM's BPF program against the input bytes.
-// Run returns the number of bytes accepted by the BPF program, and any errors
-// which occurred while processing the program.
-func (v *VM) Run(in []byte) (int, error) {
-	var (
-		// Registers of the virtual machine
-		regA       uint32
-		regX       uint32
-		regScratch [16]uint32
-
-		// OK is true if the program should continue processing the next
-		// instruction, or false if not, causing the loop to break
-		ok = true
-	)
-
-	// TODO(mdlayher): implement:
-	// - NegateA:
-	//   - would require a change from uint32 registers to int32
-	//     registers
-
-	// TODO(mdlayher): add interop tests that check signedness of ALU
-	// operations against kernel implementation, and make sure Go
-	// implementation matches behavior
-
-	for i := 0; i < len(v.filter) && ok; i++ {
-		ins := v.filter[i]
-
-		switch ins := ins.(type) {
-		case ALUOpConstant:
-			regA = aluOpConstant(ins, regA)
-		case ALUOpX:
-			regA, ok = aluOpX(ins, regA, regX)
-		case Jump:
-			i += int(ins.Skip)
-		case JumpIf:
-			jump := jumpIf(ins, regA)
-			i += jump
-		case JumpIfX:
-			jump := jumpIfX(ins, regA, regX)
-			i += jump
-		case LoadAbsolute:
-			regA, ok = loadAbsolute(ins, in)
-		case LoadConstant:
-			regA, regX = loadConstant(ins, regA, regX)
-		case LoadExtension:
-			regA = loadExtension(ins, in)
-		case LoadIndirect:
-			regA, ok = loadIndirect(ins, in, regX)
-		case LoadMemShift:
-			regX, ok = loadMemShift(ins, in)
-		case LoadScratch:
-			regA, regX = loadScratch(ins, regScratch, regA, regX)
-		case RetA:
-			return int(regA), nil
-		case RetConstant:
-			return int(ins.Val), nil
-		case StoreScratch:
-			regScratch = storeScratch(ins, regScratch, regA, regX)
-		case TAX:
-			regX = regA
-		case TXA:
-			regA = regX
-		default:
-			return 0, fmt.Errorf("unknown Instruction at index %d: %T", i, ins)
-		}
-	}
-
-	return 0, nil
-}

vendor/golang.org/x/net/bpf/vm_instructions.go

@@ -1,181 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package bpf
-
-import (
-	"encoding/binary"
-	"fmt"
-)
-
-func aluOpConstant(ins ALUOpConstant, regA uint32) uint32 {
-	return aluOpCommon(ins.Op, regA, ins.Val)
-}
-
-func aluOpX(ins ALUOpX, regA uint32, regX uint32) (uint32, bool) {
-	// Guard against division or modulus by zero by terminating
-	// the program, as the OS BPF VM does
-	if regX == 0 {
-		switch ins.Op {
-		case ALUOpDiv, ALUOpMod:
-			return 0, false
-		}
-	}
-
-	return aluOpCommon(ins.Op, regA, regX), true
-}
-
-func aluOpCommon(op ALUOp, regA uint32, value uint32) uint32 {
-	switch op {
-	case ALUOpAdd:
-		return regA + value
-	case ALUOpSub:
-		return regA - value
-	case ALUOpMul:
-		return regA * value
-	case ALUOpDiv:
-		// Division by zero not permitted by NewVM and aluOpX checks
-		return regA / value
-	case ALUOpOr:
-		return regA | value
-	case ALUOpAnd:
-		return regA & value
-	case ALUOpShiftLeft:
-		return regA << value
-	case ALUOpShiftRight:
-		return regA >> value
-	case ALUOpMod:
-		// Modulus by zero not permitted by NewVM and aluOpX checks
-		return regA % value
-	case ALUOpXor:
-		return regA ^ value
-	default:
-		return regA
-	}
-}
-
-func jumpIf(ins JumpIf, regA uint32) int {
-	return jumpIfCommon(ins.Cond, ins.SkipTrue, ins.SkipFalse, regA, ins.Val)
-}
-
-func jumpIfX(ins JumpIfX, regA uint32, regX uint32) int {
-	return jumpIfCommon(ins.Cond, ins.SkipTrue, ins.SkipFalse, regA, regX)
-}
-
-func jumpIfCommon(cond JumpTest, skipTrue, skipFalse uint8, regA uint32, value uint32) int {
-	var ok bool
-
-	switch cond {
-	case JumpEqual:
-		ok = regA == value
-	case JumpNotEqual:
-		ok = regA != value
-	case JumpGreaterThan:
-		ok = regA > value
-	case JumpLessThan:
-		ok = regA < value
-	case JumpGreaterOrEqual:
-		ok = regA >= value
-	case JumpLessOrEqual:
-		ok = regA <= value
-	case JumpBitsSet:
-		ok = (regA & value) != 0
-	case JumpBitsNotSet:
-		ok = (regA & value) == 0
-	}
-
-	if ok {
-		return int(skipTrue)
-	}
-
-	return int(skipFalse)
-}
-
-func loadAbsolute(ins LoadAbsolute, in []byte) (uint32, bool) {
-	offset := int(ins.Off)
-	size := int(ins.Size)
-
-	return loadCommon(in, offset, size)
-}
-
-func loadConstant(ins LoadConstant, regA uint32, regX uint32) (uint32, uint32) {
-	switch ins.Dst {
-	case RegA:
-		regA = ins.Val
-	case RegX:
-		regX = ins.Val
-	}
-
-	return regA, regX
-}
-
-func loadExtension(ins LoadExtension, in []byte) uint32 {
-	switch ins.Num {
-	case ExtLen:
-		return uint32(len(in))
-	default:
-		panic(fmt.Sprintf("unimplemented extension: %d", ins.Num))
-	}
-}
-
-func loadIndirect(ins LoadIndirect, in []byte, regX uint32) (uint32, bool) {
-	offset := int(ins.Off) + int(regX)
-	size := int(ins.Size)
-
-	return loadCommon(in, offset, size)
-}
-
-func loadMemShift(ins LoadMemShift, in []byte) (uint32, bool) {
-	offset := int(ins.Off)
-
-	if !inBounds(len(in), offset, 0) {
-		return 0, false
-	}
-
-	// Mask off high 4 bits and multiply low 4 bits by 4
-	return uint32(in[offset]&0x0f) * 4, true
-}
-
-func inBounds(inLen int, offset int, size int) bool {
-	return offset+size <= inLen
-}
-
-func loadCommon(in []byte, offset int, size int) (uint32, bool) {
-	if !inBounds(len(in), offset, size) {
-		return 0, false
-	}
-
-	switch size {
-	case 1:
-		return uint32(in[offset]), true
-	case 2:
-		return uint32(binary.BigEndian.Uint16(in[offset : offset+size])), true
-	case 4:
-		return uint32(binary.BigEndian.Uint32(in[offset : offset+size])), true
-	default:
-		panic(fmt.Sprintf("invalid load size: %d", size))
-	}
-}
-
-func loadScratch(ins LoadScratch, regScratch [16]uint32, regA uint32, regX uint32) (uint32, uint32) {
-	switch ins.Dst {
-	case RegA:
-		regA = regScratch[ins.N]
-	case RegX:
-		regX = regScratch[ins.N]
-	}
-
-	return regA, regX
-}
-
-func storeScratch(ins StoreScratch, regScratch [16]uint32, regA uint32, regX uint32) [16]uint32 {
-	switch ins.Src {
-	case RegA:
-		regScratch[ins.N] = regA
-	case RegX:
-		regScratch[ins.N] = regX
-	}
-
-	return regScratch
-}

vendor/golang.org/x/net/internal/iana/const.go

@@ -1,223 +0,0 @@
-// go generate gen.go
-// Code generated by the command above; DO NOT EDIT.
-
-// Package iana provides protocol number resources managed by the Internet Assigned Numbers Authority (IANA).
-package iana // import "golang.org/x/net/internal/iana"
-
-// Differentiated Services Field Codepoints (DSCP), Updated: 2018-05-04
-const (
-	DiffServCS0           = 0x00 // CS0
-	DiffServCS1           = 0x20 // CS1
-	DiffServCS2           = 0x40 // CS2
-	DiffServCS3           = 0x60 // CS3
-	DiffServCS4           = 0x80 // CS4
-	DiffServCS5           = 0xa0 // CS5
-	DiffServCS6           = 0xc0 // CS6
-	DiffServCS7           = 0xe0 // CS7
-	DiffServAF11          = 0x28 // AF11
-	DiffServAF12          = 0x30 // AF12
-	DiffServAF13          = 0x38 // AF13
-	DiffServAF21          = 0x48 // AF21
-	DiffServAF22          = 0x50 // AF22
-	DiffServAF23          = 0x58 // AF23
-	DiffServAF31          = 0x68 // AF31
-	DiffServAF32          = 0x70 // AF32
-	DiffServAF33          = 0x78 // AF33
-	DiffServAF41          = 0x88 // AF41
-	DiffServAF42          = 0x90 // AF42
-	DiffServAF43          = 0x98 // AF43
-	DiffServEF            = 0xb8 // EF
-	DiffServVOICEADMIT    = 0xb0 // VOICE-ADMIT
-	NotECNTransport       = 0x00 // Not-ECT (Not ECN-Capable Transport)
-	ECNTransport1         = 0x01 // ECT(1) (ECN-Capable Transport(1))
-	ECNTransport0         = 0x02 // ECT(0) (ECN-Capable Transport(0))
-	CongestionExperienced = 0x03 // CE (Congestion Experienced)
-)
-
-// Protocol Numbers, Updated: 2017-10-13
-const (
-	ProtocolIP             = 0   // IPv4 encapsulation, pseudo protocol number
-	ProtocolHOPOPT         = 0   // IPv6 Hop-by-Hop Option
-	ProtocolICMP           = 1   // Internet Control Message
-	ProtocolIGMP           = 2   // Internet Group Management
-	ProtocolGGP            = 3   // Gateway-to-Gateway
-	ProtocolIPv4           = 4   // IPv4 encapsulation
-	ProtocolST             = 5   // Stream
-	ProtocolTCP            = 6   // Transmission Control
-	ProtocolCBT            = 7   // CBT
-	ProtocolEGP            = 8   // Exterior Gateway Protocol
-	ProtocolIGP            = 9   // any private interior gateway (used by Cisco for their IGRP)
-	ProtocolBBNRCCMON      = 10  // BBN RCC Monitoring
-	ProtocolNVPII          = 11  // Network Voice Protocol
-	ProtocolPUP            = 12  // PUP
-	ProtocolEMCON          = 14  // EMCON
-	ProtocolXNET           = 15  // Cross Net Debugger
-	ProtocolCHAOS          = 16  // Chaos
-	ProtocolUDP            = 17  // User Datagram
-	ProtocolMUX            = 18  // Multiplexing
-	ProtocolDCNMEAS        = 19  // DCN Measurement Subsystems
-	ProtocolHMP            = 20  // Host Monitoring
-	ProtocolPRM            = 21  // Packet Radio Measurement
-	ProtocolXNSIDP         = 22  // XEROX NS IDP
-	ProtocolTRUNK1         = 23  // Trunk-1
-	ProtocolTRUNK2         = 24  // Trunk-2
-	ProtocolLEAF1          = 25  // Leaf-1
-	ProtocolLEAF2          = 26  // Leaf-2
-	ProtocolRDP            = 27  // Reliable Data Protocol
-	ProtocolIRTP           = 28  // Internet Reliable Transaction
-	ProtocolISOTP4         = 29  // ISO Transport Protocol Class 4
-	ProtocolNETBLT         = 30  // Bulk Data Transfer Protocol
-	ProtocolMFENSP         = 31  // MFE Network Services Protocol
-	ProtocolMERITINP       = 32  // MERIT Internodal Protocol
-	ProtocolDCCP           = 33  // Datagram Congestion Control Protocol
-	Protocol3PC            = 34  // Third Party Connect Protocol
-	ProtocolIDPR           = 35  // Inter-Domain Policy Routing Protocol
-	ProtocolXTP            = 36  // XTP
-	ProtocolDDP            = 37  // Datagram Delivery Protocol
-	ProtocolIDPRCMTP       = 38  // IDPR Control Message Transport Proto
-	ProtocolTPPP           = 39  // TP++ Transport Protocol
-	ProtocolIL             = 40  // IL Transport Protocol
-	ProtocolIPv6           = 41  // IPv6 encapsulation
-	ProtocolSDRP           = 42  // Source Demand Routing Protocol
-	ProtocolIPv6Route      = 43  // Routing Header for IPv6
-	ProtocolIPv6Frag       = 44  // Fragment Header for IPv6
-	ProtocolIDRP           = 45  // Inter-Domain Routing Protocol
-	ProtocolRSVP           = 46  // Reservation Protocol
-	ProtocolGRE            = 47  // Generic Routing Encapsulation
-	ProtocolDSR            = 48  // Dynamic Source Routing Protocol
-	ProtocolBNA            = 49  // BNA
-	ProtocolESP            = 50  // Encap Security Payload
-	ProtocolAH             = 51  // Authentication Header
-	ProtocolINLSP          = 52  // Integrated Net Layer Security  TUBA
-	ProtocolNARP           = 54  // NBMA Address Resolution Protocol
-	ProtocolMOBILE         = 55  // IP Mobility
-	ProtocolTLSP           = 56  // Transport Layer Security Protocol using Kryptonet key management
-	ProtocolSKIP           = 57  // SKIP
-	ProtocolIPv6ICMP       = 58  // ICMP for IPv6
-	ProtocolIPv6NoNxt      = 59  // No Next Header for IPv6
-	ProtocolIPv6Opts       = 60  // Destination Options for IPv6
-	ProtocolCFTP           = 62  // CFTP
-	ProtocolSATEXPAK       = 64  // SATNET and Backroom EXPAK
-	ProtocolKRYPTOLAN      = 65  // Kryptolan
-	ProtocolRVD            = 66  // MIT Remote Virtual Disk Protocol
-	ProtocolIPPC           = 67  // Internet Pluribus Packet Core
-	ProtocolSATMON         = 69  // SATNET Monitoring
-	ProtocolVISA           = 70  // VISA Protocol
-	ProtocolIPCV           = 71  // Internet Packet Core Utility
-	ProtocolCPNX           = 72  // Computer Protocol Network Executive
-	ProtocolCPHB           = 73  // Computer Protocol Heart Beat
-	ProtocolWSN            = 74  // Wang Span Network
-	ProtocolPVP            = 75  // Packet Video Protocol
-	ProtocolBRSATMON       = 76  // Backroom SATNET Monitoring
-	ProtocolSUNND          = 77  // SUN ND PROTOCOL-Temporary
-	ProtocolWBMON          = 78  // WIDEBAND Monitoring
-	ProtocolWBEXPAK        = 79  // WIDEBAND EXPAK
-	ProtocolISOIP          = 80  // ISO Internet Protocol
-	ProtocolVMTP           = 81  // VMTP
-	ProtocolSECUREVMTP     = 82  // SECURE-VMTP
-	ProtocolVINES          = 83  // VINES
-	ProtocolTTP            = 84  // Transaction Transport Protocol
-	ProtocolIPTM           = 84  // Internet Protocol Traffic Manager
-	ProtocolNSFNETIGP      = 85  // NSFNET-IGP
-	ProtocolDGP            = 86  // Dissimilar Gateway Protocol
-	ProtocolTCF            = 87  // TCF
-	ProtocolEIGRP          = 88  // EIGRP
-	ProtocolOSPFIGP        = 89  // OSPFIGP
-	ProtocolSpriteRPC      = 90  // Sprite RPC Protocol
-	ProtocolLARP           = 91  // Locus Address Resolution Protocol
-	ProtocolMTP            = 92  // Multicast Transport Protocol
-	ProtocolAX25           = 93  // AX.25 Frames
-	ProtocolIPIP           = 94  // IP-within-IP Encapsulation Protocol
-	ProtocolSCCSP          = 96  // Semaphore Communications Sec. Pro.
-	ProtocolETHERIP        = 97  // Ethernet-within-IP Encapsulation
-	ProtocolENCAP          = 98  // Encapsulation Header
-	ProtocolGMTP           = 100 // GMTP
-	ProtocolIFMP           = 101 // Ipsilon Flow Management Protocol
-	ProtocolPNNI           = 102 // PNNI over IP
-	ProtocolPIM            = 103 // Protocol Independent Multicast
-	ProtocolARIS           = 104 // ARIS
-	ProtocolSCPS           = 105 // SCPS
-	ProtocolQNX            = 106 // QNX
-	ProtocolAN             = 107 // Active Networks
-	ProtocolIPComp         = 108 // IP Payload Compression Protocol
-	ProtocolSNP            = 109 // Sitara Networks Protocol
-	ProtocolCompaqPeer     = 110 // Compaq Peer Protocol
-	ProtocolIPXinIP        = 111 // IPX in IP
-	ProtocolVRRP           = 112 // Virtual Router Redundancy Protocol
-	ProtocolPGM            = 113 // PGM Reliable Transport Protocol
-	ProtocolL2TP           = 115 // Layer Two Tunneling Protocol
-	ProtocolDDX            = 116 // D-II Data Exchange (DDX)
-	ProtocolIATP           = 117 // Interactive Agent Transfer Protocol
-	ProtocolSTP            = 118 // Schedule Transfer Protocol
-	ProtocolSRP            = 119 // SpectraLink Radio Protocol
-	ProtocolUTI            = 120 // UTI
-	ProtocolSMP            = 121 // Simple Message Protocol
-	ProtocolPTP            = 123 // Performance Transparency Protocol
-	ProtocolISIS           = 124 // ISIS over IPv4
-	ProtocolFIRE           = 125 // FIRE
-	ProtocolCRTP           = 126 // Combat Radio Transport Protocol
-	ProtocolCRUDP          = 127 // Combat Radio User Datagram
-	ProtocolSSCOPMCE       = 128 // SSCOPMCE
-	ProtocolIPLT           = 129 // IPLT
-	ProtocolSPS            = 130 // Secure Packet Shield
-	ProtocolPIPE           = 131 // Private IP Encapsulation within IP
-	ProtocolSCTP           = 132 // Stream Control Transmission Protocol
-	ProtocolFC             = 133 // Fibre Channel
-	ProtocolRSVPE2EIGNORE  = 134 // RSVP-E2E-IGNORE
-	ProtocolMobilityHeader = 135 // Mobility Header
-	ProtocolUDPLite        = 136 // UDPLite
-	ProtocolMPLSinIP       = 137 // MPLS-in-IP
-	ProtocolMANET          = 138 // MANET Protocols
-	ProtocolHIP            = 139 // Host Identity Protocol
-	ProtocolShim6          = 140 // Shim6 Protocol
-	ProtocolWESP           = 141 // Wrapped Encapsulating Security Payload
-	ProtocolROHC           = 142 // Robust Header Compression
-	ProtocolReserved       = 255 // Reserved
-)
-
-// Address Family Numbers, Updated: 2018-04-02
-const (
-	AddrFamilyIPv4                          = 1     // IP (IP version 4)
-	AddrFamilyIPv6                          = 2     // IP6 (IP version 6)
-	AddrFamilyNSAP                          = 3     // NSAP
-	AddrFamilyHDLC                          = 4     // HDLC (8-bit multidrop)
-	AddrFamilyBBN1822                       = 5     // BBN 1822
-	AddrFamily802                           = 6     // 802 (includes all 802 media plus Ethernet "canonical format")
-	AddrFamilyE163                          = 7     // E.163
-	AddrFamilyE164                          = 8     // E.164 (SMDS, Frame Relay, ATM)
-	AddrFamilyF69                           = 9     // F.69 (Telex)
-	AddrFamilyX121                          = 10    // X.121 (X.25, Frame Relay)
-	AddrFamilyIPX                           = 11    // IPX
-	AddrFamilyAppletalk                     = 12    // Appletalk
-	AddrFamilyDecnetIV                      = 13    // Decnet IV
-	AddrFamilyBanyanVines                   = 14    // Banyan Vines
-	AddrFamilyE164withSubaddress            = 15    // E.164 with NSAP format subaddress
-	AddrFamilyDNS                           = 16    // DNS (Domain Name System)
-	AddrFamilyDistinguishedName             = 17    // Distinguished Name
-	AddrFamilyASNumber                      = 18    // AS Number
-	AddrFamilyXTPoverIPv4                   = 19    // XTP over IP version 4
-	AddrFamilyXTPoverIPv6                   = 20    // XTP over IP version 6
-	AddrFamilyXTPnativemodeXTP              = 21    // XTP native mode XTP
-	AddrFamilyFibreChannelWorldWidePortName = 22    // Fibre Channel World-Wide Port Name
-	AddrFamilyFibreChannelWorldWideNodeName = 23    // Fibre Channel World-Wide Node Name
-	AddrFamilyGWID                          = 24    // GWID
-	AddrFamilyL2VPN                         = 25    // AFI for L2VPN information
-	AddrFamilyMPLSTPSectionEndpointID       = 26    // MPLS-TP Section Endpoint Identifier
-	AddrFamilyMPLSTPLSPEndpointID           = 27    // MPLS-TP LSP Endpoint Identifier
-	AddrFamilyMPLSTPPseudowireEndpointID    = 28    // MPLS-TP Pseudowire Endpoint Identifier
-	AddrFamilyMTIPv4                        = 29    // MT IP: Multi-Topology IP version 4
-	AddrFamilyMTIPv6                        = 30    // MT IPv6: Multi-Topology IP version 6
-	AddrFamilyEIGRPCommonServiceFamily      = 16384 // EIGRP Common Service Family
-	AddrFamilyEIGRPIPv4ServiceFamily        = 16385 // EIGRP IPv4 Service Family
-	AddrFamilyEIGRPIPv6ServiceFamily        = 16386 // EIGRP IPv6 Service Family
-	AddrFamilyLISPCanonicalAddressFormat    = 16387 // LISP Canonical Address Format (LCAF)
-	AddrFamilyBGPLS                         = 16388 // BGP-LS
-	AddrFamily48bitMAC                      = 16389 // 48-bit MAC
-	AddrFamily64bitMAC                      = 16390 // 64-bit MAC
-	AddrFamilyOUI                           = 16391 // OUI
-	AddrFamilyMACFinal24bits                = 16392 // MAC/24
-	AddrFamilyMACFinal40bits                = 16393 // MAC/40
-	AddrFamilyIPv6Initial64bits             = 16394 // IPv6/64
-	AddrFamilyRBridgePortID                 = 16395 // RBridge Port ID
-	AddrFamilyTRILLNickname                 = 16396 // TRILL Nickname
-)

vendor/golang.org/x/net/internal/iana/gen.go

@@ -1,383 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-//go:generate go run gen.go
-
-// This program generates internet protocol constants and tables by
-// reading IANA protocol registries.
-package main
-
-import (
-	"bytes"
-	"encoding/xml"
-	"fmt"
-	"go/format"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"strconv"
-	"strings"
-)
-
-var registries = []struct {
-	url   string
-	parse func(io.Writer, io.Reader) error
-}{
-	{
-		"https://www.iana.org/assignments/dscp-registry/dscp-registry.xml",
-		parseDSCPRegistry,
-	},
-	{
-		"https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xml",
-		parseProtocolNumbers,
-	},
-	{
-		"https://www.iana.org/assignments/address-family-numbers/address-family-numbers.xml",
-		parseAddrFamilyNumbers,
-	},
-}
-
-func main() {
-	var bb bytes.Buffer
-	fmt.Fprintf(&bb, "// go generate gen.go\n")
-	fmt.Fprintf(&bb, "// Code generated by the command above; DO NOT EDIT.\n\n")
-	fmt.Fprintf(&bb, "// Package iana provides protocol number resources managed by the Internet Assigned Numbers Authority (IANA).\n")
-	fmt.Fprintf(&bb, `package iana // import "golang.org/x/net/internal/iana"`+"\n\n")
-	for _, r := range registries {
-		resp, err := http.Get(r.url)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, err)
-			os.Exit(1)
-		}
-		defer resp.Body.Close()
-		if resp.StatusCode != http.StatusOK {
-			fmt.Fprintf(os.Stderr, "got HTTP status code %v for %v\n", resp.StatusCode, r.url)
-			os.Exit(1)
-		}
-		if err := r.parse(&bb, resp.Body); err != nil {
-			fmt.Fprintln(os.Stderr, err)
-			os.Exit(1)
-		}
-		fmt.Fprintf(&bb, "\n")
-	}
-	b, err := format.Source(bb.Bytes())
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-	if err := ioutil.WriteFile("const.go", b, 0644); err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
-}
-
-func parseDSCPRegistry(w io.Writer, r io.Reader) error {
-	dec := xml.NewDecoder(r)
-	var dr dscpRegistry
-	if err := dec.Decode(&dr); err != nil {
-		return err
-	}
-	fmt.Fprintf(w, "// %s, Updated: %s\n", dr.Title, dr.Updated)
-	fmt.Fprintf(w, "const (\n")
-	for _, dr := range dr.escapeDSCP() {
-		fmt.Fprintf(w, "DiffServ%s = %#02x", dr.Name, dr.Value)
-		fmt.Fprintf(w, "// %s\n", dr.OrigName)
-	}
-	for _, er := range dr.escapeECN() {
-		fmt.Fprintf(w, "%s = %#02x", er.Descr, er.Value)
-		fmt.Fprintf(w, "// %s\n", er.OrigDescr)
-	}
-	fmt.Fprintf(w, ")\n")
-	return nil
-}
-
-type dscpRegistry struct {
-	XMLName    xml.Name `xml:"registry"`
-	Title      string   `xml:"title"`
-	Updated    string   `xml:"updated"`
-	Note       string   `xml:"note"`
-	Registries []struct {
-		Title      string `xml:"title"`
-		Registries []struct {
-			Title   string `xml:"title"`
-			Records []struct {
-				Name  string `xml:"name"`
-				Space string `xml:"space"`
-			} `xml:"record"`
-		} `xml:"registry"`
-		Records []struct {
-			Value string `xml:"value"`
-			Descr string `xml:"description"`
-		} `xml:"record"`
-	} `xml:"registry"`
-}
-
-type canonDSCPRecord struct {
-	OrigName string
-	Name     string
-	Value    int
-}
-
-func (drr *dscpRegistry) escapeDSCP() []canonDSCPRecord {
-	var drs []canonDSCPRecord
-	for _, preg := range drr.Registries {
-		if !strings.Contains(preg.Title, "Differentiated Services Field Codepoints") {
-			continue
-		}
-		for _, reg := range preg.Registries {
-			if !strings.Contains(reg.Title, "Pool 1 Codepoints") {
-				continue
-			}
-			drs = make([]canonDSCPRecord, len(reg.Records))
-			sr := strings.NewReplacer(
-				"+", "",
-				"-", "",
-				"/", "",
-				".", "",
-				" ", "",
-			)
-			for i, dr := range reg.Records {
-				s := strings.TrimSpace(dr.Name)
-				drs[i].OrigName = s
-				drs[i].Name = sr.Replace(s)
-				n, err := strconv.ParseUint(dr.Space, 2, 8)
-				if err != nil {
-					continue
-				}
-				drs[i].Value = int(n) << 2
-			}
-		}
-	}
-	return drs
-}
-
-type canonECNRecord struct {
-	OrigDescr string
-	Descr     string
-	Value     int
-}
-
-func (drr *dscpRegistry) escapeECN() []canonECNRecord {
-	var ers []canonECNRecord
-	for _, reg := range drr.Registries {
-		if !strings.Contains(reg.Title, "ECN Field") {
-			continue
-		}
-		ers = make([]canonECNRecord, len(reg.Records))
-		sr := strings.NewReplacer(
-			"Capable", "",
-			"Not-ECT", "",
-			"ECT(1)", "",
-			"ECT(0)", "",
-			"CE", "",
-			"(", "",
-			")", "",
-			"+", "",
-			"-", "",
-			"/", "",
-			".", "",
-			" ", "",
-		)
-		for i, er := range reg.Records {
-			s := strings.TrimSpace(er.Descr)
-			ers[i].OrigDescr = s
-			ss := strings.Split(s, " ")
-			if len(ss) > 1 {
-				ers[i].Descr = strings.Join(ss[1:], " ")
-			} else {
-				ers[i].Descr = ss[0]
-			}
-			ers[i].Descr = sr.Replace(er.Descr)
-			n, err := strconv.ParseUint(er.Value, 2, 8)
-			if err != nil {
-				continue
-			}
-			ers[i].Value = int(n)
-		}
-	}
-	return ers
-}
-
-func parseProtocolNumbers(w io.Writer, r io.Reader) error {
-	dec := xml.NewDecoder(r)
-	var pn protocolNumbers
-	if err := dec.Decode(&pn); err != nil {
-		return err
-	}
-	prs := pn.escape()
-	prs = append([]canonProtocolRecord{{
-		Name:  "IP",
-		Descr: "IPv4 encapsulation, pseudo protocol number",
-		Value: 0,
-	}}, prs...)
-	fmt.Fprintf(w, "// %s, Updated: %s\n", pn.Title, pn.Updated)
-	fmt.Fprintf(w, "const (\n")
-	for _, pr := range prs {
-		if pr.Name == "" {
-			continue
-		}
-		fmt.Fprintf(w, "Protocol%s = %d", pr.Name, pr.Value)
-		s := pr.Descr
-		if s == "" {
-			s = pr.OrigName
-		}
-		fmt.Fprintf(w, "// %s\n", s)
-	}
-	fmt.Fprintf(w, ")\n")
-	return nil
-}
-
-type protocolNumbers struct {
-	XMLName  xml.Name `xml:"registry"`
-	Title    string   `xml:"title"`
-	Updated  string   `xml:"updated"`
-	RegTitle string   `xml:"registry>title"`
-	Note     string   `xml:"registry>note"`
-	Records  []struct {
-		Value string `xml:"value"`
-		Name  string `xml:"name"`
-		Descr string `xml:"description"`
-	} `xml:"registry>record"`
-}
-
-type canonProtocolRecord struct {
-	OrigName string
-	Name     string
-	Descr    string
-	Value    int
-}
-
-func (pn *protocolNumbers) escape() []canonProtocolRecord {
-	prs := make([]canonProtocolRecord, len(pn.Records))
-	sr := strings.NewReplacer(
-		"-in-", "in",
-		"-within-", "within",
-		"-over-", "over",
-		"+", "P",
-		"-", "",
-		"/", "",
-		".", "",
-		" ", "",
-	)
-	for i, pr := range pn.Records {
-		if strings.Contains(pr.Name, "Deprecated") ||
-			strings.Contains(pr.Name, "deprecated") {
-			continue
-		}
-		prs[i].OrigName = pr.Name
-		s := strings.TrimSpace(pr.Name)
-		switch pr.Name {
-		case "ISIS over IPv4":
-			prs[i].Name = "ISIS"
-		case "manet":
-			prs[i].Name = "MANET"
-		default:
-			prs[i].Name = sr.Replace(s)
-		}
-		ss := strings.Split(pr.Descr, "\n")
-		for i := range ss {
-			ss[i] = strings.TrimSpace(ss[i])
-		}
-		if len(ss) > 1 {
-			prs[i].Descr = strings.Join(ss, " ")
-		} else {
-			prs[i].Descr = ss[0]
-		}
-		prs[i].Value, _ = strconv.Atoi(pr.Value)
-	}
-	return prs
-}
-
-func parseAddrFamilyNumbers(w io.Writer, r io.Reader) error {
-	dec := xml.NewDecoder(r)
-	var afn addrFamilylNumbers
-	if err := dec.Decode(&afn); err != nil {
-		return err
-	}
-	afrs := afn.escape()
-	fmt.Fprintf(w, "// %s, Updated: %s\n", afn.Title, afn.Updated)
-	fmt.Fprintf(w, "const (\n")
-	for _, afr := range afrs {
-		if afr.Name == "" {
-			continue
-		}
-		fmt.Fprintf(w, "AddrFamily%s = %d", afr.Name, afr.Value)
-		fmt.Fprintf(w, "// %s\n", afr.Descr)
-	}
-	fmt.Fprintf(w, ")\n")
-	return nil
-}
-
-type addrFamilylNumbers struct {
-	XMLName  xml.Name `xml:"registry"`
-	Title    string   `xml:"title"`
-	Updated  string   `xml:"updated"`
-	RegTitle string   `xml:"registry>title"`
-	Note     string   `xml:"registry>note"`
-	Records  []struct {
-		Value string `xml:"value"`
-		Descr string `xml:"description"`
-	} `xml:"registry>record"`
-}
-
-type canonAddrFamilyRecord struct {
-	Name  string
-	Descr string
-	Value int
-}
-
-func (afn *addrFamilylNumbers) escape() []canonAddrFamilyRecord {
-	afrs := make([]canonAddrFamilyRecord, len(afn.Records))
-	sr := strings.NewReplacer(
-		"IP version 4", "IPv4",
-		"IP version 6", "IPv6",
-		"Identifier", "ID",
-		"-", "",
-		"-", "",
-		"/", "",
-		".", "",
-		" ", "",
-	)
-	for i, afr := range afn.Records {
-		if strings.Contains(afr.Descr, "Unassigned") ||
-			strings.Contains(afr.Descr, "Reserved") {
-			continue
-		}
-		afrs[i].Descr = afr.Descr
-		s := strings.TrimSpace(afr.Descr)
-		switch s {
-		case "IP (IP version 4)":
-			afrs[i].Name = "IPv4"
-		case "IP6 (IP version 6)":
-			afrs[i].Name = "IPv6"
-		case "AFI for L2VPN information":
-			afrs[i].Name = "L2VPN"
-		case "E.164 with NSAP format subaddress":
-			afrs[i].Name = "E164withSubaddress"
-		case "MT IP: Multi-Topology IP version 4":
-			afrs[i].Name = "MTIPv4"
-		case "MAC/24":
-			afrs[i].Name = "MACFinal24bits"
-		case "MAC/40":
-			afrs[i].Name = "MACFinal40bits"
-		case "IPv6/64":
-			afrs[i].Name = "IPv6Initial64bits"
-		default:
-			n := strings.Index(s, "(")
-			if n > 0 {
-				s = s[:n]
-			}
-			n = strings.Index(s, ":")
-			if n > 0 {
-				s = s[:n]
-			}
-			afrs[i].Name = sr.Replace(s)
-		}
-		afrs[i].Value, _ = strconv.Atoi(afr.Value)
-	}
-	return afrs
-}

vendor/golang.org/x/net/internal/socket/cmsghdr.go

@@ -1,11 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin dragonfly freebsd linux netbsd openbsd solaris
-
-package socket
-
-func (h *cmsghdr) len() int { return int(h.Len) }
-func (h *cmsghdr) lvl() int { return int(h.Level) }
-func (h *cmsghdr) typ() int { return int(h.Type) }

vendor/golang.org/x/net/internal/socket/cmsghdr_bsd.go

@@ -1,13 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin dragonfly freebsd netbsd openbsd
-
-package socket
-
-func (h *cmsghdr) set(l, lvl, typ int) {
-	h.Len = uint32(l)
-	h.Level = int32(lvl)
-	h.Type = int32(typ)
-}

vendor/golang.org/x/net/internal/socket/cmsghdr_linux_32bit.go

@@ -1,14 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build arm mips mipsle 386
-// +build linux
-
-package socket
-
-func (h *cmsghdr) set(l, lvl, typ int) {
-	h.Len = uint32(l)
-	h.Level = int32(lvl)
-	h.Type = int32(typ)
-}

vendor/golang.org/x/net/internal/socket/cmsghdr_linux_64bit.go

@@ -1,14 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build arm64 amd64 ppc64 ppc64le mips64 mips64le s390x
-// +build linux
-
-package socket
-
-func (h *cmsghdr) set(l, lvl, typ int) {
-	h.Len = uint64(l)
-	h.Level = int32(lvl)
-	h.Type = int32(typ)
-}

vendor/golang.org/x/net/internal/socket/cmsghdr_solaris_64bit.go

@@ -1,14 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build amd64
-// +build solaris
-
-package socket
-
-func (h *cmsghdr) set(l, lvl, typ int) {
-	h.Len = uint32(l)
-	h.Level = int32(lvl)
-	h.Type = int32(typ)
-}

vendor/golang.org/x/net/internal/socket/cmsghdr_stub.go

@@ -1,17 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris
-
-package socket
-
-type cmsghdr struct{}
-
-const sizeofCmsghdr = 0
-
-func (h *cmsghdr) len() int { return 0 }
-func (h *cmsghdr) lvl() int { return 0 }
-func (h *cmsghdr) typ() int { return 0 }
-
-func (h *cmsghdr) set(l, lvl, typ int) {}

vendor/golang.org/x/net/internal/socket/defs_darwin.go

@@ -1,44 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// +godefs map struct_in_addr [4]byte /* in_addr */
-// +godefs map struct_in6_addr [16]byte /* in6_addr */
-
-package socket
-
-/*
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-*/
-import "C"
-
-const (
-	sysAF_UNSPEC = C.AF_UNSPEC
-	sysAF_INET   = C.AF_INET
-	sysAF_INET6  = C.AF_INET6
-
-	sysSOCK_RAW = C.SOCK_RAW
-)
-
-type iovec C.struct_iovec
-
-type msghdr C.struct_msghdr
-
-type cmsghdr C.struct_cmsghdr
-
-type sockaddrInet C.struct_sockaddr_in
-
-type sockaddrInet6 C.struct_sockaddr_in6
-
-const (
-	sizeofIovec   = C.sizeof_struct_iovec
-	sizeofMsghdr  = C.sizeof_struct_msghdr
-	sizeofCmsghdr = C.sizeof_struct_cmsghdr
-
-	sizeofSockaddrInet  = C.sizeof_struct_sockaddr_in
-	sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6
-)

vendor/golang.org/x/net/internal/socket/defs_dragonfly.go

@@ -1,44 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// +godefs map struct_in_addr [4]byte /* in_addr */
-// +godefs map struct_in6_addr [16]byte /* in6_addr */
-
-package socket
-
-/*
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-*/
-import "C"
-
-const (
-	sysAF_UNSPEC = C.AF_UNSPEC
-	sysAF_INET   = C.AF_INET
-	sysAF_INET6  = C.AF_INET6
-
-	sysSOCK_RAW = C.SOCK_RAW
-)
-
-type iovec C.struct_iovec
-
-type msghdr C.struct_msghdr
-
-type cmsghdr C.struct_cmsghdr
-
-type sockaddrInet C.struct_sockaddr_in
-
-type sockaddrInet6 C.struct_sockaddr_in6
-
-const (
-	sizeofIovec   = C.sizeof_struct_iovec
-	sizeofMsghdr  = C.sizeof_struct_msghdr
-	sizeofCmsghdr = C.sizeof_struct_cmsghdr
-
-	sizeofSockaddrInet  = C.sizeof_struct_sockaddr_in
-	sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6
-)

vendor/golang.org/x/net/internal/socket/defs_freebsd.go

@@ -1,44 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// +godefs map struct_in_addr [4]byte /* in_addr */
-// +godefs map struct_in6_addr [16]byte /* in6_addr */
-
-package socket
-
-/*
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-*/
-import "C"
-
-const (
-	sysAF_UNSPEC = C.AF_UNSPEC
-	sysAF_INET   = C.AF_INET
-	sysAF_INET6  = C.AF_INET6
-
-	sysSOCK_RAW = C.SOCK_RAW
-)
-
-type iovec C.struct_iovec
-
-type msghdr C.struct_msghdr
-
-type cmsghdr C.struct_cmsghdr
-
-type sockaddrInet C.struct_sockaddr_in
-
-type sockaddrInet6 C.struct_sockaddr_in6
-
-const (
-	sizeofIovec   = C.sizeof_struct_iovec
-	sizeofMsghdr  = C.sizeof_struct_msghdr
-	sizeofCmsghdr = C.sizeof_struct_cmsghdr
-
-	sizeofSockaddrInet  = C.sizeof_struct_sockaddr_in
-	sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6
-)

vendor/golang.org/x/net/internal/socket/defs_linux.go

@@ -1,49 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// +godefs map struct_in_addr [4]byte /* in_addr */
-// +godefs map struct_in6_addr [16]byte /* in6_addr */
-
-package socket
-
-/*
-#include <linux/in.h>
-#include <linux/in6.h>
-
-#define _GNU_SOURCE
-#include <sys/socket.h>
-*/
-import "C"
-
-const (
-	sysAF_UNSPEC = C.AF_UNSPEC
-	sysAF_INET   = C.AF_INET
-	sysAF_INET6  = C.AF_INET6
-
-	sysSOCK_RAW = C.SOCK_RAW
-)
-
-type iovec C.struct_iovec
-
-type msghdr C.struct_msghdr
-
-type mmsghdr C.struct_mmsghdr
-
-type cmsghdr C.struct_cmsghdr
-
-type sockaddrInet C.struct_sockaddr_in
-
-type sockaddrInet6 C.struct_sockaddr_in6
-
-const (
-	sizeofIovec   = C.sizeof_struct_iovec
-	sizeofMsghdr  = C.sizeof_struct_msghdr
-	sizeofMmsghdr = C.sizeof_struct_mmsghdr
-	sizeofCmsghdr = C.sizeof_struct_cmsghdr
-
-	sizeofSockaddrInet  = C.sizeof_struct_sockaddr_in
-	sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6
-)

vendor/golang.org/x/net/internal/socket/defs_netbsd.go

@@ -1,47 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// +godefs map struct_in_addr [4]byte /* in_addr */
-// +godefs map struct_in6_addr [16]byte /* in6_addr */
-
-package socket
-
-/*
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-*/
-import "C"
-
-const (
-	sysAF_UNSPEC = C.AF_UNSPEC
-	sysAF_INET   = C.AF_INET
-	sysAF_INET6  = C.AF_INET6
-
-	sysSOCK_RAW = C.SOCK_RAW
-)
-
-type iovec C.struct_iovec
-
-type msghdr C.struct_msghdr
-
-type mmsghdr C.struct_mmsghdr
-
-type cmsghdr C.struct_cmsghdr
-
-type sockaddrInet C.struct_sockaddr_in
-
-type sockaddrInet6 C.struct_sockaddr_in6
-
-const (
-	sizeofIovec   = C.sizeof_struct_iovec
-	sizeofMsghdr  = C.sizeof_struct_msghdr
-	sizeofMmsghdr = C.sizeof_struct_mmsghdr
-	sizeofCmsghdr = C.sizeof_struct_cmsghdr
-
-	sizeofSockaddrInet  = C.sizeof_struct_sockaddr_in
-	sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6
-)

vendor/golang.org/x/net/internal/socket/defs_openbsd.go

@@ -1,44 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// +godefs map struct_in_addr [4]byte /* in_addr */
-// +godefs map struct_in6_addr [16]byte /* in6_addr */
-
-package socket
-
-/*
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-*/
-import "C"
-
-const (
-	sysAF_UNSPEC = C.AF_UNSPEC
-	sysAF_INET   = C.AF_INET
-	sysAF_INET6  = C.AF_INET6
-
-	sysSOCK_RAW = C.SOCK_RAW
-)
-
-type iovec C.struct_iovec
-
-type msghdr C.struct_msghdr
-
-type cmsghdr C.struct_cmsghdr
-
-type sockaddrInet C.struct_sockaddr_in
-
-type sockaddrInet6 C.struct_sockaddr_in6
-
-const (
-	sizeofIovec   = C.sizeof_struct_iovec
-	sizeofMsghdr  = C.sizeof_struct_msghdr
-	sizeofCmsghdr = C.sizeof_struct_cmsghdr
-
-	sizeofSockaddrInet  = C.sizeof_struct_sockaddr_in
-	sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6
-)

vendor/golang.org/x/net/internal/socket/defs_solaris.go

@@ -1,44 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// +godefs map struct_in_addr [4]byte /* in_addr */
-// +godefs map struct_in6_addr [16]byte /* in6_addr */
-
-package socket
-
-/*
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-*/
-import "C"
-
-const (
-	sysAF_UNSPEC = C.AF_UNSPEC
-	sysAF_INET   = C.AF_INET
-	sysAF_INET6  = C.AF_INET6
-
-	sysSOCK_RAW = C.SOCK_RAW
-)
-
-type iovec C.struct_iovec
-
-type msghdr C.struct_msghdr
-
-type cmsghdr C.struct_cmsghdr
-
-type sockaddrInet C.struct_sockaddr_in
-
-type sockaddrInet6 C.struct_sockaddr_in6
-
-const (
-	sizeofIovec   = C.sizeof_struct_iovec
-	sizeofMsghdr  = C.sizeof_struct_msghdr
-	sizeofCmsghdr = C.sizeof_struct_cmsghdr
-
-	sizeofSockaddrInet  = C.sizeof_struct_sockaddr_in
-	sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6
-)

vendor/golang.org/x/net/internal/socket/empty.s

@@ -1,7 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin,go1.12
-
-// This exists solely so we can linkname in symbols from syscall.

vendor/golang.org/x/net/internal/socket/error_unix.go

@@ -1,31 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin dragonfly freebsd linux netbsd openbsd solaris
-
-package socket
-
-import "syscall"
-
-var (
-	errEAGAIN error = syscall.EAGAIN
-	errEINVAL error = syscall.EINVAL
-	errENOENT error = syscall.ENOENT
-)
-
-// errnoErr returns common boxed Errno values, to prevent allocations
-// at runtime.
-func errnoErr(errno syscall.Errno) error {
-	switch errno {
-	case 0:
-		return nil
-	case syscall.EAGAIN:
-		return errEAGAIN
-	case syscall.EINVAL:
-		return errEINVAL
-	case syscall.ENOENT:
-		return errENOENT
-	}
-	return errno
-}

vendor/golang.org/x/net/internal/socket/error_windows.go

@@ -1,26 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package socket
-
-import "syscall"
-
-var (
-	errERROR_IO_PENDING error = syscall.ERROR_IO_PENDING
-	errEINVAL           error = syscall.EINVAL
-)
-
-// errnoErr returns common boxed Errno values, to prevent allocations
-// at runtime.
-func errnoErr(errno syscall.Errno) error {
-	switch errno {
-	case 0:
-		return nil
-	case syscall.ERROR_IO_PENDING:
-		return errERROR_IO_PENDING
-	case syscall.EINVAL:
-		return errEINVAL
-	}
-	return errno
-}

vendor/golang.org/x/net/internal/socket/iovec_32bit.go

@@ -1,19 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build arm mips mipsle 386
-// +build darwin dragonfly freebsd linux netbsd openbsd
-
-package socket
-
-import "unsafe"
-
-func (v *iovec) set(b []byte) {
-	l := len(b)
-	if l == 0 {
-		return
-	}
-	v.Base = (*byte)(unsafe.Pointer(&b[0]))
-	v.Len = uint32(l)
-}

vendor/golang.org/x/net/internal/socket/iovec_64bit.go

@@ -1,19 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build arm64 amd64 ppc64 ppc64le mips64 mips64le s390x
-// +build darwin dragonfly freebsd linux netbsd openbsd
-
-package socket
-
-import "unsafe"
-
-func (v *iovec) set(b []byte) {
-	l := len(b)
-	if l == 0 {
-		return
-	}
-	v.Base = (*byte)(unsafe.Pointer(&b[0]))
-	v.Len = uint64(l)
-}

vendor/golang.org/x/net/internal/socket/iovec_solaris_64bit.go

@@ -1,19 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build amd64
-// +build solaris
-
-package socket
-
-import "unsafe"
-
-func (v *iovec) set(b []byte) {
-	l := len(b)
-	if l == 0 {
-		return
-	}
-	v.Base = (*int8)(unsafe.Pointer(&b[0]))
-	v.Len = uint64(l)
-}

vendor/golang.org/x/net/internal/socket/iovec_stub.go

@@ -1,11 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris
-
-package socket
-
-type iovec struct{}
-
-func (v *iovec) set(b []byte) {}

vendor/golang.org/x/net/internal/socket/mmsghdr_stub.go

@@ -1,21 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !linux,!netbsd
-
-package socket
-
-import "net"
-
-type mmsghdr struct{}
-
-type mmsghdrs []mmsghdr
-
-func (hs mmsghdrs) pack(ms []Message, parseFn func([]byte, string) (net.Addr, error), marshalFn func(net.Addr) []byte) error {
-	return nil
-}
-
-func (hs mmsghdrs) unpack(ms []Message, parseFn func([]byte, string) (net.Addr, error), hint string) error {
-	return nil
-}

vendor/golang.org/x/net/internal/socket/mmsghdr_unix.go

@@ -1,42 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux netbsd
-
-package socket
-
-import "net"
-
-type mmsghdrs []mmsghdr
-
-func (hs mmsghdrs) pack(ms []Message, parseFn func([]byte, string) (net.Addr, error), marshalFn func(net.Addr) []byte) error {
-	for i := range hs {
-		vs := make([]iovec, len(ms[i].Buffers))
-		var sa []byte
-		if parseFn != nil {
-			sa = make([]byte, sizeofSockaddrInet6)
-		}
-		if marshalFn != nil {
-			sa = marshalFn(ms[i].Addr)
-		}
-		hs[i].Hdr.pack(vs, ms[i].Buffers, ms[i].OOB, sa)
-	}
-	return nil
-}
-
-func (hs mmsghdrs) unpack(ms []Message, parseFn func([]byte, string) (net.Addr, error), hint string) error {
-	for i := range hs {
-		ms[i].N = int(hs[i].Len)
-		ms[i].NN = hs[i].Hdr.controllen()
-		ms[i].Flags = hs[i].Hdr.flags()
-		if parseFn != nil {
-			var err error
-			ms[i].Addr, err = parseFn(hs[i].Hdr.name(), hint)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}

vendor/golang.org/x/net/internal/socket/msghdr_bsd.go

@@ -1,39 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin dragonfly freebsd netbsd openbsd
-
-package socket
-
-import "unsafe"
-
-func (h *msghdr) pack(vs []iovec, bs [][]byte, oob []byte, sa []byte) {
-	for i := range vs {
-		vs[i].set(bs[i])
-	}
-	h.setIov(vs)
-	if len(oob) > 0 {
-		h.Control = (*byte)(unsafe.Pointer(&oob[0]))
-		h.Controllen = uint32(len(oob))
-	}
-	if sa != nil {
-		h.Name = (*byte)(unsafe.Pointer(&sa[0]))
-		h.Namelen = uint32(len(sa))
-	}
-}
-
-func (h *msghdr) name() []byte {
-	if h.Name != nil && h.Namelen > 0 {
-		return (*[sizeofSockaddrInet6]byte)(unsafe.Pointer(h.Name))[:h.Namelen]
-	}
-	return nil
-}
-
-func (h *msghdr) controllen() int {
-	return int(h.Controllen)
-}
-
-func (h *msghdr) flags() int {
-	return int(h.Flags)
-}

vendor/golang.org/x/net/internal/socket/msghdr_bsdvar.go

@@ -1,16 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin dragonfly freebsd netbsd
-
-package socket
-
-func (h *msghdr) setIov(vs []iovec) {
-	l := len(vs)
-	if l == 0 {
-		return
-	}
-	h.Iov = &vs[0]
-	h.Iovlen = int32(l)
-}

vendor/golang.org/x/net/internal/socket/msghdr_linux.go

@@ -1,36 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package socket
-
-import "unsafe"
-
-func (h *msghdr) pack(vs []iovec, bs [][]byte, oob []byte, sa []byte) {
-	for i := range vs {
-		vs[i].set(bs[i])
-	}
-	h.setIov(vs)
-	if len(oob) > 0 {
-		h.setControl(oob)
-	}
-	if sa != nil {
-		h.Name = (*byte)(unsafe.Pointer(&sa[0]))
-		h.Namelen = uint32(len(sa))
-	}
-}
-
-func (h *msghdr) name() []byte {
-	if h.Name != nil && h.Namelen > 0 {
-		return (*[sizeofSockaddrInet6]byte)(unsafe.Pointer(h.Name))[:h.Namelen]
-	}
-	return nil
-}
-
-func (h *msghdr) controllen() int {
-	return int(h.Controllen)
-}
-
-func (h *msghdr) flags() int {
-	return int(h.Flags)
-}

vendor/golang.org/x/net/internal/socket/msghdr_linux_32bit.go

@@ -1,24 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build arm mips mipsle 386
-// +build linux
-
-package socket
-
-import "unsafe"
-
-func (h *msghdr) setIov(vs []iovec) {
-	l := len(vs)
-	if l == 0 {
-		return
-	}
-	h.Iov = &vs[0]
-	h.Iovlen = uint32(l)
-}
-
-func (h *msghdr) setControl(b []byte) {
-	h.Control = (*byte)(unsafe.Pointer(&b[0]))
-	h.Controllen = uint32(len(b))
-}

vendor/golang.org/x/net/internal/socket/msghdr_linux_64bit.go

@@ -1,24 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build arm64 amd64 ppc64 ppc64le mips64 mips64le s390x
-// +build linux
-
-package socket
-
-import "unsafe"
-
-func (h *msghdr) setIov(vs []iovec) {
-	l := len(vs)
-	if l == 0 {
-		return
-	}
-	h.Iov = &vs[0]
-	h.Iovlen = uint64(l)
-}
-
-func (h *msghdr) setControl(b []byte) {
-	h.Control = (*byte)(unsafe.Pointer(&b[0]))
-	h.Controllen = uint64(len(b))
-}