rc: add executeId to job statuses - fixes #8972

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

docs/content/rc.md

@@ -257,9 +257,9 @@ Each rc call is classified as a job and it is assigned its own id. By default
 jobs are executed immediately as they are created or synchronously.
 
 If `_async` has a true value when supplied to an rc call then it will
-return immediately with a job id and the task will be run in the
-background.  The `job/status` call can be used to get information of
-the background job.  The job can be queried for up to 1 minute after
+return immediately with a job id and execute id, and the task will be run in the
+background. The `job/status` call can be used to get information of
+the background job. The job can be queried for up to 1 minute after
 it has finished.
 
 It is recommended that potentially long running jobs, e.g. `sync/sync`,
@@ -272,10 +272,16 @@ Starting a job with the `_async` flag:
 ```console
 $ rclone rc --json '{ "p1": [1,"2",null,4], "p2": { "a":1, "b":2 }, "_async": true }' rc/noop
 {
-    "jobid": 2
+    "jobid": 2,
+    "executeId": "d794c33c-463e-4acf-b911-f4b23e4f40b7"
 }
 ```
 
+The `jobid` is a unique identifier for the job within this rclone instance.
+The `executeId` identifies the rclone process instance and changes after
+rclone restart. Together, the pair (`executeId`, `jobid`) uniquely identifies
+a job across rclone restarts.
+
 Query the status to see if the job has finished.  For more information
 on the meaning of these return parameters see the `job/status` call.
 
@@ -285,6 +291,7 @@ $ rclone rc --json '{ "jobid":2 }' job/status
     "duration": 0.000124163,
     "endTime": "2018-10-27T11:38:07.911245881+01:00",
     "error": "",
+    "executeId": "d794c33c-463e-4acf-b911-f4b23e4f40b7",
     "finished": true,
     "id": 2,
     "output": {
@@ -305,17 +312,31 @@ $ rclone rc --json '{ "jobid":2 }' job/status
 }
 ```
 
-`job/list` can be used to show the running or recently completed jobs
+`job/list` can be used to show running or recently completed jobs along with their status
 
 ```console
 $ rclone rc job/list
 {
+    "executeId": "d794c33c-463e-4acf-b911-f4b23e4f40b7",
+    "finished_ids": [
+        1
+    ],
     "jobids": [
+        1,
+        2
+    ],
+    "running_ids": [
         2
     ]
 }
 ```
 
+This shows:
+- `executeId` - the current rclone instance ID (same for all jobs, changes after restart)
+- `jobids` - array of all job IDs (both running and finished)
+- `running_ids` - array of currently running job IDs
+- `finished_ids` - array of finished job IDs
+
 ### Setting config flags with _config
 
 If you wish to set config (the equivalent of the global flags) for the

fs/rc/jobs/job.go

@@ -34,6 +34,7 @@ func init() {
 type Job struct {
 	mu        sync.Mutex
 	ID        int64     `json:"id"`
+	ExecuteID string    `json:"executeId"`
 	Group     string    `json:"group"`
 	StartTime time.Time `json:"startTime"`
 	EndTime   time.Time `json:"endTime"`
@@ -123,8 +124,9 @@ type Jobs struct {
 }
 
 var (
-	running   = newJobs()
-	jobID     atomic.Int64
+	running = newJobs()
+	jobID   atomic.Int64
+	// executeID is a unique ID for this rclone execution
 	executeID = uuid.New().String()
 )
 
@@ -313,6 +315,7 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo
 	}
 	job = &Job{
 		ID:        id,
+		ExecuteID: executeID,
 		Group:     group,
 		StartTime: time.Now(),
 		Stop:      stop,
@@ -329,6 +332,7 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo
 		go job.run(ctx, fn, in)
 		out = make(rc.Params)
 		out["jobid"] = job.ID
+		out["executeId"] = job.ExecuteID
 		err = nil
 	} else {
 		job.run(ctx, fn, in)
@@ -386,6 +390,7 @@ Results:
 - error - error from the job or empty string for no error
 - finished - boolean whether the job has finished or not
 - id - as passed in above
+- executeId - rclone instance ID (changes after restart); combined with id uniquely identifies a job
 - startTime - time the job started (e.g. "2018-10-26T18:50:20.528336039+01:00")
 - success - boolean - true for success false otherwise
 - output - output of the job as would have been returned if called synchronously

fs/rc/jobs/job_test.go

@@ -56,7 +56,7 @@ func TestJobsExpire(t *testing.T) {
 		return in, nil
 	}, rc.Params{"_async": true})
 	require.NoError(t, err)
-	assert.Equal(t, 1, len(out))
+	assert.Equal(t, 2, len(out), "check output has jobid and executeId")
 	<-wait
 	assert.Equal(t, job.ID, gotJobID, "check can get JobID from ctx")
 	assert.Equal(t, job, gotJob, "check can get Job from ctx")
@@ -96,6 +96,18 @@ func TestJobsIDs(t *testing.T) {
 	assert.Equal(t, wantIDs, gotIDs)
 }
 
+func TestJobsExecuteIDs(t *testing.T) {
+	ctx := context.Background()
+	jobs := newJobs()
+	job1, _, err := jobs.NewJob(ctx, noopFn, rc.Params{"_async": true})
+	require.NoError(t, err)
+	job2, _, err := jobs.NewJob(ctx, noopFn, rc.Params{"_async": true})
+	require.NoError(t, err)
+	assert.Equal(t, executeID, job1.ExecuteID, "execute ID should match global executeID")
+	assert.Equal(t, executeID, job2.ExecuteID, "execute ID should match global executeID")
+	assert.True(t, job1.ExecuteID == job2.ExecuteID, "just to be sure, all the jobs share the same executeID")
+}
+
 func TestJobsGet(t *testing.T) {
 	ctx := context.Background()
 	jobs := newJobs()
@@ -234,7 +246,8 @@ func TestJobsNewJob(t *testing.T) {
 	job, out, err := jobs.NewJob(ctx, noopFn, rc.Params{"_async": true})
 	require.NoError(t, err)
 	assert.Equal(t, int64(1), job.ID)
-	assert.Equal(t, rc.Params{"jobid": int64(1)}, out)
+	assert.Equal(t, executeID, job.ExecuteID)
+	assert.Equal(t, rc.Params{"jobid": int64(1), "executeId": executeID}, out)
 	assert.Equal(t, job, jobs.Get(1))
 	assert.NotEmpty(t, job.Stop)
 }
@@ -244,8 +257,9 @@ func TestStartJob(t *testing.T) {
 	jobID.Store(0)
 	job, out, err := NewJob(ctx, longFn, rc.Params{"_async": true})
 	assert.NoError(t, err)
-	assert.Equal(t, rc.Params{"jobid": int64(1)}, out)
+	assert.Equal(t, rc.Params{"jobid": int64(1), "executeId": executeID}, out)
 	assert.Equal(t, int64(1), job.ID)
+	assert.Equal(t, executeID, job.ExecuteID)
 }
 
 func TestExecuteJob(t *testing.T) {
@@ -350,6 +364,7 @@ func TestRcJobStatus(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, out)
 	assert.Equal(t, float64(1), out["id"])
+	assert.Equal(t, executeID, out["executeId"])
 	assert.Equal(t, "", out["error"])
 	assert.Equal(t, false, out["finished"])
 	assert.Equal(t, false, out["success"])
@@ -377,6 +392,7 @@ func TestRcJobList(t *testing.T) {
 	out1, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
 	require.NotNil(t, out1)
+	assert.Equal(t, executeID, out1["executeId"], "should have executeId")
 	assert.Equal(t, []int64{1}, out1["jobids"], "should have job listed")
 	assert.Equal(t, []int64{1}, out1["running_ids"], "should have running job")
 	assert.Equal(t, []int64{}, out1["finished_ids"], "should not have finished job")
@@ -392,7 +408,6 @@ func TestRcJobList(t *testing.T) {
 	require.NotNil(t, out2)
 	assert.Equal(t, 2, len(out2["jobids"].([]int64)), "should have all jobs listed")
 
-	require.NotNil(t, out1["executeId"], "should have executeId")
 	assert.Equal(t, out1["executeId"], out2["executeId"], "executeId should be the same")
 }
 

go.mod

@@ -2,9 +2,7 @@ module github.com/rclone/rclone
 
 go 1.24.4
 
-godebug (
-    x509negativeserial=1
-)
+godebug x509negativeserial=1
 
 require (
 	bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5
@@ -90,12 +88,12 @@ require (
 	github.com/zeebo/xxh3 v1.0.2
 	go.etcd.io/bbolt v1.4.3
 	goftp.io/server/v2 v2.0.2
-	golang.org/x/crypto v0.43.0
-	golang.org/x/net v0.46.0
+	golang.org/x/crypto v0.45.0
+	golang.org/x/net v0.47.0
 	golang.org/x/oauth2 v0.33.0
 	golang.org/x/sync v0.18.0
 	golang.org/x/sys v0.38.0
-	golang.org/x/text v0.30.0
+	golang.org/x/text v0.31.0
 	golang.org/x/time v0.14.0
 	google.golang.org/api v0.255.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
@@ -278,5 +276,5 @@ require (
 	github.com/pkg/xattr v0.4.12
 	github.com/pquerna/otp v1.5.0
 	golang.org/x/mobile v0.0.0-20251021151156-188f512ec823
-	golang.org/x/term v0.36.0
+	golang.org/x/term v0.37.0
 )

go.sum

@@ -754,8 +754,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -836,8 +836,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
-golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -923,8 +923,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
-golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -939,8 +939,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
-golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=