From 321488441e804c1b4b7573bad1d22786079d5b94 Mon Sep 17 00:00:00 2001 From: Nikolay Kiryanov Date: Wed, 19 Nov 2025 03:22:47 +0300 Subject: [PATCH] rc: add `executeId` to job statuses - fixes #8972 --- docs/content/rc.md | 31 ++++++++++++++++++++++++++----- fs/rc/jobs/job.go | 9 +++++++-- fs/rc/jobs/job_test.go | 23 +++++++++++++++++++---- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/docs/content/rc.md b/docs/content/rc.md index b057b2c65..8a4953ebf 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -257,9 +257,9 @@ Each rc call is classified as a job and it is assigned its own id. By default jobs are executed immediately as they are created or synchronously. If `_async` has a true value when supplied to an rc call then it will -return immediately with a job id and the task will be run in the -background. The `job/status` call can be used to get information of -the background job. The job can be queried for up to 1 minute after +return immediately with a job id and execute id, and the task will be run in the +background. The `job/status` call can be used to get information of +the background job. The job can be queried for up to 1 minute after it has finished. It is recommended that potentially long running jobs, e.g. `sync/sync`, @@ -272,10 +272,16 @@ Starting a job with the `_async` flag: ```console $ rclone rc --json '{ "p1": [1,"2",null,4], "p2": { "a":1, "b":2 }, "_async": true }' rc/noop { - "jobid": 2 + "jobid": 2, + "executeId": "d794c33c-463e-4acf-b911-f4b23e4f40b7" } ``` +The `jobid` is a unique identifier for the job within this rclone instance. +The `executeId` identifies the rclone process instance and changes after +rclone restart. Together, the pair (`executeId`, `jobid`) uniquely identifies +a job across rclone restarts. + Query the status to see if the job has finished. For more information on the meaning of these return parameters see the `job/status` call. @@ -285,6 +291,7 @@ $ rclone rc --json '{ "jobid":2 }' job/status "duration": 0.000124163, "endTime": "2018-10-27T11:38:07.911245881+01:00", "error": "", + "executeId": "d794c33c-463e-4acf-b911-f4b23e4f40b7", "finished": true, "id": 2, "output": { @@ -305,17 +312,31 @@ $ rclone rc --json '{ "jobid":2 }' job/status } ``` -`job/list` can be used to show the running or recently completed jobs +`job/list` can be used to show running or recently completed jobs along with their status ```console $ rclone rc job/list { + "executeId": "d794c33c-463e-4acf-b911-f4b23e4f40b7", + "finished_ids": [ + 1 + ], "jobids": [ + 1, + 2 + ], + "running_ids": [ 2 ] } ``` +This shows: +- `executeId` - the current rclone instance ID (same for all jobs, changes after restart) +- `jobids` - array of all job IDs (both running and finished) +- `running_ids` - array of currently running job IDs +- `finished_ids` - array of finished job IDs + ### Setting config flags with _config If you wish to set config (the equivalent of the global flags) for the diff --git a/fs/rc/jobs/job.go b/fs/rc/jobs/job.go index 1a7976c27..2dfcf4000 100644 --- a/fs/rc/jobs/job.go +++ b/fs/rc/jobs/job.go @@ -34,6 +34,7 @@ func init() { type Job struct { mu sync.Mutex ID int64 `json:"id"` + ExecuteID string `json:"executeId"` Group string `json:"group"` StartTime time.Time `json:"startTime"` EndTime time.Time `json:"endTime"` @@ -123,8 +124,9 @@ type Jobs struct { } var ( - running = newJobs() - jobID atomic.Int64 + running = newJobs() + jobID atomic.Int64 + // executeID is a unique ID for this rclone execution executeID = uuid.New().String() ) @@ -313,6 +315,7 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo } job = &Job{ ID: id, + ExecuteID: executeID, Group: group, StartTime: time.Now(), Stop: stop, @@ -329,6 +332,7 @@ func (jobs *Jobs) NewJob(ctx context.Context, fn rc.Func, in rc.Params) (job *Jo go job.run(ctx, fn, in) out = make(rc.Params) out["jobid"] = job.ID + out["executeId"] = job.ExecuteID err = nil } else { job.run(ctx, fn, in) @@ -386,6 +390,7 @@ Results: - error - error from the job or empty string for no error - finished - boolean whether the job has finished or not - id - as passed in above +- executeId - rclone instance ID (changes after restart); combined with id uniquely identifies a job - startTime - time the job started (e.g. "2018-10-26T18:50:20.528336039+01:00") - success - boolean - true for success false otherwise - output - output of the job as would have been returned if called synchronously diff --git a/fs/rc/jobs/job_test.go b/fs/rc/jobs/job_test.go index 616ad0ef2..7a3d45e09 100644 --- a/fs/rc/jobs/job_test.go +++ b/fs/rc/jobs/job_test.go @@ -56,7 +56,7 @@ func TestJobsExpire(t *testing.T) { return in, nil }, rc.Params{"_async": true}) require.NoError(t, err) - assert.Equal(t, 1, len(out)) + assert.Equal(t, 2, len(out), "check output has jobid and executeId") <-wait assert.Equal(t, job.ID, gotJobID, "check can get JobID from ctx") assert.Equal(t, job, gotJob, "check can get Job from ctx") @@ -96,6 +96,18 @@ func TestJobsIDs(t *testing.T) { assert.Equal(t, wantIDs, gotIDs) } +func TestJobsExecuteIDs(t *testing.T) { + ctx := context.Background() + jobs := newJobs() + job1, _, err := jobs.NewJob(ctx, noopFn, rc.Params{"_async": true}) + require.NoError(t, err) + job2, _, err := jobs.NewJob(ctx, noopFn, rc.Params{"_async": true}) + require.NoError(t, err) + assert.Equal(t, executeID, job1.ExecuteID, "execute ID should match global executeID") + assert.Equal(t, executeID, job2.ExecuteID, "execute ID should match global executeID") + assert.True(t, job1.ExecuteID == job2.ExecuteID, "just to be sure, all the jobs share the same executeID") +} + func TestJobsGet(t *testing.T) { ctx := context.Background() jobs := newJobs() @@ -234,7 +246,8 @@ func TestJobsNewJob(t *testing.T) { job, out, err := jobs.NewJob(ctx, noopFn, rc.Params{"_async": true}) require.NoError(t, err) assert.Equal(t, int64(1), job.ID) - assert.Equal(t, rc.Params{"jobid": int64(1)}, out) + assert.Equal(t, executeID, job.ExecuteID) + assert.Equal(t, rc.Params{"jobid": int64(1), "executeId": executeID}, out) assert.Equal(t, job, jobs.Get(1)) assert.NotEmpty(t, job.Stop) } @@ -244,8 +257,9 @@ func TestStartJob(t *testing.T) { jobID.Store(0) job, out, err := NewJob(ctx, longFn, rc.Params{"_async": true}) assert.NoError(t, err) - assert.Equal(t, rc.Params{"jobid": int64(1)}, out) + assert.Equal(t, rc.Params{"jobid": int64(1), "executeId": executeID}, out) assert.Equal(t, int64(1), job.ID) + assert.Equal(t, executeID, job.ExecuteID) } func TestExecuteJob(t *testing.T) { @@ -350,6 +364,7 @@ func TestRcJobStatus(t *testing.T) { require.NoError(t, err) require.NotNil(t, out) assert.Equal(t, float64(1), out["id"]) + assert.Equal(t, executeID, out["executeId"]) assert.Equal(t, "", out["error"]) assert.Equal(t, false, out["finished"]) assert.Equal(t, false, out["success"]) @@ -377,6 +392,7 @@ func TestRcJobList(t *testing.T) { out1, err := call.Fn(context.Background(), in) require.NoError(t, err) require.NotNil(t, out1) + assert.Equal(t, executeID, out1["executeId"], "should have executeId") assert.Equal(t, []int64{1}, out1["jobids"], "should have job listed") assert.Equal(t, []int64{1}, out1["running_ids"], "should have running job") assert.Equal(t, []int64{}, out1["finished_ids"], "should not have finished job") @@ -392,7 +408,6 @@ func TestRcJobList(t *testing.T) { require.NotNil(t, out2) assert.Equal(t, 2, len(out2["jobids"].([]int64)), "should have all jobs listed") - require.NotNil(t, out1["executeId"], "should have executeId") assert.Equal(t, out1["executeId"], out2["executeId"], "executeId should be the same") }