From fee471634386b72dd2df229f3d52442c081f12d0 Mon Sep 17 00:00:00 2001 From: nielash Date: Sun, 24 Aug 2025 03:10:15 -0400 Subject: [PATCH] bin: add bisync.md generator This change adds make_bisync_docs.go step to dynamically update the list of ignored and failed tests in bisync.md --- Makefile | 1 + bin/make_bisync_docs.go | 159 ++++++++++++++++++++++++++++++++++++++++ docs/content/bisync.md | 8 +- 3 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 bin/make_bisync_docs.go diff --git a/Makefile b/Makefile index 05f7bc28d..03242ef8d 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,7 @@ commanddocs: rclone -@rmdir -p '$$HOME/.config/rclone' XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/ @[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1) + go run bin/make_bisync_docs.go ./docs/content/ backenddocs: rclone bin/make_backend_docs.py -@rmdir -p '$$HOME/.config/rclone' diff --git a/bin/make_bisync_docs.go b/bin/make_bisync_docs.go new file mode 100644 index 000000000..3f0dc125c --- /dev/null +++ b/bin/make_bisync_docs.go @@ -0,0 +1,159 @@ +//go:build ignore + +package main + +import ( + "bytes" + "cmp" + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fstest/runs" + "github.com/stretchr/testify/assert/yaml" +) + +var path = flag.String("path", "./docs/content/", "root path") + +const ( + configFile = "fstest/test_all/config.yaml" + startListIgnores = "" + endListIgnores = "" + startListFailures = "" + endListFailures = "" + integrationTestsJSONURL = "https://pub.rclone.org/integration-tests/current/index.json" + integrationTestsHTMLURL = "https://pub.rclone.org/integration-tests/current/" +) + +func main() { + err := replaceBetween(*path, startListIgnores, endListIgnores, getIgnores) + if err != nil { + fs.Errorf(*path, "error replacing ignores: %v", err) + } + err = replaceBetween(*path, startListFailures, endListFailures, getFailures) + if err != nil { + fs.Errorf(*path, "error replacing failures: %v", err) + } +} + +// replaceBetween replaces the text between startSep and endSep with fn() +func replaceBetween(path, startSep, endSep string, fn func() (string, error)) error { + b, err := os.ReadFile(filepath.Join(path, "bisync.md")) + if err != nil { + return err + } + doc := string(b) + + before, after, found := strings.Cut(doc, startSep) + if !found { + return fmt.Errorf("could not find: %v", startSep) + } + _, after, found = strings.Cut(after, endSep) + if !found { + return fmt.Errorf("could not find: %v", endSep) + } + + replaceSection, err := fn() + if err != nil { + return err + } + + newDoc := before + startSep + "\n" + strings.TrimSpace(replaceSection) + "\n" + endSep + after + + err = os.WriteFile(filepath.Join(path, "bisync.md"), []byte(newDoc), 0777) + if err != nil { + return err + } + return nil +} + +// getIgnores updates the list of ignores from config.yaml +func getIgnores() (string, error) { + config, err := parseConfig() + if err != nil { + return "", fmt.Errorf("failed to parse config: %v", err) + } + s := "" + slices.SortFunc(config.Backends, func(a, b runs.Backend) int { + return cmp.Compare(a.Remote, b.Remote) + }) + for _, backend := range config.Backends { + include := false + + if slices.Contains(backend.IgnoreTests, "cmd/bisync") { + include = true + s += fmt.Sprintf("- `%s` (`%s`)\n", strings.TrimSuffix(backend.Remote, ":"), backend.Backend) + } + + for _, ignore := range backend.Ignore { + if strings.Contains(strings.ToLower(ignore), "bisync") { + if !include { // don't have header row yet + s += fmt.Sprintf("- `%s` (`%s`)\n", strings.TrimSuffix(backend.Remote, ":"), backend.Backend) + } + include = true + s += fmt.Sprintf(" - `%s`\n", ignore) + // TODO: might be neat to add a "reason" param displaying the reason the test is ignored + } + } + } + return s, nil +} + +// getFailures updates the list of currently failing tests from the integration tests server +func getFailures() (string, error) { + var buf bytes.Buffer + err := operations.CopyURLToWriter(context.Background(), integrationTestsJSONURL, &buf) + if err != nil { + return "", err + } + + r := runs.Report{} + err = json.Unmarshal(buf.Bytes(), &r) + if err != nil { + return "", fmt.Errorf("failed to unmarshal json: %v", err) + } + + s := "" + for _, run := range r.Failed { + for i, t := range run.FailedTests { + if strings.Contains(strings.ToLower(t), "bisync") { + + if i == 0 { // don't have header row yet + s += fmt.Sprintf("- `%s` (`%s`)\n", strings.TrimSuffix(run.Remote, ":"), run.Backend) + } + + url := integrationTestsHTMLURL + run.TrialName + url = url[:len(url)-5] + "1.txt" // numbers higher than 1 could change from night to night + s += fmt.Sprintf(" - [`%s`](%v)\n", t, url) + + if i == 4 && len(run.FailedTests) > 5 { // stop after 5 + s += fmt.Sprintf(" - [%v more](%v)\n", len(run.FailedTests)-5, integrationTestsHTMLURL) + break + } + } + } + } + s += fmt.Sprintf("- Updated: %v", r.DateTime) + return s, nil +} + +// parseConfig reads and parses the config.yaml file +func parseConfig() (*runs.Config, error) { + d, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + config := &runs.Config{} + err = yaml.Unmarshal(d, &config) + if err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + return config, nil +} diff --git a/docs/content/bisync.md b/docs/content/bisync.md index 2473b58ea..a72bc8e24 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -1046,7 +1046,7 @@ encodings.) The following backends have known issues that need more investigation: - + - `TestGoFile` (`gofile`) - [`TestBisyncRemoteLocal/all_changed`](https://pub.rclone.org/integration-tests/current/gofile-cmd.bisync-TestGoFile-1.txt) - [`TestBisyncRemoteLocal/backupdir`](https://pub.rclone.org/integration-tests/current/gofile-cmd.bisync-TestGoFile-1.txt) @@ -1055,12 +1055,12 @@ The following backends have known issues that need more investigation: - [`TestBisyncRemoteLocal/check_access`](https://pub.rclone.org/integration-tests/current/gofile-cmd.bisync-TestGoFile-1.txt) - [78 more](https://pub.rclone.org/integration-tests/current/) - Updated: 2025-08-21-010015 - + The following backends either have not been tested recently or have known issues that are deemed unfixable for the time being: - + - `TestCache` (`cache`) - `TestFileLu` (`filelu`) - `TestFilesCom` (`filescom`) @@ -1085,7 +1085,7 @@ that are deemed unfixable for the time being: - `TestWebdavNextcloud` (`webdav`) - `TestWebdavOwncloud` (`webdav`) - `TestnStorage` (`netstorage`) - + ([more info](https://github.com/rclone/rclone/blob/master/fstest/test_all/config.yaml)) The above lists are updated for each stable release of rclone. For test results