diff --git a/main.go b/main.go index 68722fc..ba9753c 100644 --- a/main.go +++ b/main.go @@ -66,13 +66,20 @@ func main() { func updateSnapshots(cnf *restic.Config, mnu *resticmenu) { err := wrapper.UpdateLatestSnapshots(cnf) if err != nil { - handleError(err) + te, ok := err.(*restic.ResticCmdTimeoutError) + if ok { + log.Warn().Err(te).Msg("update snapshot timeout") + mnu.latestSnapshot.SetTitle("‼️ Latest: timeout while updating") + mnu.backupNowEnable() + } else { + handleError(err) + } return } updateLatestSnapshotTitle(mnu) updateFutureSnapshotTitle(cnf, mnu) - mnu.backupNowSucceeded() + mnu.backupNowEnable() } func updateLatestSnapshotTitle(mnu *resticmenu) { @@ -112,7 +119,7 @@ func (mnu *resticmenu) backupNowFailed(err error) { mnu.backupNow.SetTitle("‼️ Backup now - latest failed!") } -func (mnu *resticmenu) backupNowSucceeded() { +func (mnu *resticmenu) backupNowEnable() { mnu.backupNow.Enable() mnu.backupNow.SetTitle("Backup now") } @@ -132,7 +139,7 @@ func onSystrayReady() { backupCheckTime := make(chan bool, 1) hourlyBackupCheckFn := func() { - time.Sleep(1 * time.Hour) + time.Sleep(1 * time.Minute) backupCheckTime <- true } go hourlyBackupCheckFn() diff --git a/restic/wrapper.go b/restic/wrapper.go index 920291b..bcaf605 100644 --- a/restic/wrapper.go +++ b/restic/wrapper.go @@ -27,6 +27,14 @@ type Wrapper struct { mountCommand *exec.Cmd } +type ResticCmdTimeoutError struct { + KillErr error +} + +func (r *ResticCmdTimeoutError) Error() string { + return fmt.Errorf("backup output timeout, repository server down?: %w", r.KillErr).Error() +} + func (w *Wrapper) HasSnapshots() bool { return len(w.LatestSnapshots) > 0 } @@ -70,43 +78,6 @@ func resticCmdFn(args ...string) *exec.Cmd { var resticCmd = resticCmdFn var cmdTimeout = 10 * time.Second -/* -UpdateLatestSnapshots updates LatestSnapshots or returns an error. -Expected restic snapshot JSON format: -[ - - { - "time": "2023-03-01T16:15:34.111513+01:00", - "tree": "d603aa4c6ce2bdef784dbdcd36461970d7f0cc8083d31f46d23cdb9bef172f0a", - "paths": [ - "/Users/user" - ], - "hostname": "M1-Air.local", - "username": "user", - "uid": 501, - "gid": 20, - "id": "31ae2a213c5750c4f86ebe8a8e989a5d4de2963c911e7513f47ca227723a0d95", - "short_id": "31ae2a21" - } - -] -*/ -func (w *Wrapper) UpdateLatestSnapshots(c *Config) error { - cmd := resticCmd("--json", "--password-file", PasswordFile(), "-r", c.Repository, "snapshots") - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("restic snapshots cmd: %s: %w", string(out), err) - } - - err = json.Unmarshal(out, &w.LatestSnapshots) - if err != nil { - return err - } - - log.Info().Any("snapshots", w.LatestSnapshots).Msg("update") - return nil -} - func (w *Wrapper) MountBackups(c *Config) error { c.CreateMountDirIfDoesntExist() w.Cleanup() @@ -144,6 +115,39 @@ func openFolder(folder string) error { return nil } +// UpdateLatestSnapshots updates LatestSnapshots or returns an error. If timed out, returns an error as well. +func (w *Wrapper) UpdateLatestSnapshots(c *Config) error { + cmd := resticCmd("--json", "--password-file", PasswordFile(), "-r", c.Repository, "snapshots") + + done := make(chan bool, 1) + var out []byte + var err error + go func() { + out, err = cmd.CombinedOutput() + done <- true + }() + + select { + case <-done: + case <-time.After(cmdTimeout): + return &ResticCmdTimeoutError{ + KillErr: cmd.Process.Kill(), + } + } + + if err != nil { + return fmt.Errorf("restic snapshots cmd: %s: %w", string(out), err) + } + + err = json.Unmarshal(out, &w.LatestSnapshots) + if err != nil { + return err + } + + log.Info().Any("snapshots", w.LatestSnapshots).Msg("update") + return nil +} + // Backup uses "restic backup" to create a new snapshot. This is a blocking call. func (w *Wrapper) Backup(c *Config) error { cmd := resticCmd("--password-file", PasswordFile(), "-r", c.Repository, "--exclude-file", ExcludeFile(), "backup", c.Backup, "--no-scan") @@ -169,8 +173,9 @@ func (w *Wrapper) Backup(c *Config) error { case <-busy: busy = nil // It's alive! continue with cmd.Wait() case <-time.After(cmdTimeout): - killErr := cmd.Process.Kill() - return fmt.Errorf("backup output timeout, repository server down?: %w", killErr) + return &ResticCmdTimeoutError{ + KillErr: cmd.Process.Kill(), + } } err := cmd.Wait() diff --git a/restic/wrapper_test.go b/restic/wrapper_test.go index a12644c..c01ebc8 100644 --- a/restic/wrapper_test.go +++ b/restic/wrapper_test.go @@ -2,6 +2,7 @@ package restic import ( "bytes" + "encoding/json" "fmt" "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" @@ -66,6 +67,35 @@ func fakeExecCommand(whichTest string) func(args ...string) *exec.Cmd { } } +func TestCmdnapshotSuccessWithinTime(t *testing.T) { + if os.Getenv("GO_TEST_PROCESS") != "1" { + return + } + snapshots := []ResticSnapshot{ + { + Id: "id1", + Time: timeNow(), + Tree: "tree", + Paths: []string{ + "/path/one", + "/path/two", + }, + }, + { + Id: "id2", + Time: timeNow(), + Tree: "tree2", + Paths: []string{ + "/path2/one", + "/path2/two", + }, + }, + } + bytes, _ := json.Marshal(snapshots) + fmt.Fprintf(os.Stdout, string(bytes)) + os.Exit(0) +} + func TestCmdSuccessWithinTime(t *testing.T) { if os.Getenv("GO_TEST_PROCESS") != "1" { return @@ -161,3 +191,27 @@ func TestBackup_TakesLongerThanNeededToOutput_TimesOut(t *testing.T) { assert.ErrorContainsf(t, err, "backup output timeout", "timeout expected") } + +func TestUpdateLatestSnapshot_TakesLongerThanNeededToOutput_TimesOut(t *testing.T) { + resticCmd = fakeExecCommand("TestCmdTakesLongerThan2sToOutputAnything") + cmdTimeout = 2 * time.Second + + wrapper := Wrapper{} + err := wrapper.UpdateLatestSnapshots(&Config{}) + + assert.Empty(t, wrapper.LatestSnapshots) + assert.ErrorContainsf(t, err, "backup output timeout", "timeout expected") +} + +func TestUpdateLatestSnapshots_FromJSONOutputOfRestic(t *testing.T) { + resticCmd = fakeExecCommand("TestCmdnapshotSuccessWithinTime") + + wrapper := Wrapper{} + err := wrapper.UpdateLatestSnapshots(&Config{}) + + assert.NoError(t, err) + assert.Equal(t, "id1", wrapper.LatestSnapshots[0].Id) + assert.Equal(t, "/path/two", wrapper.LatestSnapshots[0].Paths[1]) + assert.Equal(t, "id2", wrapper.LatestSnapshots[1].Id) + assert.Equal(t, "/path2/two", wrapper.LatestSnapshots[1].Paths[1]) +}