implement a timeout system for snapshot checking
This commit is contained in:
parent
acee286a19
commit
ec12985ee7
15
main.go
15
main.go
|
@ -66,13 +66,20 @@ func main() {
|
||||||
func updateSnapshots(cnf *restic.Config, mnu *resticmenu) {
|
func updateSnapshots(cnf *restic.Config, mnu *resticmenu) {
|
||||||
err := wrapper.UpdateLatestSnapshots(cnf)
|
err := wrapper.UpdateLatestSnapshots(cnf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
handleError(err)
|
te, ok := err.(*restic.ResticCmdTimeoutError)
|
||||||
|
if ok {
|
||||||
|
log.Warn().Err(te).Msg("update snapshot timeout")
|
||||||
|
mnu.latestSnapshot.SetTitle("‼️ Latest: timeout while updating")
|
||||||
|
mnu.backupNowEnable()
|
||||||
|
} else {
|
||||||
|
handleError(err)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
updateLatestSnapshotTitle(mnu)
|
updateLatestSnapshotTitle(mnu)
|
||||||
updateFutureSnapshotTitle(cnf, mnu)
|
updateFutureSnapshotTitle(cnf, mnu)
|
||||||
mnu.backupNowSucceeded()
|
mnu.backupNowEnable()
|
||||||
}
|
}
|
||||||
|
|
||||||
func updateLatestSnapshotTitle(mnu *resticmenu) {
|
func updateLatestSnapshotTitle(mnu *resticmenu) {
|
||||||
|
@ -112,7 +119,7 @@ func (mnu *resticmenu) backupNowFailed(err error) {
|
||||||
mnu.backupNow.SetTitle("‼️ Backup now - latest failed!")
|
mnu.backupNow.SetTitle("‼️ Backup now - latest failed!")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mnu *resticmenu) backupNowSucceeded() {
|
func (mnu *resticmenu) backupNowEnable() {
|
||||||
mnu.backupNow.Enable()
|
mnu.backupNow.Enable()
|
||||||
mnu.backupNow.SetTitle("Backup now")
|
mnu.backupNow.SetTitle("Backup now")
|
||||||
}
|
}
|
||||||
|
@ -132,7 +139,7 @@ func onSystrayReady() {
|
||||||
|
|
||||||
backupCheckTime := make(chan bool, 1)
|
backupCheckTime := make(chan bool, 1)
|
||||||
hourlyBackupCheckFn := func() {
|
hourlyBackupCheckFn := func() {
|
||||||
time.Sleep(1 * time.Hour)
|
time.Sleep(1 * time.Minute)
|
||||||
backupCheckTime <- true
|
backupCheckTime <- true
|
||||||
}
|
}
|
||||||
go hourlyBackupCheckFn()
|
go hourlyBackupCheckFn()
|
||||||
|
|
|
@ -27,6 +27,14 @@ type Wrapper struct {
|
||||||
mountCommand *exec.Cmd
|
mountCommand *exec.Cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ResticCmdTimeoutError struct {
|
||||||
|
KillErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ResticCmdTimeoutError) Error() string {
|
||||||
|
return fmt.Errorf("backup output timeout, repository server down?: %w", r.KillErr).Error()
|
||||||
|
}
|
||||||
|
|
||||||
func (w *Wrapper) HasSnapshots() bool {
|
func (w *Wrapper) HasSnapshots() bool {
|
||||||
return len(w.LatestSnapshots) > 0
|
return len(w.LatestSnapshots) > 0
|
||||||
}
|
}
|
||||||
|
@ -70,43 +78,6 @@ func resticCmdFn(args ...string) *exec.Cmd {
|
||||||
var resticCmd = resticCmdFn
|
var resticCmd = resticCmdFn
|
||||||
var cmdTimeout = 10 * time.Second
|
var cmdTimeout = 10 * time.Second
|
||||||
|
|
||||||
/*
|
|
||||||
UpdateLatestSnapshots updates LatestSnapshots or returns an error.
|
|
||||||
Expected restic snapshot JSON format:
|
|
||||||
[
|
|
||||||
|
|
||||||
{
|
|
||||||
"time": "2023-03-01T16:15:34.111513+01:00",
|
|
||||||
"tree": "d603aa4c6ce2bdef784dbdcd36461970d7f0cc8083d31f46d23cdb9bef172f0a",
|
|
||||||
"paths": [
|
|
||||||
"/Users/user"
|
|
||||||
],
|
|
||||||
"hostname": "M1-Air.local",
|
|
||||||
"username": "user",
|
|
||||||
"uid": 501,
|
|
||||||
"gid": 20,
|
|
||||||
"id": "31ae2a213c5750c4f86ebe8a8e989a5d4de2963c911e7513f47ca227723a0d95",
|
|
||||||
"short_id": "31ae2a21"
|
|
||||||
}
|
|
||||||
|
|
||||||
]
|
|
||||||
*/
|
|
||||||
func (w *Wrapper) UpdateLatestSnapshots(c *Config) error {
|
|
||||||
cmd := resticCmd("--json", "--password-file", PasswordFile(), "-r", c.Repository, "snapshots")
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("restic snapshots cmd: %s: %w", string(out), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = json.Unmarshal(out, &w.LatestSnapshots)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info().Any("snapshots", w.LatestSnapshots).Msg("update")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *Wrapper) MountBackups(c *Config) error {
|
func (w *Wrapper) MountBackups(c *Config) error {
|
||||||
c.CreateMountDirIfDoesntExist()
|
c.CreateMountDirIfDoesntExist()
|
||||||
w.Cleanup()
|
w.Cleanup()
|
||||||
|
@ -144,6 +115,39 @@ func openFolder(folder string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UpdateLatestSnapshots updates LatestSnapshots or returns an error. If timed out, returns an error as well.
|
||||||
|
func (w *Wrapper) UpdateLatestSnapshots(c *Config) error {
|
||||||
|
cmd := resticCmd("--json", "--password-file", PasswordFile(), "-r", c.Repository, "snapshots")
|
||||||
|
|
||||||
|
done := make(chan bool, 1)
|
||||||
|
var out []byte
|
||||||
|
var err error
|
||||||
|
go func() {
|
||||||
|
out, err = cmd.CombinedOutput()
|
||||||
|
done <- true
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(cmdTimeout):
|
||||||
|
return &ResticCmdTimeoutError{
|
||||||
|
KillErr: cmd.Process.Kill(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("restic snapshots cmd: %s: %w", string(out), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = json.Unmarshal(out, &w.LatestSnapshots)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info().Any("snapshots", w.LatestSnapshots).Msg("update")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Backup uses "restic backup" to create a new snapshot. This is a blocking call.
|
// Backup uses "restic backup" to create a new snapshot. This is a blocking call.
|
||||||
func (w *Wrapper) Backup(c *Config) error {
|
func (w *Wrapper) Backup(c *Config) error {
|
||||||
cmd := resticCmd("--password-file", PasswordFile(), "-r", c.Repository, "--exclude-file", ExcludeFile(), "backup", c.Backup, "--no-scan")
|
cmd := resticCmd("--password-file", PasswordFile(), "-r", c.Repository, "--exclude-file", ExcludeFile(), "backup", c.Backup, "--no-scan")
|
||||||
|
@ -169,8 +173,9 @@ func (w *Wrapper) Backup(c *Config) error {
|
||||||
case <-busy:
|
case <-busy:
|
||||||
busy = nil // It's alive! continue with cmd.Wait()
|
busy = nil // It's alive! continue with cmd.Wait()
|
||||||
case <-time.After(cmdTimeout):
|
case <-time.After(cmdTimeout):
|
||||||
killErr := cmd.Process.Kill()
|
return &ResticCmdTimeoutError{
|
||||||
return fmt.Errorf("backup output timeout, repository server down?: %w", killErr)
|
KillErr: cmd.Process.Kill(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err := cmd.Wait()
|
err := cmd.Wait()
|
||||||
|
|
|
@ -2,6 +2,7 @@ package restic
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
@ -66,6 +67,35 @@ func fakeExecCommand(whichTest string) func(args ...string) *exec.Cmd {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCmdnapshotSuccessWithinTime(t *testing.T) {
|
||||||
|
if os.Getenv("GO_TEST_PROCESS") != "1" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
snapshots := []ResticSnapshot{
|
||||||
|
{
|
||||||
|
Id: "id1",
|
||||||
|
Time: timeNow(),
|
||||||
|
Tree: "tree",
|
||||||
|
Paths: []string{
|
||||||
|
"/path/one",
|
||||||
|
"/path/two",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Id: "id2",
|
||||||
|
Time: timeNow(),
|
||||||
|
Tree: "tree2",
|
||||||
|
Paths: []string{
|
||||||
|
"/path2/one",
|
||||||
|
"/path2/two",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
bytes, _ := json.Marshal(snapshots)
|
||||||
|
fmt.Fprintf(os.Stdout, string(bytes))
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
func TestCmdSuccessWithinTime(t *testing.T) {
|
func TestCmdSuccessWithinTime(t *testing.T) {
|
||||||
if os.Getenv("GO_TEST_PROCESS") != "1" {
|
if os.Getenv("GO_TEST_PROCESS") != "1" {
|
||||||
return
|
return
|
||||||
|
@ -161,3 +191,27 @@ func TestBackup_TakesLongerThanNeededToOutput_TimesOut(t *testing.T) {
|
||||||
|
|
||||||
assert.ErrorContainsf(t, err, "backup output timeout", "timeout expected")
|
assert.ErrorContainsf(t, err, "backup output timeout", "timeout expected")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUpdateLatestSnapshot_TakesLongerThanNeededToOutput_TimesOut(t *testing.T) {
|
||||||
|
resticCmd = fakeExecCommand("TestCmdTakesLongerThan2sToOutputAnything")
|
||||||
|
cmdTimeout = 2 * time.Second
|
||||||
|
|
||||||
|
wrapper := Wrapper{}
|
||||||
|
err := wrapper.UpdateLatestSnapshots(&Config{})
|
||||||
|
|
||||||
|
assert.Empty(t, wrapper.LatestSnapshots)
|
||||||
|
assert.ErrorContainsf(t, err, "backup output timeout", "timeout expected")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateLatestSnapshots_FromJSONOutputOfRestic(t *testing.T) {
|
||||||
|
resticCmd = fakeExecCommand("TestCmdnapshotSuccessWithinTime")
|
||||||
|
|
||||||
|
wrapper := Wrapper{}
|
||||||
|
err := wrapper.UpdateLatestSnapshots(&Config{})
|
||||||
|
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, "id1", wrapper.LatestSnapshots[0].Id)
|
||||||
|
assert.Equal(t, "/path/two", wrapper.LatestSnapshots[0].Paths[1])
|
||||||
|
assert.Equal(t, "id2", wrapper.LatestSnapshots[1].Id)
|
||||||
|
assert.Equal(t, "/path2/two", wrapper.LatestSnapshots[1].Paths[1])
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue