Skip to content

Commit

Permalink
Refactor parsing of output of zfs send -n -I
Browse files Browse the repository at this point in the history 
With `-I` output is multiline with one line for every intermediary snapshot,
like:

incremental	zrepl_daily_20240523_172615_CEST	pool/ds@zrepl_hourly_20240523_173543_CEST	624
incremental	zrepl_hourly_20240523_173543_CEST	pool/ds@zrepl_hourly_20240523_183541_CEST	624
size	1248
  • Loading branch information
@dsh2dsh
dsh2dsh committed May 23, 2024
1 parent dfcd804 commit 30c3d66
Show file tree
Hide file tree
Showing 2 changed files with 171 additions and 69 deletions.
114 changes: 63 additions & 51 deletions zfs/zfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1018,81 +1018,93 @@ type DrySendInfo struct {
SizeEstimate uint64 // 0 if size estimate is not possible
}

var (
// keep same number of capture groups for unmarshalInfoLine homogeneity

sendDryRunInfoLineRegexFull = regexp.MustCompile(`^(?P<type>full)\t()(?P<to>[^\t]+@[^\t]+)(\t(?P<size>[0-9]+))?$`)
// cannot enforce '[#@]' in incremental source, see test cases
sendDryRunInfoLineRegexIncremental = regexp.MustCompile(`^(?P<type>incremental)\t(?P<from>[^\t]+)\t(?P<to>[^\t]+@[^\t]+)(\t(?P<size>[0-9]+))?$`)
)

// see test cases for example output
func (s *DrySendInfo) unmarshalZFSOutput(output []byte) error {
debug("DrySendInfo.unmarshalZFSOutput: output=%q", output)
scan := bufio.NewScanner(bytes.NewReader(output))
for scan.Scan() {
l := scan.Text()
if regexMatched, err := s.unmarshalInfoLine(l); err != nil {
if err := s.unmarshalInfoLine(l); err != nil {
return fmt.Errorf("line %q: %w", l, err)
} else if regexMatched {
return nil
}
}
return fmt.Errorf("no match for info line (regex1 %s) (regex2 %s)",
sendDryRunInfoLineRegexFull, sendDryRunInfoLineRegexIncremental)

if s.Type != DrySendTypeFull && s.Type != DrySendTypeIncremental {
return fmt.Errorf("no match for info line in:\n%s", output)
}
return nil
}

// unmarshal info line, looks like this:
//
// full zroot/test/a@1 5389768
// incremental zroot/test/a@1 zroot/test/a@2 5383936
// size 5383936
//
// => see test cases
func (s *DrySendInfo) unmarshalInfoLine(l string) (regexMatched bool, err error) {
mFull := sendDryRunInfoLineRegexFull.FindStringSubmatch(l)
mInc := sendDryRunInfoLineRegexIncremental.FindStringSubmatch(l)
var matchingExpr *regexp.Regexp
var m []string
if mFull == nil && mInc == nil {
return false, nil
} else if mFull != nil && mInc != nil {
panic(fmt.Sprintf("ambiguous ZFS dry send output: %q", l))
} else if mFull != nil {
matchingExpr, m = sendDryRunInfoLineRegexFull, mFull
} else if mInc != nil {
matchingExpr, m = sendDryRunInfoLineRegexIncremental, mInc
}

fields := make(map[string]string, matchingExpr.NumSubexp())
for i, name := range matchingExpr.SubexpNames() {
if i != 0 {
fields[name] = m[i]
}
func (s *DrySendInfo) unmarshalInfoLine(l string) error {
dryFields := strings.SplitN(l, "\t", 5)
n := len(dryFields)
if n == 0 {
return nil
}

s.Type, err = DrySendTypeFromString(fields["type"])
if err != nil {
return true, err
snapType := dryFields[0]
var from, to, size string
switch {
case snapType == "full" && n > 1:
to = dryFields[1]
if n > 2 {
size = dryFields[2]
}
case snapType == "incremental" && n > 2:
from, to = dryFields[1], dryFields[2]
if n > 3 {
size = dryFields[3]
}
if s.From == "" {
s.From = from
}
case snapType == "size" && n > 1:
size = dryFields[1]
default:
return nil
}

s.From = fields["from"]
s.To = fields["to"]
toFS, _, _, err := DecomposeVersionString(s.To)
if err != nil {
return true, fmt.Errorf("'to' is not a valid filesystem version: %s", err)
if snapType == "full" || snapType == "incremental" {
if sendType, err := DrySendTypeFromString(snapType); err != nil {
return err
} else if s.Type == "" {
s.Type = sendType
} else if sendType != s.Type {
return fmt.Errorf("dry send type changed from %q to %q", s.Type, sendType)
}
s.To = to
if s.Filesystem == "" {
toFS, _, _, err := DecomposeVersionString(to)
if err != nil {
return fmt.Errorf("'to' is not a valid filesystem version: %s", err)
}
s.Filesystem = toFS
}
if size == "" {
// workaround for OpenZFS 0.7 prior to
//
// https://github.com/openzfs/zfs/commit/835db58592d7d947e5818eb7281882e2a46073e0#diff-66bd524398bcd2ac70d90925ab6d8073L1245
//
// see https://github.com/zrepl/zrepl/issues/289
return nil
}
}
s.Filesystem = toFS

if fields["size"] == "" {
// workaround for OpenZFS 0.7 prior to https://github.com/openzfs/zfs/commit/835db58592d7d947e5818eb7281882e2a46073e0#diff-66bd524398bcd2ac70d90925ab6d8073L1245
// see https://github.com/zrepl/zrepl/issues/289
fields["size"] = "0"
}
s.SizeEstimate, err = strconv.ParseUint(fields["size"], 10, 64)
if err != nil {
return true, fmt.Errorf("cannot not parse size: %s", err)
if sizeEstimate, err := strconv.ParseUint(size, 10, 64); err != nil {
return fmt.Errorf("cannot not parse size %q: %s", size, err)
} else if snapType == "size" {
s.SizeEstimate = max(sizeEstimate, s.SizeEstimate)
} else {
s.SizeEstimate += sizeEstimate
}
return true, nil
return nil
}

// to may be "", in which case a full ZFS send is done
Expand Down
126 changes: 108 additions & 18 deletions zfs/zfs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,6 @@ func TestZFSPropertySource(t *testing.T) {
}
}

func TestDrySendRegexesHaveSameCaptureGroupCount(t *testing.T) {
assert.Equal(t, sendDryRunInfoLineRegexFull.NumSubexp(), sendDryRunInfoLineRegexIncremental.NumSubexp())
}

func TestDrySendInfo(t *testing.T) {
// # full send
// $ zfs send -Pnv -t 1-9baebea70-b8-789c636064000310a500c4ec50360710e72765a52697303030419460caa7a515a79680647ce0f26c48f2499525a9c5405ac3c90fabfe92fcf4d2cc140686b30972c7850efd0cd24092e704cbe725e6a632305415e5e797e803cd2ad14f743084b805001b201795
Expand Down Expand Up @@ -179,7 +175,8 @@ full p1 with/ spaces d1@2 with space

tcs := []tc{
{
name: "fullSend", in: fullSend,
name: "fullSend",
in: fullSend,
exp: &DrySendInfo{
Type: DrySendTypeFull,
Filesystem: "zroot/test/a",
Expand All @@ -189,7 +186,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "incSend", in: incSend,
name: "incSend",
in: incSend,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "zroot/test/a",
Expand All @@ -199,7 +197,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "incSendBookmark", in: incSendBookmark,
name: "incSendBookmark",
in: incSendBookmark,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "zroot/test/a",
Expand All @@ -209,7 +208,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "incNoToken", in: incNoToken,
name: "incNoToken",
in: incNoToken,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "zroot/test/a",
Expand All @@ -221,7 +221,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "fullNoToken", in: fullNoToken,
name: "fullNoToken",
in: fullNoToken,
exp: &DrySendInfo{
Type: DrySendTypeFull,
Filesystem: "zroot/test/a",
Expand All @@ -231,7 +232,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "fullWithSpaces", in: fullWithSpaces,
name: "fullWithSpaces",
in: fullWithSpaces,
exp: &DrySendInfo{
Type: DrySendTypeFull,
Filesystem: "pool1/otherjob/ds with spaces",
Expand All @@ -241,7 +243,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "fullWithSpacesInIntermediateComponent", in: fullWithSpacesInIntermediateComponent,
name: "fullWithSpacesInIntermediateComponent",
in: fullWithSpacesInIntermediateComponent,
exp: &DrySendInfo{
Type: DrySendTypeFull,
Filesystem: "pool1/otherjob/another ds with spaces/childfs",
Expand All @@ -251,7 +254,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "incrementalWithSpaces", in: incrementalWithSpaces,
name: "incrementalWithSpaces",
in: incrementalWithSpaces,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "pool1/otherjob/another ds with spaces",
Expand All @@ -261,7 +265,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "incrementalWithSpacesInIntermediateComponent", in: incrementalWithSpacesInIntermediateComponent,
name: "incrementalWithSpacesInIntermediateComponent",
in: incrementalWithSpacesInIntermediateComponent,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "pool1/otherjob/another ds with spaces/childfs",
Expand All @@ -271,7 +276,8 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "incrementalZeroSizedOpenZFS_pre0.7.12", in: incZeroSized_0_7_12,
name: "incrementalZeroSizedOpenZFS_pre0.7.12",
in: incZeroSized_0_7_12,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "p1 with/ spaces d1",
Expand All @@ -281,14 +287,98 @@ full p1 with/ spaces d1@2 with space
},
},
{
name: "fullZeroSizedOpenZFS_pre0.7.12", in: fullZeroSized_0_7_12,
name: "fullZeroSizedOpenZFS_pre0.7.12",
in: fullZeroSized_0_7_12,
exp: &DrySendInfo{
Type: DrySendTypeFull,
Filesystem: "p1 with/ spaces d1",
To: "p1 with/ spaces d1@2 with space",
SizeEstimate: 0,
},
},
{
name: "incremental package without size",
in: `
incremental snap1 pool1/ds@snap2 624
incremental snap2 pool1/ds@snap3 624
`,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "pool1/ds",
From: "snap1",
To: "pool1/ds@snap3",
SizeEstimate: 1248,
},
},
{
name: "incremental package with size",
in: `
incremental snap1 pool1/ds@snap2 624
incremental snap2 pool1/ds@snap3 624
size 1248
`,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "pool1/ds",
From: "snap1",
To: "pool1/ds@snap3",
SizeEstimate: 1248,
},
},
{
name: "dry send type changed",
in: `
full pool1/ds@snap2 624
incremental snap2 pool1/ds@snap3 624
size 1248
`,
expErr: true,
},
{
name: "no match for info line",
in: `
size 1248
`,
expErr: true,
},
{
name: "cannot not parse size",
in: `
incremental snap2 pool1/ds@snap3 624
size XXX
`,
expErr: true,
},
{
name: "incremental package with size less",
in: `
incremental snap1 pool1/ds@snap2 624
incremental snap2 pool1/ds@snap3 624
size 1000
`,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "pool1/ds",
From: "snap1",
To: "pool1/ds@snap3",
SizeEstimate: 1248,
},
},
{
name: "incremental package with size more",
in: `
incremental snap1 pool1/ds@snap2 624
incremental snap2 pool1/ds@snap3 624
size 1500
`,
exp: &DrySendInfo{
Type: DrySendTypeIncremental,
Filesystem: "pool1/ds",
From: "snap1",
To: "pool1/ds@snap3",
SizeEstimate: 1500,
},
},
}

for _, tc := range tcs {
Expand All @@ -300,9 +390,9 @@ full p1 with/ spaces d1@2 with space
t.Logf("err=%T %s", err, err)

if tc.expErr {
assert.Error(t, err)
}
if tc.exp != nil {
require.Error(t, err)
} else {
require.NoError(t, err)
assert.Equal(t, tc.exp, &si)
}
})
Expand Down

0 comments on commit 30c3d66

Please sign in to comment.