fs: add DiffDirChanges function to get changeset fast

Makefile

@@ -57,7 +57,7 @@ test:
 
 root-test:
 	@echo "+ $@"
-	@go test -exec sudo ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
+	@go test -exec sudo ${TEST_REQUIRES_ROOT_PACKAGES} -test.root -test.v
 
 test-compile:
 	@echo "+ $@"

fs/diff.go

@@ -18,6 +18,7 @@ package fs
 
 import (
 	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
@@ -102,9 +103,6 @@ func Changes(ctx context.Context, a, b string, changeFn ChangeFunc) error {
 	if a == "" {
 		logrus.Debugf("Using single walk diff for %s", b)
 		return addDirChanges(ctx, changeFn, b)
-	} else if diffOptions := detectDirDiff(b, a); diffOptions != nil {
-		logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, a)
-		return diffDirChanges(ctx, changeFn, a, diffOptions)
 	}
 
 	logrus.Debugf("Using double walk diff for %s from %s", b, a)
@@ -134,24 +132,53 @@ func addDirChanges(ctx context.Context, changeFn ChangeFunc, root string) error
 	})
 }
 
+// DiffChangeSource is the source of diff directory.
+type DiffSource int
+
+const (
+	// DiffSourceOverlayFS indicates that a diff directory is from
+	// OverlayFS.
+	DiffSourceOverlayFS DiffSource = iota
+)
+
 // diffDirOptions is used when the diff can be directly calculated from
 // a diff directory to its base, without walking both trees.
 type diffDirOptions struct {
-	diffDir      string
-	skipChange   func(string) (bool, error)
-	deleteChange func(string, string, os.FileInfo) (string, error)
+	skipChange   func(string, os.FileInfo) (bool, error)
+	deleteChange func(string, string, os.FileInfo, ChangeFunc) (bool, error)
 }
 
-// diffDirChanges walks the diff directory and compares changes against the base.
-func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *diffDirOptions) error {
+// DiffDirChanges walks the diff directory and compares changes against the base.
+//
+// NOTE: If all the children of a dir are removed, or that dir are recreated
+// after remove, we will mark non-existing `.wh..opq` file as deleted. It's
+// unlikely to create explicit whiteout files for all the children and all
+// descendants. And based on OCI spec, it's not possible to create a file or
+// dir with a name beginning with `.wh.`. So, after `.wh..opq` file has been
+// deleted, the ChangeFunc, the receiver will add whiteout prefix to create a
+// opaque whiteout `.wh..wh..opq`.
+//
+// REF: https://github.com/opencontainers/image-spec/blob/v1.0/layer.md#whiteouts
+func DiffDirChanges(ctx context.Context, baseDir, diffDir string, source DiffSource, changeFn ChangeFunc) error {
+	var o *diffDirOptions
+
+	switch source {
+	case DiffSourceOverlayFS:
+		o = &diffDirOptions{
+			deleteChange: overlayFSWhiteoutConvert,
+		}
+	default:
+		return errors.New("unknown diff change source")
+	}
+
 	changedDirs := make(map[string]struct{})
-	return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error {
+	return filepath.Walk(diffDir, func(path string, f os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
 		// Rebase path
-		path, err = filepath.Rel(o.diffDir, path)
+		path, err = filepath.Rel(diffDir, path)
 		if err != nil {
 			return err
 		}
@@ -163,38 +190,45 @@ func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *di
 			return nil
 		}
 
-		// TODO: handle opaqueness, start new double walker at this
-		// location to get deletes, and skip tree in single walker
-
 		if o.skipChange != nil {
-			if skip, err := o.skipChange(path); skip {
+			if skip, err := o.skipChange(path, f); skip {
 				return err
 			}
 		}
 
 		var kind ChangeKind
 
-		deletedFile, err := o.deleteChange(o.diffDir, path, f)
-		if err != nil {
-			return err
+		deletedFile := false
+
+		if o.deleteChange != nil {
+			deletedFile, err = o.deleteChange(diffDir, path, f, changeFn)
+			if err != nil {
+				return err
+			}
+
+			_, err = os.Stat(filepath.Join(baseDir, path))
+			if err != nil {
+				if !os.IsNotExist(err) {
+					return err
+				}
+				deletedFile = false
+			}
 		}
 
 		// Find out what kind of modification happened
-		if deletedFile != "" {
-			path = deletedFile
+		if deletedFile {
 			kind = ChangeKindDelete
-			f = nil
 		} else {
 			// Otherwise, the file was added
 			kind = ChangeKindAdd
 
-			// ...Unless it already existed in a base, in which case, it's a modification
-			stat, err := os.Stat(filepath.Join(base, path))
+			// ...Unless it already existed in a baseDir, in which case, it's a modification
+			stat, err := os.Stat(filepath.Join(baseDir, path))
 			if err != nil && !os.IsNotExist(err) {
 				return err
 			}
 			if err == nil {
-				// The file existed in the base, so that's a modification
+				// The file existed in the baseDir, so that's a modification
 
 				// However, if it's a directory, maybe it wasn't actually modified.
 				// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
@@ -215,17 +249,22 @@ func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *di
 		if f.IsDir() {
 			changedDirs[path] = struct{}{}
 		}
+
 		if kind == ChangeKindAdd || kind == ChangeKindDelete {
 			parent := filepath.Dir(path)
+
 			if _, ok := changedDirs[parent]; !ok && parent != "/" {
-				pi, err := os.Stat(filepath.Join(o.diffDir, parent))
+				pi, err := os.Stat(filepath.Join(diffDir, parent))
 				if err := changeFn(ChangeKindModify, parent, pi, err); err != nil {
 					return err
 				}
 				changedDirs[parent] = struct{}{}
 			}
 		}
 
+		if kind == ChangeKindDelete {
+			f = nil
+		}
 		return changeFn(kind, path, f, nil)
 	})
 }

fs/diff_linux.go

@@ -0,0 +1,101 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/containerd/continuity/devices"
+	"github.com/containerd/continuity/sysx"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// whiteoutPrefix prefix means file is a whiteout. If this is followed
+	// by a filename this means that file has been removed from the base
+	// layer.
+	//
+	// See https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
+	whiteoutPrefix = ".wh."
+)
+
+// overlayFSWhiteoutConvert detects whiteouts and opaque directories.
+//
+// It returns deleted indicator if the file is a character device with 0/0
+// device number. And call changeFn with ChangeKindDelete for opaque
+// directories.
+//
+// Check: https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt
+func overlayFSWhiteoutConvert(diffDir, path string, f os.FileInfo, changeFn ChangeFunc) (deleted bool, _ error) {
+	if f.Mode()&os.ModeCharDevice != 0 {
+		if _, ok := f.Sys().(*syscall.Stat_t); !ok {
+			return false, nil
+		}
+
+		maj, min, err := devices.DeviceInfo(f)
+		if err != nil {
+			return false, err
+		}
+		return (maj == 0 && min == 0), nil
+	}
+
+	if f.IsDir() {
+		originalPath := filepath.Join(diffDir, path)
+		opaque, err := getOpaqueValue(originalPath)
+		if err != nil {
+			if errors.Is(err, unix.ENODATA) {
+				return false, nil
+			}
+			return false, err
+		}
+
+		if len(opaque) == 1 && opaque[0] == 'y' {
+			opaqueDirPath := filepath.Join(path, whiteoutPrefix+".opq")
+			return false, changeFn(ChangeKindDelete, opaqueDirPath, nil, nil)
+		}
+	}
+	return false, nil
+}
+
+// getOpaqueValue returns opaque value for a given file.
+func getOpaqueValue(filePath string) ([]byte, error) {
+	for _, xattr := range []string{
+		"trusted.overlay.opaque",
+		// TODO(fuweid):
+		//
+		// user.overlay.* is available since 5.11. We should check
+		// kernel version before read.
+		//
+		// REF: https://github.com/torvalds/linux/commit/2d2f2d7322ff43e0fe92bf8cccdc0b09449bf2e1
+		"user.overlay.opaque",
+	} {
+		opaque, err := sysx.LGetxattr(filePath, xattr)
+		if err != nil {
+			if errors.Is(err, unix.ENODATA) || errors.Is(err, unix.ENOTSUP) {
+				continue
+			}
+			return nil, fmt.Errorf("failed to retrieve %s attr: %w", xattr, err)
+		}
+		return opaque, nil
+	}
+	return nil, unix.ENODATA
+}

fs/diff_nonlinux.go

@@ -0,0 +1,29 @@
+//go:build !linux
+// +build !linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fs
+
+import (
+	"errors"
+	"os"
+)
+
+func overlayFSWhiteoutConvert(string, string, os.FileInfo, ChangeFunc) (bool, error) {
+	return false, errors.New("unsupported")
+}