PackageManager/Client/Filesystem/FilesystemDiff.go

package Filesystem

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"sync"

	"github.com/vbauerster/mpb"
	bolt "go.etcd.io/bbolt"
	"golang.org/x/sync/semaphore"

	"PackageManager/Client/Database"
	"PackageManager/Client/ProgressBar"
	"PackageManager/Color"
	"PackageManager/Variables"
)

var (
	fsStatusWG sync.WaitGroup
	FsWalkWG   sync.WaitGroup
)

type FilesystemStatus struct {
	NewFiles      []string
	PickedFiles   []string
	ModifiedFiles []string
	MissingFiles  []string
}

func ShowFilesystemDiff(root string) error {
	var (
		fsStatus    FilesystemStatus
		picksBucket *bolt.Bucket
		pickedFiles []string
		err         error
	)

	fsStatus, err = GetFilesystemDiff(root)
	if err != nil {
		return err
	}

	err = Database.FsDB.View(func(tx *bolt.Tx) error {
		picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
		return picksBucket.ForEach(func(key, _ []byte) error {
			pickedFiles = append(pickedFiles, string(key))
			return nil
		})
	})
	if err != nil {
		return err
	}

	fmt.Println("New files:")
	PrintFilesOrLength(fsStatus.NewFiles, Color.Green)

	fmt.Println("Added files:")
	PrintFilesOrLength(pickedFiles, Color.Green)

	fmt.Println("Modified files:")
	PrintFilesOrLength(fsStatus.ModifiedFiles, Color.Warning)

	fmt.Println("Deleted files:")
	PrintFilesOrLength(fsStatus.MissingFiles, Color.Fatal)

	return nil
}

func GetFilesystemLength(root string) (int, error) {
	var (
		rootStat os.FileInfo
		fsCount  int = 0
		err      error
	)

	rootStat, err = os.Stat(root)
	if err != nil {
		return fsCount, err
	}

	if rootStat.IsDir() && root[len(root)-1:] != "/" {
		root = root + "/"
	}

	err = filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {

		// Ignore path in Variables.PruneRegexPaths
		if i.IsDir() && matchAny(p, PruneRegex) {
			return filepath.SkipDir
		}

		// Ignore path in Variables.IgnoreRegexPaths
		if matchAny(p, IgnoreRegex) {
			return nil
		}

		if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
			return nil
		}

		fsCount++

		return nil
	})

	return fsCount, err
}

func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) {
	var (
		newFileObject   FileObject
		knownFileObject FileObject
		pick, known     []byte
		err             error
	)

	defer func() {
		bar.Increment()
	}()

	pick = picksBucket.Get([]byte(p))
	known = indexBucket.Get([]byte(p))

	if pick != nil {
		fsStatusWG.Wait()
		fsStatusWG.Add(1)
		fsStatus.PickedFiles = append(fsStatus.PickedFiles, p)
		fsStatusWG.Done()
		return
	}

	if known != nil {
		newFileObject, err = CreateFileObject(p)
		if err != nil {
			return
		}

		knownFileObject, err = FromBytes(known)
		if err != nil {
			return
		}

		err = newFileObject.IsDifferent(knownFileObject)
		if err != nil {
			fsStatusWG.Wait()
			fsStatusWG.Add(1)
			fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p)
			fsStatusWG.Done()
		}

		return
	}

	fsStatusWG.Wait()
	fsStatusWG.Add(1)
	fsStatus.NewFiles = append(fsStatus.NewFiles, p)
	fsStatusWG.Done()
}

func GetFilesystemDiff(root string) (FilesystemStatus, error) {
	var (
		fsStatus    FilesystemStatus = FilesystemStatus{}
		sem         *semaphore.Weighted
		picksBucket *bolt.Bucket
		indexBucket *bolt.Bucket
		rootStat    os.FileInfo
		bar         *mpb.Bar
		fsCount     int
		poolSize    int
		err         error
	)

	poolSize = runtime.NumCPU()
	sem = semaphore.NewWeighted(int64(poolSize))

	rootStat, err = os.Stat(root)
	if err != nil {
		return fsStatus, err
	}

	if rootStat.IsDir() && root[len(root)-1:] != "/" {
		root = root + "/"
	}

	fsCount, err = GetFilesystemLength(root)
	if err != nil {
		return fsStatus, err
	}

	bar = ProgressBar.InitBar("Scanning...", fsCount)

	err = Database.FsDB.View(func(tx *bolt.Tx) error {

		picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
		indexBucket = tx.Bucket(Variables.FsHashIndexBucket)

		filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
			// Ignore path in Variables.PruneRegexPaths
			if i.IsDir() && matchAny(p, PruneRegex) {
				return filepath.SkipDir
			}

			// Ignore path in Variables.IgnoreRegexPaths
			if matchAny(p, IgnoreRegex) {
				return nil
			}

			if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
				return nil
			}

			FsWalkWG.Add(1)
			sem.Acquire(context.Background(), 1)
			go func() {
				fsStatus.parseFile(indexBucket, picksBucket, p, bar)
				sem.Release(1)
				FsWalkWG.Done()
			}()

			FsWalkWG.Wait()
			return nil
		})

		indexBucket.ForEach(func(k, v []byte) error {
			_, err = os.Lstat(string(k))
			if os.IsNotExist(err) {
				fsStatus.MissingFiles = append(fsStatus.MissingFiles, string(k))
			}
			return nil
		})

		return nil
	})

	return fsStatus, err
}