You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
240 lines
4.6 KiB
240 lines
4.6 KiB
package Filesystem
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sync"
|
|
|
|
"github.com/vbauerster/mpb"
|
|
bolt "go.etcd.io/bbolt"
|
|
"golang.org/x/sync/semaphore"
|
|
|
|
"PackageManager/Client/Database"
|
|
"PackageManager/Client/ProgressBar"
|
|
"PackageManager/Color"
|
|
"PackageManager/Variables"
|
|
)
|
|
|
|
var (
|
|
fsStatusWG sync.WaitGroup
|
|
FsWalkWG sync.WaitGroup
|
|
)
|
|
|
|
type FilesystemStatus struct {
|
|
NewFiles []string
|
|
PickedFiles []string
|
|
ModifiedFiles []string
|
|
MissingFiles []string
|
|
}
|
|
|
|
func ShowFilesystemDiff(root string) error {
|
|
var (
|
|
fsStatus FilesystemStatus
|
|
picksBucket *bolt.Bucket
|
|
pickedFiles []string
|
|
err error
|
|
)
|
|
|
|
fsStatus, err = GetFilesystemDiff(root)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = Database.FsDB.View(func(tx *bolt.Tx) error {
|
|
picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
|
|
return picksBucket.ForEach(func(key, _ []byte) error {
|
|
pickedFiles = append(pickedFiles, string(key))
|
|
return nil
|
|
})
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
fmt.Println("New files:")
|
|
PrintFilesOrLength(fsStatus.NewFiles, Color.Green)
|
|
|
|
fmt.Println("Added files:")
|
|
PrintFilesOrLength(pickedFiles, Color.Green)
|
|
|
|
fmt.Println("Modified files:")
|
|
PrintFilesOrLength(fsStatus.ModifiedFiles, Color.Warning)
|
|
|
|
fmt.Println("Deleted files:")
|
|
PrintFilesOrLength(fsStatus.MissingFiles, Color.Fatal)
|
|
|
|
return nil
|
|
}
|
|
|
|
func GetFilesystemLength(root string) (int, error) {
|
|
var (
|
|
rootStat os.FileInfo
|
|
fsCount int = 0
|
|
err error
|
|
)
|
|
|
|
rootStat, err = os.Stat(root)
|
|
if err != nil {
|
|
return fsCount, err
|
|
}
|
|
|
|
if rootStat.IsDir() && root[len(root)-1:] != "/" {
|
|
root = root + "/"
|
|
}
|
|
|
|
err = filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
|
|
|
|
// Ignore path in Variables.PruneRegexPaths
|
|
if i.IsDir() && matchAny(p, PruneRegex) {
|
|
return filepath.SkipDir
|
|
}
|
|
|
|
// Ignore path in Variables.IgnoreRegexPaths
|
|
if matchAny(p, IgnoreRegex) {
|
|
return nil
|
|
}
|
|
|
|
if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
|
|
return nil
|
|
}
|
|
|
|
fsCount++
|
|
|
|
return nil
|
|
})
|
|
|
|
return fsCount, err
|
|
}
|
|
|
|
func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) {
|
|
var (
|
|
newFileObject FileObject
|
|
knownFileObject FileObject
|
|
pick, known []byte
|
|
err error
|
|
)
|
|
|
|
defer func() {
|
|
bar.Increment()
|
|
}()
|
|
|
|
pick = picksBucket.Get([]byte(p))
|
|
known = indexBucket.Get([]byte(p))
|
|
|
|
if pick != nil {
|
|
fsStatusWG.Wait()
|
|
fsStatusWG.Add(1)
|
|
fsStatus.PickedFiles = append(fsStatus.PickedFiles, p)
|
|
fsStatusWG.Done()
|
|
return
|
|
}
|
|
|
|
if known != nil {
|
|
newFileObject, err = CreateFileObject(p)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
knownFileObject, err = FromBytes(known)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
err = newFileObject.IsDifferent(knownFileObject)
|
|
if err != nil {
|
|
fsStatusWG.Wait()
|
|
fsStatusWG.Add(1)
|
|
fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p)
|
|
fsStatusWG.Done()
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
fsStatusWG.Wait()
|
|
fsStatusWG.Add(1)
|
|
fsStatus.NewFiles = append(fsStatus.NewFiles, p)
|
|
fsStatusWG.Done()
|
|
}
|
|
|
|
func GetFilesystemDiff(root string) (FilesystemStatus, error) {
|
|
var (
|
|
fsStatus FilesystemStatus = FilesystemStatus{}
|
|
sem *semaphore.Weighted
|
|
picksBucket *bolt.Bucket
|
|
indexBucket *bolt.Bucket
|
|
rootStat os.FileInfo
|
|
bar *mpb.Bar
|
|
fsCount int
|
|
poolSize int
|
|
err error
|
|
)
|
|
|
|
poolSize = runtime.NumCPU()
|
|
sem = semaphore.NewWeighted(int64(poolSize))
|
|
|
|
rootStat, err = os.Stat(root)
|
|
if err != nil {
|
|
return fsStatus, err
|
|
}
|
|
|
|
if rootStat.IsDir() && root[len(root)-1:] != "/" {
|
|
root = root + "/"
|
|
}
|
|
|
|
fsCount, err = GetFilesystemLength(root)
|
|
if err != nil {
|
|
return fsStatus, err
|
|
}
|
|
|
|
bar = ProgressBar.InitBar("Scanning...", fsCount)
|
|
|
|
err = Database.FsDB.View(func(tx *bolt.Tx) error {
|
|
|
|
picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
|
|
indexBucket = tx.Bucket(Variables.FsHashIndexBucket)
|
|
|
|
filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
|
|
|
|
// Ignore path in Variables.PruneRegexPaths
|
|
if i.IsDir() && matchAny(p, PruneRegex) {
|
|
return filepath.SkipDir
|
|
}
|
|
|
|
// Ignore path in Variables.IgnoreRegexPaths
|
|
if matchAny(p, IgnoreRegex) {
|
|
return nil
|
|
}
|
|
|
|
if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
|
|
return nil
|
|
}
|
|
|
|
FsWalkWG.Add(1)
|
|
sem.Acquire(context.Background(), 1)
|
|
go func() {
|
|
fsStatus.parseFile(indexBucket, picksBucket, p, bar)
|
|
sem.Release(1)
|
|
FsWalkWG.Done()
|
|
}()
|
|
|
|
FsWalkWG.Wait()
|
|
return nil
|
|
})
|
|
|
|
indexBucket.ForEach(func(k, v []byte) error {
|
|
_, err = os.Lstat(string(k))
|
|
if os.IsNotExist(err) {
|
|
fsStatus.MissingFiles = append(fsStatus.MissingFiles, string(k))
|
|
}
|
|
return nil
|
|
})
|
|
|
|
return nil
|
|
})
|
|
|
|
return fsStatus, err
|
|
}
|