You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
227 lines
4.2 KiB
227 lines
4.2 KiB
package Filesystem
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sync"
|
|
|
|
"github.com/vbauerster/mpb"
|
|
bolt "go.etcd.io/bbolt"
|
|
"golang.org/x/sync/semaphore"
|
|
|
|
"PackageManager/Client/Database"
|
|
"PackageManager/Client/ProgressBar"
|
|
"PackageManager/Color"
|
|
"PackageManager/Variables"
|
|
)
|
|
|
|
var (
|
|
fsStatusWG sync.WaitGroup
|
|
FsWalkWG sync.WaitGroup
|
|
)
|
|
|
|
type FilesystemStatus struct {
|
|
NewFiles []string
|
|
PickedFiles []string
|
|
ModifiedFiles []string
|
|
MissingFiles []string
|
|
}
|
|
|
|
func ShowFilesystemDiff(root string) error {
|
|
var (
|
|
fsStatus FilesystemStatus
|
|
//f string
|
|
e error
|
|
)
|
|
|
|
fsStatus, e = GetFilesystemDiff(root)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
|
|
fmt.Println("New files:")
|
|
PrintFilesOrLength(fsStatus.NewFiles, Color.Green)
|
|
|
|
fmt.Println("Added files:")
|
|
PrintFilesOrLength(fsStatus.PickedFiles, Color.Green)
|
|
|
|
fmt.Println("Modified files:")
|
|
PrintFilesOrLength(fsStatus.ModifiedFiles, Color.Warning)
|
|
|
|
fmt.Println("Deleted files:")
|
|
PrintFilesOrLength(fsStatus.MissingFiles, Color.Fatal)
|
|
|
|
return nil
|
|
}
|
|
|
|
func GetFilesystemLength(root string) (int, error) {
|
|
var (
|
|
rootStat os.FileInfo
|
|
fsCount int = 0
|
|
e error
|
|
)
|
|
|
|
rootStat, e = os.Stat(root)
|
|
if e != nil {
|
|
return fsCount, e
|
|
}
|
|
|
|
if rootStat.IsDir() && root[len(root)-1:] != "/" {
|
|
root = root + "/"
|
|
}
|
|
|
|
filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
|
|
|
|
// Ignore path in Variables.PruneRegexPaths
|
|
if i.IsDir() && matchAny(p, PruneRegex) {
|
|
return filepath.SkipDir
|
|
}
|
|
|
|
// Ignore path in Variables.IgnoreRegexPaths
|
|
if matchAny(p, IgnoreRegex) {
|
|
return nil
|
|
}
|
|
|
|
if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
|
|
return nil
|
|
}
|
|
|
|
fsCount++
|
|
|
|
return nil
|
|
})
|
|
|
|
return fsCount, e
|
|
}
|
|
|
|
func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) {
|
|
var (
|
|
newFileObject FileObject
|
|
knownFileObject FileObject
|
|
pick, known []byte
|
|
e error
|
|
)
|
|
|
|
defer func() {
|
|
bar.Increment()
|
|
}()
|
|
|
|
pick = picksBucket.Get([]byte(p))
|
|
known = indexBucket.Get([]byte(p))
|
|
|
|
if pick != nil {
|
|
fsStatusWG.Wait()
|
|
fsStatusWG.Add(1)
|
|
fsStatus.PickedFiles = append(fsStatus.PickedFiles, p)
|
|
fsStatusWG.Done()
|
|
return
|
|
}
|
|
|
|
if known != nil {
|
|
newFileObject, e = CreateFileObject(p)
|
|
if e != nil {
|
|
return
|
|
}
|
|
|
|
knownFileObject, e = FromBytes(known)
|
|
if e != nil {
|
|
return
|
|
}
|
|
|
|
e = newFileObject.IsDifferent(knownFileObject)
|
|
if e != nil {
|
|
fsStatusWG.Wait()
|
|
fsStatusWG.Add(1)
|
|
fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p)
|
|
fsStatusWG.Done()
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
fsStatus.NewFiles = append(fsStatus.NewFiles, p)
|
|
|
|
return
|
|
}
|
|
|
|
func GetFilesystemDiff(root string) (FilesystemStatus, error) {
|
|
var (
|
|
fsStatus FilesystemStatus = FilesystemStatus{}
|
|
sem *semaphore.Weighted
|
|
picksBucket *bolt.Bucket
|
|
indexBucket *bolt.Bucket
|
|
rootStat os.FileInfo
|
|
bar *mpb.Bar
|
|
fsCount int
|
|
poolSize int
|
|
e error
|
|
)
|
|
|
|
poolSize = runtime.NumCPU()
|
|
sem = semaphore.NewWeighted(int64(poolSize))
|
|
|
|
rootStat, e = os.Stat(root)
|
|
if e != nil {
|
|
return fsStatus, e
|
|
}
|
|
|
|
if rootStat.IsDir() && root[len(root)-1:] != "/" {
|
|
root = root + "/"
|
|
}
|
|
|
|
fsCount, e = GetFilesystemLength(root)
|
|
if e != nil {
|
|
return fsStatus, e
|
|
}
|
|
|
|
bar = ProgressBar.InitBar("Scanning...", fsCount)
|
|
|
|
e = Database.FsDB.View(func(tx *bolt.Tx) error {
|
|
|
|
picksBucket = tx.Bucket(Variables.FsHashPicksBucket)
|
|
indexBucket = tx.Bucket(Variables.FsHashIndexBucket)
|
|
|
|
filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
|
|
|
|
// Ignore path in Variables.PruneRegexPaths
|
|
if i.IsDir() && matchAny(p, PruneRegex) {
|
|
return filepath.SkipDir
|
|
}
|
|
|
|
// Ignore path in Variables.IgnoreRegexPaths
|
|
if matchAny(p, IgnoreRegex) {
|
|
return nil
|
|
}
|
|
|
|
if !i.Mode().IsRegular() && (i.Mode()&os.ModeSymlink == 0) {
|
|
return nil
|
|
}
|
|
|
|
FsWalkWG.Add(1)
|
|
sem.Acquire(context.Background(), 1)
|
|
go func() {
|
|
fsStatus.parseFile(indexBucket, picksBucket, p, bar)
|
|
sem.Release(1)
|
|
FsWalkWG.Done()
|
|
}()
|
|
|
|
FsWalkWG.Wait()
|
|
return nil
|
|
})
|
|
|
|
indexBucket.ForEach(func(k, v []byte) error {
|
|
_, e = os.Lstat(string(k))
|
|
if os.IsNotExist(e) {
|
|
fsStatus.MissingFiles = append(fsStatus.MissingFiles, string(k))
|
|
}
|
|
return nil
|
|
})
|
|
|
|
return nil
|
|
})
|
|
|
|
return fsStatus, e
|
|
}
|