From 0b698ee1178e768c4091dbee1fba583c24e5d1ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A2=96=E9=80=B8?= <49649786+Zuoqiu-Yingyi@users.noreply.github.com> Date: Mon, 30 Dec 2024 02:49:15 +0800 Subject: [PATCH] :art: Use local file system sync & backup --- kernel/conf/sync.go | 10 +++++++ kernel/go.mod | 5 +++- kernel/go.sum | 4 +-- kernel/model/conf.go | 7 +++++ kernel/model/repository.go | 18 +++++++++---- kernel/model/sync.go | 15 ++++++++--- kernel/util/net.go | 14 +++++++--- kernel/util/path.go | 53 +++++++++++++++++++++++++++++++------- 8 files changed, 102 insertions(+), 24 deletions(-) diff --git a/kernel/conf/sync.go b/kernel/conf/sync.go index 488cb9e5701..1f23e9642ca 100644 --- a/kernel/conf/sync.go +++ b/kernel/conf/sync.go @@ -28,6 +28,7 @@ type Sync struct { Provider int `json:"provider"` // 云端存储服务提供者 S3 *S3 `json:"s3"` // S3 对象存储服务配置 WebDAV *WebDAV `json:"webdav"` // WebDAV 服务配置 + Local *Local `json:"local"` // 本地文件系统 服务配置 } func NewSync() *Sync { @@ -63,10 +64,17 @@ type WebDAV struct { ConcurrentReqs int `json:"concurrentReqs"` // 并发请求数 } +type Local struct { + Endpoint string `json:"endpoint"` // 服务端点 (本地文件系统目录) + Timeout int `json:"timeout"` // 超时时间，单位：秒 + ConcurrentReqs int `json:"concurrentReqs"` // 并发请求数 +} + const ( ProviderSiYuan = 0 // ProviderSiYuan 为思源官方提供的云端存储服务 ProviderS3 = 2 // ProviderS3 为 S3 协议对象存储提供的云端存储服务 ProviderWebDAV = 3 // ProviderWebDAV 为 WebDAV 协议提供的云端存储服务 + ProviderLocal = 4 // ProviderLocal 为本地文件系统提供的存储服务 ) func ProviderToStr(provider int) string { @@ -77,6 +85,8 @@ func ProviderToStr(provider int) string { return "S3" case ProviderWebDAV: return "WebDAV" + case ProviderLocal: + return "Local File System" } return "Unknown" } diff --git a/kernel/go.mod b/kernel/go.mod index aec53cb0cb5..797c340cdeb 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -74,6 +74,7 @@ require ( golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b golang.org/x/mod v0.22.0 golang.org/x/net v0.33.0 + golang.org/x/sys v0.28.0 golang.org/x/text v0.21.0 golang.org/x/time v0.6.0 gopkg.in/yaml.v3 v3.0.1 @@ -166,6 +167,7 @@ require ( github.com/restic/chunker v0.4.0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect + github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.4.0 // indirect @@ -186,7 +188,6 @@ require ( golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect golang.org/x/tools v0.28.0 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -196,6 +197,8 @@ replace github.com/mattn/go-sqlite3 => github.com/88250/go-sqlite3 v1.14.13-0.20 replace github.com/pdfcpu/pdfcpu => github.com/88250/pdfcpu v0.3.14-0.20241201033812-5a93b7586a01 +replace github.com/siyuan-note/dejavu => D:\Sync\GitHub\dejavu + //replace github.com/88250/lute => F:\golang\gopath\src\github.com\88250\lute //replace github.com/siyuan-note/dejavu => D:\88250\dejavu //replace github.com/siyuan-note/riff => D:\88250\riff diff --git a/kernel/go.sum b/kernel/go.sum index 97ed7ab2aba..649d950b939 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -339,6 +339,8 @@ github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7 github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM= github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 h1:d54EL9l+XteliUfUCGsEwwuk65dmmxX85VXF+9T6+50= +github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285/go.mod h1:fxIDly1xtudczrZeOOlfaUvd2OPb2qZAPuWdU2BsBTk= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -365,8 +367,6 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA= github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= -github.com/siyuan-note/dejavu v0.0.0-20241225015029-d441b991d87e h1:VOdd8vUFzMiT9NsuezgczDyxStkaCLAiRrUlKx+IMc8= -github.com/siyuan-note/dejavu v0.0.0-20241225015029-d441b991d87e/go.mod h1:+gaHpNFsoVagw956l456Cm4eZclB/gXsjkW4YmvWmJM= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis= github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8= diff --git a/kernel/model/conf.go b/kernel/model/conf.go index 1b2eae6691a..d80d043fc1d 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -360,6 +360,13 @@ func InitConf() { Conf.Sync.WebDAV.Endpoint = util.NormalizeEndpoint(Conf.Sync.WebDAV.Endpoint) Conf.Sync.WebDAV.Timeout = util.NormalizeTimeout(Conf.Sync.WebDAV.Timeout) Conf.Sync.WebDAV.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.WebDAV.ConcurrentReqs, conf.ProviderWebDAV) + if nil == Conf.Sync.Local { + Conf.Sync.Local = &conf.Local{} + } + Conf.Sync.Local.Endpoint = util.NormalizeLocalPath(Conf.Sync.Local.Endpoint) + Conf.Sync.Local.Timeout = util.NormalizeTimeout(Conf.Sync.Local.Timeout) + Conf.Sync.Local.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.Local.ConcurrentReqs, conf.ProviderLocal) + if util.ContainerDocker == util.Container { Conf.Sync.Perception = false } diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 359bfc97fb6..47c9dac7074 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -803,7 +803,7 @@ func DownloadCloudSnapshot(tag, id string) (err error) { util.PushErrMsg(Conf.Language(29), 5000) return } - case conf.ProviderWebDAV, conf.ProviderS3: + case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal: if !IsPaidUser() { util.PushErrMsg(Conf.Language(214), 5000) return @@ -845,7 +845,7 @@ func UploadCloudSnapshot(tag, id string) (err error) { util.PushErrMsg(Conf.Language(29), 5000) return } - case conf.ProviderWebDAV, conf.ProviderS3: + case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal: if !IsPaidUser() { util.PushErrMsg(Conf.Language(214), 5000) return @@ -891,7 +891,7 @@ func RemoveCloudRepoTag(tag string) (err error) { util.PushErrMsg(Conf.Language(29), 5000) return } - case conf.ProviderWebDAV, conf.ProviderS3: + case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal: if !IsPaidUser() { util.PushErrMsg(Conf.Language(214), 5000) return @@ -923,7 +923,7 @@ func GetCloudRepoTagSnapshots() (ret []*dejavu.Log, err error) { util.PushErrMsg(Conf.Language(29), 5000) return } - case conf.ProviderWebDAV, conf.ProviderS3: + case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal: if !IsPaidUser() { util.PushErrMsg(Conf.Language(214), 5000) return @@ -959,7 +959,7 @@ func GetCloudRepoSnapshots(page int) (ret []*dejavu.Log, pageCount, totalCount i util.PushErrMsg(Conf.Language(29), 5000) return } - case conf.ProviderWebDAV, conf.ProviderS3: + case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal: if !IsPaidUser() { util.PushErrMsg(Conf.Language(214), 5000) return @@ -1845,6 +1845,8 @@ func newRepository() (ret *dejavu.Repo, err error) { webdavClient.SetTimeout(time.Duration(cloudConf.WebDAV.Timeout) * time.Second) webdavClient.SetTransport(httpclient.NewTransport(cloudConf.WebDAV.SkipTlsVerify)) cloudRepo = cloud.NewWebDAV(&cloud.BaseCloud{Conf: cloudConf}, webdavClient) + case conf.ProviderLocal: + cloudRepo = cloud.NewLocal(&cloud.BaseCloud{Conf: cloudConf}) default: err = fmt.Errorf("unknown cloud provider [%d]", Conf.Sync.Provider) return @@ -2129,6 +2131,12 @@ func buildCloudConf() (ret *cloud.Conf, err error) { Timeout: Conf.Sync.WebDAV.Timeout, ConcurrentReqs: Conf.Sync.WebDAV.ConcurrentReqs, } + case conf.ProviderLocal: + ret.Local = &cloud.ConfLocal{ + Endpoint: Conf.Sync.Local.Endpoint, + Timeout: Conf.Sync.Local.Timeout, + ConcurrentReqs: Conf.Sync.Local.ConcurrentReqs, + } default: err = fmt.Errorf("invalid provider [%d]", Conf.Sync.Provider) return diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 3a104342eec..cb6f9aa7bf2 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -252,7 +252,7 @@ func checkSync(boot, exit, byHand bool) bool { if !IsSubscriber() { return false } - case conf.ProviderWebDAV, conf.ProviderS3: + case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal: if !IsPaidUser() { return false } @@ -477,7 +477,10 @@ var ( ) func CreateCloudSyncDir(name string) (err error) { - if conf.ProviderSiYuan != Conf.Sync.Provider { + switch Conf.Sync.Provider { + case conf.ProviderSiYuan, conf.ProviderLocal: + break + default: err = errors.New(Conf.Language(131)) return } @@ -502,7 +505,10 @@ func CreateCloudSyncDir(name string) (err error) { } func RemoveCloudSyncDir(name string) (err error) { - if conf.ProviderSiYuan != Conf.Sync.Provider { + switch Conf.Sync.Provider { + case conf.ProviderSiYuan, conf.ProviderLocal: + break + default: err = errors.New(Conf.Language(131)) return } @@ -678,6 +684,9 @@ func isProviderOnline(byHand bool) (ret bool) { checkURL = Conf.Sync.WebDAV.Endpoint skipTlsVerify = Conf.Sync.WebDAV.SkipTlsVerify timeout = Conf.Sync.WebDAV.Timeout * 1000 + case conf.ProviderLocal: + checkURL = "file://" + Conf.Sync.Local.Endpoint + timeout = Conf.Sync.Local.Timeout * 1000 default: logging.LogWarnf("unknown provider: %d", Conf.Sync.Provider) return false diff --git a/kernel/util/net.go b/kernel/util/net.go index 60b8a2fe9d0..1b49a83df35 100644 --- a/kernel/util/net.go +++ b/kernel/util/net.go @@ -20,6 +20,7 @@ import ( "net" "net/http" "net/url" + "os" "strconv" "strings" "time" @@ -73,15 +74,20 @@ func IsLocalOrigin(origin string) bool { } func IsOnline(checkURL string, skipTlsVerify bool, timeout int) bool { - _, err := url.Parse(checkURL) - if err != nil { - logging.LogWarnf("invalid check URL [%s]", checkURL) + if "" == checkURL { return false } - if "" == checkURL { + u, err := url.Parse(checkURL) + if err != nil { + logging.LogWarnf("invalid check URL [%s]", checkURL) return false } + if u.Scheme == "file" { + filePath := strings.TrimPrefix(checkURL, "file://") + _, err := os.Stat(filePath) + return err == nil + } if isOnline(checkURL, skipTlsVerify, timeout) { return true diff --git a/kernel/util/path.go b/kernel/util/path.go index ed562a514d5..b9f4f9f7b47 100644 --- a/kernel/util/path.go +++ b/kernel/util/path.go @@ -188,16 +188,39 @@ func GetChildDocDepth(treeAbsPath string) (ret int) { } func NormalizeConcurrentReqs(concurrentReqs int, provider int) int { - if 1 > concurrentReqs { - if 2 == provider { // S3 - return 8 - } else if 3 == provider { // WebDAV - return 1 + switch provider { + case 0: // SiYuan + switch { + case concurrentReqs < 1: + concurrentReqs = 8 + case concurrentReqs > 16: + concurrentReqs = 16 + default: + } + case 2: // S3 + switch { + case concurrentReqs < 1: + concurrentReqs = 8 + case concurrentReqs > 16: + concurrentReqs = 16 + default: + } + case 3: // WebDAV + switch { + case concurrentReqs < 1: + concurrentReqs = 1 + case concurrentReqs > 16: + concurrentReqs = 16 + default: + } + case 4: // Local File System + switch { + case concurrentReqs < 1: + concurrentReqs = 16 + case concurrentReqs > 1024: + concurrentReqs = 1024 + default: } - return 8 - } - if 16 < concurrentReqs { - return 16 } return concurrentReqs } @@ -229,6 +252,18 @@ func NormalizeEndpoint(endpoint string) string { return endpoint } +func NormalizeLocalPath(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if "" == endpoint { + return "" + } + endpoint = filepath.ToSlash(filepath.Clean(endpoint)) + if !strings.HasSuffix(endpoint, "/") { + endpoint = endpoint + "/" + } + return endpoint +} + func FilterMoveDocFromPaths(fromPaths []string, toPath string) (ret []string) { tmp := FilterSelfChildDocs(fromPaths) for _, fromPath := range tmp {