Skip to content

Commit

Permalink
🎨 Use local file system sync & backup
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zuoqiu-Yingyi
Zuoqiu-Yingyi committed Dec 29, 2024
1 parent 245986d commit 0b698ee
Show file tree
Hide file tree
Showing 8 changed files with 102 additions and 24 deletions.
10 changes: 10 additions & 0 deletions kernel/conf/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type Sync struct {
Provider int `json:"provider"` // 云端存储服务提供者
S3 *S3 `json:"s3"` // S3 对象存储服务配置
WebDAV *WebDAV `json:"webdav"` // WebDAV 服务配置
Local *Local `json:"local"` // 本地文件系统 服务配置
}

func NewSync() *Sync {
Expand Down Expand Up @@ -63,10 +64,17 @@ type WebDAV struct {
ConcurrentReqs int `json:"concurrentReqs"` // 并发请求数
}

type Local struct {
Endpoint string `json:"endpoint"` // 服务端点 (本地文件系统目录)
Timeout int `json:"timeout"` // 超时时间，单位：秒
ConcurrentReqs int `json:"concurrentReqs"` // 并发请求数
}

const (
ProviderSiYuan = 0 // ProviderSiYuan 为思源官方提供的云端存储服务
ProviderS3 = 2 // ProviderS3 为 S3 协议对象存储提供的云端存储服务
ProviderWebDAV = 3 // ProviderWebDAV 为 WebDAV 协议提供的云端存储服务
ProviderLocal = 4 // ProviderLocal 为本地文件系统提供的存储服务
)

func ProviderToStr(provider int) string {
Expand All @@ -77,6 +85,8 @@ func ProviderToStr(provider int) string {
return "S3"
case ProviderWebDAV:
return "WebDAV"
case ProviderLocal:
return "Local File System"
}
return "Unknown"
}
5 changes: 4 additions & 1 deletion kernel/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ require (
golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b
golang.org/x/mod v0.22.0
golang.org/x/net v0.33.0
golang.org/x/sys v0.28.0
golang.org/x/text v0.21.0
golang.org/x/time v0.6.0
gopkg.in/yaml.v3 v3.0.1
Expand Down Expand Up @@ -166,6 +167,7 @@ require (
github.com/restic/chunker v0.4.0 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.3 // indirect
github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
Expand All @@ -186,7 +188,6 @@ require (
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/tools v0.28.0 // indirect
google.golang.org/protobuf v1.35.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand All @@ -196,6 +197,8 @@ replace github.com/mattn/go-sqlite3 => github.com/88250/go-sqlite3 v1.14.13-0.20

replace github.com/pdfcpu/pdfcpu => github.com/88250/pdfcpu v0.3.14-0.20241201033812-5a93b7586a01

replace github.com/siyuan-note/dejavu => D:\Sync\GitHub\dejavu

//replace github.com/88250/lute => F:\golang\gopath\src\github.com\88250\lute
//replace github.com/siyuan-note/dejavu => D:\88250\dejavu
//replace github.com/siyuan-note/riff => D:\88250\riff
Expand Down
4 changes: 2 additions & 2 deletions kernel/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,8 @@ github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM=
github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 h1:d54EL9l+XteliUfUCGsEwwuk65dmmxX85VXF+9T6+50=
github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285/go.mod h1:fxIDly1xtudczrZeOOlfaUvd2OPb2qZAPuWdU2BsBTk=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
Expand All @@ -365,8 +367,6 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
github.com/siyuan-note/dejavu v0.0.0-20241225015029-d441b991d87e h1:VOdd8vUFzMiT9NsuezgczDyxStkaCLAiRrUlKx+IMc8=
github.com/siyuan-note/dejavu v0.0.0-20241225015029-d441b991d87e/go.mod h1:+gaHpNFsoVagw956l456Cm4eZclB/gXsjkW4YmvWmJM=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=
Expand Down
7 changes: 7 additions & 0 deletions kernel/model/conf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,13 @@ func InitConf() {
Conf.Sync.WebDAV.Endpoint = util.NormalizeEndpoint(Conf.Sync.WebDAV.Endpoint)
Conf.Sync.WebDAV.Timeout = util.NormalizeTimeout(Conf.Sync.WebDAV.Timeout)
Conf.Sync.WebDAV.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.WebDAV.ConcurrentReqs, conf.ProviderWebDAV)
if nil == Conf.Sync.Local {
Conf.Sync.Local = &conf.Local{}
}
Conf.Sync.Local.Endpoint = util.NormalizeLocalPath(Conf.Sync.Local.Endpoint)
Conf.Sync.Local.Timeout = util.NormalizeTimeout(Conf.Sync.Local.Timeout)
Conf.Sync.Local.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.Local.ConcurrentReqs, conf.ProviderLocal)

if util.ContainerDocker == util.Container {
Conf.Sync.Perception = false
}
Expand Down
18 changes: 13 additions & 5 deletions kernel/model/repository.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -803,7 +803,7 @@ func DownloadCloudSnapshot(tag, id string) (err error) {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3:
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
Expand Down Expand Up @@ -845,7 +845,7 @@ func UploadCloudSnapshot(tag, id string) (err error) {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3:
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
Expand Down Expand Up @@ -891,7 +891,7 @@ func RemoveCloudRepoTag(tag string) (err error) {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3:
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
Expand Down Expand Up @@ -923,7 +923,7 @@ func GetCloudRepoTagSnapshots() (ret []*dejavu.Log, err error) {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3:
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
Expand Down Expand Up @@ -959,7 +959,7 @@ func GetCloudRepoSnapshots(page int) (ret []*dejavu.Log, pageCount, totalCount i
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3:
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
Expand Down Expand Up @@ -1845,6 +1845,8 @@ func newRepository() (ret *dejavu.Repo, err error) {
webdavClient.SetTimeout(time.Duration(cloudConf.WebDAV.Timeout) * time.Second)
webdavClient.SetTransport(httpclient.NewTransport(cloudConf.WebDAV.SkipTlsVerify))
cloudRepo = cloud.NewWebDAV(&cloud.BaseCloud{Conf: cloudConf}, webdavClient)
case conf.ProviderLocal:
cloudRepo = cloud.NewLocal(&cloud.BaseCloud{Conf: cloudConf})
default:
err = fmt.Errorf("unknown cloud provider [%d]", Conf.Sync.Provider)
return
Expand Down Expand Up @@ -2129,6 +2131,12 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
Timeout: Conf.Sync.WebDAV.Timeout,
ConcurrentReqs: Conf.Sync.WebDAV.ConcurrentReqs,
}
case conf.ProviderLocal:
ret.Local = &cloud.ConfLocal{
Endpoint: Conf.Sync.Local.Endpoint,
Timeout: Conf.Sync.Local.Timeout,
ConcurrentReqs: Conf.Sync.Local.ConcurrentReqs,
}
default:
err = fmt.Errorf("invalid provider [%d]", Conf.Sync.Provider)
return
Expand Down
15 changes: 12 additions & 3 deletions kernel/model/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ func checkSync(boot, exit, byHand bool) bool {
if !IsSubscriber() {
return false
}
case conf.ProviderWebDAV, conf.ProviderS3:
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
return false
}
Expand Down Expand Up @@ -477,7 +477,10 @@ var (
)

func CreateCloudSyncDir(name string) (err error) {
if conf.ProviderSiYuan != Conf.Sync.Provider {
switch Conf.Sync.Provider {
case conf.ProviderSiYuan, conf.ProviderLocal:
break
default:
err = errors.New(Conf.Language(131))
return
}
Expand All @@ -502,7 +505,10 @@ func CreateCloudSyncDir(name string) (err error) {
}

func RemoveCloudSyncDir(name string) (err error) {
if conf.ProviderSiYuan != Conf.Sync.Provider {
switch Conf.Sync.Provider {
case conf.ProviderSiYuan, conf.ProviderLocal:
break
default:
err = errors.New(Conf.Language(131))
return
}
Expand Down Expand Up @@ -678,6 +684,9 @@ func isProviderOnline(byHand bool) (ret bool) {
checkURL = Conf.Sync.WebDAV.Endpoint
skipTlsVerify = Conf.Sync.WebDAV.SkipTlsVerify
timeout = Conf.Sync.WebDAV.Timeout * 1000
case conf.ProviderLocal:
checkURL = "file://" + Conf.Sync.Local.Endpoint
timeout = Conf.Sync.Local.Timeout * 1000
default:
logging.LogWarnf("unknown provider: %d", Conf.Sync.Provider)
return false
Expand Down
14 changes: 10 additions & 4 deletions kernel/util/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"net"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -73,15 +74,20 @@ func IsLocalOrigin(origin string) bool {
}

func IsOnline(checkURL string, skipTlsVerify bool, timeout int) bool {
_, err := url.Parse(checkURL)
if err != nil {
logging.LogWarnf("invalid check URL [%s]", checkURL)
if "" == checkURL {
return false
}

if "" == checkURL {
u, err := url.Parse(checkURL)
if err != nil {
logging.LogWarnf("invalid check URL [%s]", checkURL)
return false
}
if u.Scheme == "file" {
filePath := strings.TrimPrefix(checkURL, "file://")
_, err := os.Stat(filePath)
return err == nil
}

if isOnline(checkURL, skipTlsVerify, timeout) {
return true
Expand Down
53 changes: 44 additions & 9 deletions kernel/util/path.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,16 +188,39 @@ func GetChildDocDepth(treeAbsPath string) (ret int) {
}

func NormalizeConcurrentReqs(concurrentReqs int, provider int) int {
if 1 > concurrentReqs {
if 2 == provider { // S3
return 8
} else if 3 == provider { // WebDAV
return 1
switch provider {
case 0: // SiYuan
switch {
case concurrentReqs < 1:
concurrentReqs = 8
case concurrentReqs > 16:
concurrentReqs = 16
default:
}
case 2: // S3
switch {
case concurrentReqs < 1:
concurrentReqs = 8
case concurrentReqs > 16:
concurrentReqs = 16
default:
}
case 3: // WebDAV
switch {
case concurrentReqs < 1:
concurrentReqs = 1
case concurrentReqs > 16:
concurrentReqs = 16
default:
}
case 4: // Local File System
switch {
case concurrentReqs < 1:
concurrentReqs = 16
case concurrentReqs > 1024:
concurrentReqs = 1024
default:
}
return 8
}
if 16 < concurrentReqs {
return 16
}
return concurrentReqs
}
Expand Down Expand Up @@ -229,6 +252,18 @@ func NormalizeEndpoint(endpoint string) string {
return endpoint
}

func NormalizeLocalPath(endpoint string) string {
endpoint = strings.TrimSpace(endpoint)
if "" == endpoint {
return ""
}
endpoint = filepath.ToSlash(filepath.Clean(endpoint))
if !strings.HasSuffix(endpoint, "/") {
endpoint = endpoint + "/"
}
return endpoint
}

func FilterMoveDocFromPaths(fromPaths []string, toPath string) (ret []string) {
tmp := FilterSelfChildDocs(fromPaths)
for _, fromPath := range tmp {
Expand Down

0 comments on commit 0b698ee

Please sign in to comment.