Skip to content

Commit f27ebe1

Browse files
aFlyBird0aFlyBird0
committed
fix: lock error(and commit code written half a year ago which is not tested)
1 parent 90c27aa commit f27ebe1

18 files changed

+111
-68
lines changed

‎.goreleaser.yaml

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -119,12 +119,7 @@ builds:
119119
# - docker.io/{{ .Env.USERNAME }}/cubox-archiver:latest-darwin-arm64
120120
# - docker.io/{{ .Env.USERNAME }}/cubox-archiver:latest-darwin-amd64
121121
archives:
122-
- replacements:
123-
darwin: Darwin
124-
linux: Linux
125-
windows: Windows
126-
386: i386
127-
amd64: x86_64
122+
- name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
128123
format: binary # 不打包成 tar.gz，直接上传二进制文件
129124
checksum:
130125
name_template: 'checksums.txt'

‎core/archiver.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package core
22

3+
// Archiver 归档器，能够提供原有的数据的Keys，并且能够操作数据
34
type Archiver interface {
45
Operator
56
KeysInitiator

‎core/deduplicate.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package core
22

33
import (
44
"sync"
5-
6-
"github.com/aFlyBird0/cubox-archiver/core/cubox"
75
)
86

97
type Deduplicate struct {
@@ -38,10 +36,11 @@ func NewDeduplicateWithKeysInitiator(initiator KeysInitiator) (*Deduplicate, err
3836
return NewDeduplicateWithKeys(keys), nil
3937
}
4038

41-
func (d *Deduplicate) Remain(item *cubox.Item) bool {
39+
func (d *Deduplicate) Remain(item *Item) bool {
4240
key := item.UserSearchEngineID
4341
d.mu.RLock()
4442
if _, ok := d.keys[key]; ok {
43+
d.mu.RUnlock()
4544
return false
4645
}
4746
d.mu.RUnlock()

‎core/filter.go

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
11
package core
22

3-
import (
4-
"github.com/aFlyBird0/cubox-archiver/core/cubox"
5-
)
6-
73
type Filter interface {
8-
Remain(item *cubox.Item) bool
4+
Remain(item *Item) bool
95
}
106

7+
// KeysInitiator 能够获取已有的数据的Keys
118
type KeysInitiator interface {
129
ExistingKeys() (map[string]struct{}, error) // 获取已有的数据的Keys
1310
}
@@ -20,7 +17,7 @@ func NewFilterAllRemain() *AllRemain {
2017
type AllRemain struct {
2118
}
2219

23-
func (f *AllRemain) Remain(item *cubox.Item) bool {
20+
func (f *AllRemain) Remain(item *Item) bool {
2421
return true
2522
}
2623

@@ -33,7 +30,7 @@ func NewFilterFirstN(n int) *FirstN {
3330
return &FirstN{n: n}
3431
}
3532

36-
func (f *FirstN) Remain(item *cubox.Item) bool {
33+
func (f *FirstN) Remain(item *Item) bool {
3734
if f.n <= 0 {
3835
return false
3936
}

‎core/operator.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,11 @@ package core
33
import (
44
"github.com/reugn/go-streams"
55
"github.com/sirupsen/logrus"
6-
7-
"github.com/aFlyBird0/cubox-archiver/core/cubox"
86
)
97

8+
// Operator 能对 Cubox Item 进行处理
109
type Operator interface {
11-
Operate(item *cubox.Item) error // 处理单个数据
10+
Operate(item *Item) error // 处理单个数据
1211
}
1312

1413
type OperatorChain struct {
@@ -20,7 +19,7 @@ func NewOperatorChain(operators ...Operator) *OperatorChain {
2019
return &OperatorChain{operators: operators}
2120
}
2221

23-
func (c *OperatorChain) Operate(item *cubox.Item) error {
22+
func (c *OperatorChain) Operate(item *Item) error {
2423
for _, operator := range c.operators {
2524
err := operator.Operate(item)
2625
if err != nil {
@@ -49,7 +48,7 @@ func NewOperatorSink(operator Operator, done chan<- struct{}) streams.Sink {
4948
// 处理主逻辑
5049
func (o *OperatorSink) init(done chan<- struct{}) {
5150
for itemAny := range o.in {
52-
item := itemAny.(*cubox.Item)
51+
item := itemAny.(*Item)
5352
err := o.operator.Operate(item)
5453
if err != nil {
5554
logrus.Errorf("operate item: %v, err: %v", item, err)

‎core/cubox/interface.go renamed to ‎core/source.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
package cubox
1+
package core
22

3+
// Source 能够提供 Cubox Item 的数据源
34
type Source interface {
45
List(items chan *Item) // 需要该方法主动关闭 chan
56
}

‎core/cubox/type.go renamed to ‎core/type.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package cubox
1+
package core
22

33
import (
44
"fmt"

‎impl/csv/operator.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"encoding/csv"
55
"os"
66

7-
"github.com/aFlyBird0/cubox-archiver/core/cubox"
7+
"github.com/aFlyBird0/cubox-archiver/core"
88
)
99

1010
type Operator struct {
@@ -15,7 +15,7 @@ func NewCsvOperator(filename string) *Operator {
1515
return &Operator{filename: filename}
1616
}
1717

18-
func (o *Operator) Operate(item *cubox.Item) error {
18+
func (o *Operator) Operate(item *core.Item) error {
1919
file, err := os.OpenFile(o.filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
2020
if err != nil {
2121
return err

‎impl/cubox/archivedCubox.go

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/parnurzeal/gorequest"
1111
"github.com/sirupsen/logrus"
1212

13-
"github.com/aFlyBird0/cubox-archiver/core/cubox"
13+
"github.com/aFlyBird0/cubox-archiver/core"
1414
"github.com/aFlyBird0/cubox-archiver/util"
1515
)
1616

@@ -23,9 +23,9 @@ type ArchivedCuboxSource struct {
2323
cookie string
2424
}
2525

26-
var _ = cubox.Source(&ArchivedCuboxSource{})
26+
var _ = core.Source(&ArchivedCuboxSource{})
2727

28-
func (client *ArchivedCuboxSource) List(cuboxChan chan *cubox.Item) {
28+
func (client *ArchivedCuboxSource) List(cuboxChan chan *core.Item) {
2929
const archiving = true
3030
// 先请求第一页试试
3131
items, pageCount, totalCounts := client.requestCubox(archiving, 1, "")
@@ -45,7 +45,7 @@ func (client *ArchivedCuboxSource) List(cuboxChan chan *cubox.Item) {
4545
}
4646
}
4747

48-
func (client *ArchivedCuboxSource) requestCubox(archiving bool, page int, lastBookmarkId string) (res []*cubox.Item, pageCount, totalCount int) {
48+
func (client *ArchivedCuboxSource) requestCubox(archiving bool, page int, lastBookmarkId string) (res []*core.Item, pageCount, totalCount int) {
4949
const url = "https://cubox.pro/c/api/v2/search_engine/my"
5050

5151
dataResp := cuboxItemResponse{}
@@ -67,7 +67,7 @@ func (client *ArchivedCuboxSource) requestCubox(archiving bool, page int, lastBo
6767
logrus.Fatalln(fmt.Sprintf("failed to request cubox content, http code: %v, body: %v", httpResp.StatusCode, body))
6868
}
6969

70-
res = make([]*cubox.Item, 0, len(dataResp.Data))
70+
res = make([]*core.Item, 0, len(dataResp.Data))
7171

7272
for _, itemRaw := range dataResp.Data {
7373
if itemRaw != nil {
@@ -78,7 +78,7 @@ func (client *ArchivedCuboxSource) requestCubox(archiving bool, page int, lastBo
7878
return res, dataResp.PageCount, dataResp.TotalCounts
7979
}
8080

81-
func (client *ArchivedCuboxSource) handleNextPages(cuboxChan chan<- *cubox.Item, archiving bool, pageCount int, lastBookmarkId string) {
81+
func (client *ArchivedCuboxSource) handleNextPages(cuboxChan chan<- *core.Item, archiving bool, pageCount int, lastBookmarkId string) {
8282
for page := 2; page <= pageCount; page += 1 {
8383
time.Sleep(time.Second * 1)
8484
items, _, _ := client.requestCubox(archiving, page, lastBookmarkId)
@@ -93,8 +93,8 @@ func (client *ArchivedCuboxSource) handleNextPages(cuboxChan chan<- *cubox.Item,
9393
}
9494

9595
// 将 cuboxItemRaw 转换成 Item
96-
func (client *ArchivedCuboxSource) convertCuboxItem(raw *cuboxItemRaw) (item *cubox.Item) {
97-
item = &cubox.Item{}
96+
func (client *ArchivedCuboxSource) convertCuboxItem(raw *cuboxItemRaw) (item *core.Item) {
97+
item = &core.Item{}
9898
item.UserSearchEngineID = raw.UserSearchEngineID
9999
item.Title = raw.Title
100100
item.Description = raw.Description
@@ -110,7 +110,7 @@ func (client *ArchivedCuboxSource) convertCuboxItem(raw *cuboxItemRaw) (item *cu
110110
if err != nil {
111111
logrus.Errorf("convert time failed, err: %v", err)
112112
}
113-
item.Tags = append(item.Tags, cubox.Tag{TagID: tag.TagID, Name: tag.Name, Rank: tag.Rank, UpdateTime: updateTime, ParentId: tag.ParentId})
113+
item.Tags = append(item.Tags, core.Tag{TagID: tag.TagID, Name: tag.Name, Rank: tag.Rank, UpdateTime: updateTime, ParentId: tag.ParentId})
114114
}
115115
item.GroupId = raw.GroupId
116116
item.GroupName = raw.GroupName
@@ -127,7 +127,7 @@ func (client *ArchivedCuboxSource) convertCuboxItem(raw *cuboxItemRaw) (item *cu
127127
item.Status = raw.Status
128128
//item.Finished = raw.Finished
129129
//item.InBlackOrWhiteList = raw.InBlackOrWhiteList
130-
item.Type = cubox.CuboxContentType(raw.Type)
130+
item.Type = core.CuboxContentType(raw.Type)
131131

132132
// 把链接全部 encode 一下（因为Notion里的链接会被自动解码，导致去重失败）
133133
item.TargetURL, err = util.EncodeURL(item.TargetURL)
@@ -144,7 +144,7 @@ func (client *ArchivedCuboxSource) convertCuboxItem(raw *cuboxItemRaw) (item *cu
144144
}
145145

146146
// 单独处理文本和随手记类型的内容
147-
if item.Type == cubox.Text || item.Type == cubox.ShortHand {
147+
if item.Type == core.Text || item.Type == core.ShortHand {
148148
getContentURL := "https://cubox.pro/c/api/bookmark/content"
149149
request := gorequest.New().Get(getContentURL).Timeout(time.Second*10).
150150
Param("bookmarkId", item.UserSearchEngineID)

‎impl/cubox/deleteCuboxOperator.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"github.com/parnurzeal/gorequest"
55
"go.uber.org/multierr"
66

7-
"github.com/aFlyBird0/cubox-archiver/core/cubox"
7+
"github.com/aFlyBird0/cubox-archiver/core"
88
)
99

1010
type DeleteCuboxOperator struct {
@@ -16,7 +16,7 @@ func NewDeleteCuboxOperator(auth, cookie string) *DeleteCuboxOperator {
1616
return &DeleteCuboxOperator{auth: auth, cookie: cookie}
1717
}
1818

19-
func (o *DeleteCuboxOperator) Operate(item *cubox.Item) error {
19+
func (o *DeleteCuboxOperator) Operate(item *core.Item) error {
2020
url := "https://cubox.pro/c/api/search_engine/delete/" + item.UserSearchEngineID
2121

2222
req := gorequest.New().Post(url)

‎impl/db/archiver.go

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,36 @@ import (
44
"gorm.io/gorm"
55

66
"github.com/aFlyBird0/cubox-archiver/core"
7-
"github.com/aFlyBird0/cubox-archiver/core/cubox"
87
)
98

9+
const defaultTableName = "cubox_archiver"
10+
1011
type Archiver struct {
11-
db *gorm.DB
12+
db *gorm.DB
13+
tableName string
1214
}
1315

1416
type Item struct {
1517
//gorm.Model
16-
cubox.Item
17-
}
18-
19-
func (item *Item) TableName() string {
20-
// todo: set table name by config file
21-
return "cubox_archiver"
18+
core.Item
2219
}
2320

2421
// todo: generate different db client by config file
25-
func NewArchiver(db *gorm.DB) *Archiver {
26-
return &Archiver{db: db}
22+
func NewArchiver(db *gorm.DB, tableName string) *Archiver {
23+
if tableName == "" {
24+
tableName = defaultTableName
25+
}
26+
27+
return &Archiver{db: db, tableName: tableName}
2728
}
2829

29-
func (a *Archiver) Operate(item *cubox.Item) error {
30-
return a.db.Create(&Item{Item: *item}).Error
30+
func (a *Archiver) Operate(item *core.Item) error {
31+
return a.dbWithTableName().Create(&Item{Item: *item}).Error
3132
}
3233

3334
func (a *Archiver) ExistingKeys() (map[string]struct{}, error) {
3435
items := make([]Item, 0)
35-
if err := a.db.Select("user_search_engine_id").Find(&items).Error; err != nil {
36+
if err := a.dbWithTableName().Select("user_search_engine_id").Find(&items).Error; err != nil {
3637
return nil, err
3738
}
3839
keys := make(map[string]struct{})
@@ -42,4 +43,8 @@ func (a *Archiver) ExistingKeys() (map[string]struct{}, error) {
4243
return keys, nil
4344
}
4445

46+
func (a *Archiver) dbWithTableName() *gorm.DB {
47+
return a.db.Table(a.tableName)
48+
}
49+
4550
var _ core.Archiver = (*Archiver)(nil)

‎impl/db/source.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package db
2+
3+
import (
4+
"github.com/sirupsen/logrus"
5+
"gorm.io/gorm"
6+
7+
"github.com/aFlyBird0/cubox-archiver/core"
8+
)
9+
10+
type Source struct {
11+
db *gorm.DB
12+
tableName string
13+
}
14+
15+
type SourceItem struct {
16+
//gorm.Model
17+
core.Item
18+
}
19+
20+
func NewSource(db *gorm.DB, tableName string) *Source {
21+
if tableName == "" {
22+
tableName = defaultTableName
23+
}
24+
25+
return &Source{db: db, tableName: tableName}
26+
}
27+
28+
func (s *Source) List(items chan *core.Item) {
29+
defer close(items)
30+
sourceItems := make([]SourceItem, 0)
31+
// read from db per 1000 items
32+
for offset := 0; ; offset += 1000 {
33+
if err := s.dbWithTableName().Limit(1000).Offset(offset).Find(&sourceItems).Error; err != nil {
34+
logrus.Errorf("read from db: %v", err)
35+
break
36+
}
37+
if len(sourceItems) == 0 {
38+
break
39+
}
40+
for _, sourceItem := range sourceItems {
41+
items <- &sourceItem.Item
42+
}
43+
}
44+
}
45+
46+
func (s *Source) dbWithTableName() *gorm.DB {
47+
return s.db.Table(s.tableName)
48+
}
49+
50+
var _ core.Source = (*Source)(nil)

‎impl/notion/operator.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,12 @@ import (
88
"github.com/jomei/notionapi"
99
"github.com/sirupsen/logrus"
1010

11-
"github.com/aFlyBird0/cubox-archiver/core/cubox"
11+
"github.com/aFlyBird0/cubox-archiver/core"
1212
"github.com/aFlyBird0/cubox-archiver/util"
1313
)
1414

15-
func (o *Archiver) Operate(item *cubox.Item) error {
15+
func (o *Archiver) Operate(item *core.Item) error {
16+
//logrus.Infof("开始同步【%s】到Notion", item.Title)
1617
_, err := o.createNewPage(item)
1718
if err != nil {
1819
return fmt.Errorf("create new page: %w", err)
@@ -22,7 +23,7 @@ func (o *Archiver) Operate(item *cubox.Item) error {
2223
return nil
2324
}
2425

25-
func (o *Archiver) createNewPage(item *cubox.Item) (*notionapi.Page, error) {
26+
func (o *Archiver) createNewPage(item *core.Item) (*notionapi.Page, error) {
2627
nameText := notionapi.RichText{
2728
Text: &notionapi.Text{Content: item.Title},
2829
}

0 commit comments

Comments
 (0)