Skip to content

Commit a1da4f0

Browse files
conico974Nicolas Dorseuil
conico974
and
Nicolas Dorseuil
authored
Move DO to a 3rd worker (#3270)
Co-authored-by: Nicolas Dorseuil <nicolas@gitbook.io>
1 parent 666e842 commit a1da4f0

File tree

8 files changed

+315
-23
lines changed

8 files changed

+315
-23
lines changed

‎.github/composite/deploy-cloudflare/action.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,16 @@ runs:
6464
GITBOOK_RUNTIME: cloudflare
6565
shell: bash
6666

67+
- name: Upload the DO worker
68+
uses: cloudflare/wrangler-action@v3.14.0
69+
with:
70+
apiToken: ${{ inputs.apiToken }}
71+
accountId: ${{ inputs.accountId }}
72+
workingDirectory: ./
73+
wranglerVersion: '4.10.0'
74+
environment: ${{ inputs.environment }}
75+
command: deploy --config ./packages/gitbook-v2/openNext/customWorkers/doWrangler.jsonc
76+
6777
- id: upload_server
6878
name: Upload server to Cloudflare
6979
uses: cloudflare/wrangler-action@v3.14.0

‎packages/gitbook-v2/openNext/customWorkers/default.js

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,26 @@
11
import { runWithCloudflareRequestContext } from '../../.open-next/cloudflare/init.js';
22

3+
import { DurableObject } from 'cloudflare:workers';
4+
5+
// Only needed to run locally, in prod we'll use the one from do.js
6+
export class R2WriteBuffer extends DurableObject {
7+
writePromise;
8+
9+
async write(cacheKey, value) {
10+
// We are already writing to this key
11+
if (this.writePromise) {
12+
return;
13+
}
14+
15+
this.writePromise = this.env.NEXT_INC_CACHE_R2_BUCKET.put(cacheKey, value);
16+
this.ctx.waitUntil(
17+
this.writePromise.finally(() => {
18+
this.writePromise = undefined;
19+
})
20+
);
21+
}
22+
}
23+
324
export default {
425
async fetch(request, env, ctx) {
526
return runWithCloudflareRequestContext(request, env, ctx, async () => {

‎packages/gitbook-v2/openNext/customWorkers/defaultWrangler.jsonc

Lines changed: 56 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,20 @@
2929
"binding": "WORKER_SELF_REFERENCE",
3030
"service": "gitbook-open-v2-server-dev"
3131
}
32+
],
33+
"durable_objects": {
34+
"bindings": [
35+
{
36+
"name": "WRITE_BUFFER",
37+
"class_name": "R2WriteBuffer"
38+
}
39+
]
40+
},
41+
"migrations": [
42+
{
43+
"tag": "v1",
44+
"new_sqlite_classes": ["R2WriteBuffer"]
45+
}
3246
]
3347
},
3448
"preview": {
@@ -48,9 +62,26 @@
4862
"binding": "WORKER_SELF_REFERENCE",
4963
"service": "gitbook-open-v2-server-preview"
5064
}
51-
]
52-
// No durable objects on preview, as they block the generation of preview URLs
53-
// and we don't need tags invalidation on preview
65+
],
66+
"durable_objects": {
67+
"bindings": [
68+
{
69+
"name": "WRITE_BUFFER",
70+
"class_name": "R2WriteBuffer",
71+
"script_name": "gitbook-open-v2-do-preview"
72+
},
73+
{
74+
"name": "NEXT_TAG_CACHE_DO_SHARDED",
75+
"class_name": "DOShardedTagCache",
76+
"script_name": "gitbook-open-v2-do-preview"
77+
},
78+
{
79+
"name": "NEXT_CACHE_DO_QUEUE",
80+
"class_name": "DOQueueHandler",
81+
"script_name": "gitbook-open-v2-do-preview"
82+
}
83+
]
84+
}
5485
},
5586
"staging": {
5687
"r2_buckets": [
@@ -67,12 +98,20 @@
6798
],
6899
"durable_objects": {
69100
"bindings": [
70-
// We do not need to define migrations for external DOs,
71-
// In fact, defining migrations for external DOs will crash
101+
{
102+
"name": "WRITE_BUFFER",
103+
"class_name": "R2WriteBuffer",
104+
"script_name": "gitbook-open-v2-do-staging"
105+
},
72106
{
73107
"name": "NEXT_TAG_CACHE_DO_SHARDED",
74108
"class_name": "DOShardedTagCache",
75-
"script_name": "gitbook-open-v2-staging"
109+
"script_name": "gitbook-open-v2-do-staging"
110+
},
111+
{
112+
"name": "NEXT_CACHE_DO_QUEUE",
113+
"class_name": "DOQueueHandler",
114+
"script_name": "gitbook-open-v2-do-staging"
76115
}
77116
]
78117
},
@@ -98,11 +137,19 @@
98137
"durable_objects": {
99138
"bindings": [
100139
{
101-
// We do not need to define migrations for external DOs,
102-
// In fact, defining migrations for external DOs will crash
140+
"name": "WRITE_BUFFER",
141+
"class_name": "R2WriteBuffer",
142+
"script_name": "gitbook-open-v2-do-production"
143+
},
144+
{
103145
"name": "NEXT_TAG_CACHE_DO_SHARDED",
104146
"class_name": "DOShardedTagCache",
105-
"script_name": "gitbook-open-v2-production"
147+
"script_name": "gitbook-open-v2-do-production"
148+
},
149+
{
150+
"name": "NEXT_CACHE_DO_QUEUE",
151+
"class_name": "DOQueueHandler",
152+
"script_name": "gitbook-open-v2-do-production"
106153
}
107154
]
108155
},

‎packages/gitbook-v2/openNext/customWorkers/do.js

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// This worker only purposes it to host the different DO that we will need in the other workers.
2+
import { DurableObject } from 'cloudflare:workers';
3+
4+
// `use cache` could cause multiple writes to the same key to happen concurrently, there is a limit of 1 write per key/second
5+
// so we need to buffer writes to the R2 bucket to avoid hitting this limit.
6+
export class R2WriteBuffer extends DurableObject {
7+
writePromise;
8+
9+
async write(cacheKey, value) {
10+
// We are already writing to this key
11+
if (this.writePromise) {
12+
return;
13+
}
14+
15+
this.writePromise = this.env.NEXT_INC_CACHE_R2_BUCKET.put(cacheKey, value);
16+
this.ctx.waitUntil(
17+
this.writePromise.finally(() => {
18+
this.writePromise = undefined;
19+
})
20+
);
21+
}
22+
}
23+
24+
export { DOQueueHandler } from '../../.open-next/.build/durable-objects/queue.js';
25+
26+
export { DOShardedTagCache } from '../../.open-next/.build/durable-objects/sharded-tag-cache.js';
27+
28+
export default {
29+
async fetch() {
30+
// This worker does not handle any requests, it only provides Durable Objects
31+
return new Response('This worker is not meant to handle requests directly', {
32+
status: 400,
33+
headers: {
34+
'Content-Type': 'text/plain',
35+
},
36+
});
37+
},
38+
};

‎packages/gitbook-v2/openNext/customWorkers/doWrangler.jsonc

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
{
2+
"main": "do.js",
3+
"name": "gitbook-open-v2-do",
4+
"compatibility_date": "2025-04-14",
5+
"compatibility_flags": [
6+
"nodejs_compat",
7+
"allow_importable_env",
8+
"global_fetch_strictly_public"
9+
],
10+
"observability": {
11+
"enabled": true
12+
},
13+
"env": {
14+
"preview": {
15+
"vars": {
16+
"STAGE": "preview",
17+
"NEXT_CACHE_DO_QUEUE_DISABLE_SQLITE": "true"
18+
},
19+
"r2_buckets": [
20+
{
21+
"binding": "NEXT_INC_CACHE_R2_BUCKET",
22+
"bucket_name": "gitbook-open-v2-cache-preview"
23+
}
24+
],
25+
"durable_objects": {
26+
"bindings": [
27+
{
28+
"name": "NEXT_CACHE_DO_QUEUE",
29+
"class_name": "DOQueueHandler"
30+
},
31+
{
32+
"name": "NEXT_TAG_CACHE_DO_SHARDED",
33+
"class_name": "DOShardedTagCache"
34+
},
35+
{
36+
"name": "WRITE_BUFFER",
37+
"class_name": "R2WriteBuffer"
38+
}
39+
]
40+
},
41+
"migrations": [
42+
{
43+
"tag": "v1",
44+
"new_sqlite_classes": ["DOQueueHandler", "DOShardedTagCache", "R2WriteBuffer"]
45+
}
46+
]
47+
},
48+
"staging": {
49+
"vars": {
50+
"STAGE": "staging",
51+
"NEXT_CACHE_DO_QUEUE_DISABLE_SQLITE": "true"
52+
},
53+
"r2_buckets": [
54+
{
55+
"binding": "NEXT_INC_CACHE_R2_BUCKET",
56+
"bucket_name": "gitbook-open-v2-cache-staging"
57+
}
58+
],
59+
"tail_consumers": [
60+
{
61+
"service": "gitbook-x-staging-tail"
62+
}
63+
],
64+
"durable_objects": {
65+
"bindings": [
66+
{
67+
"name": "NEXT_CACHE_DO_QUEUE",
68+
"class_name": "DOQueueHandler"
69+
},
70+
{
71+
"name": "NEXT_TAG_CACHE_DO_SHARDED",
72+
"class_name": "DOShardedTagCache"
73+
},
74+
{
75+
"name": "WRITE_BUFFER",
76+
"class_name": "R2WriteBuffer"
77+
}
78+
]
79+
},
80+
"migrations": [
81+
{
82+
"tag": "v1",
83+
"new_sqlite_classes": ["DOQueueHandler", "DOShardedTagCache", "R2WriteBuffer"]
84+
}
85+
]
86+
},
87+
"production": {
88+
"vars": {
89+
"NEXT_CACHE_DO_QUEUE_DISABLE_SQLITE": "true",
90+
"STAGE": "production"
91+
},
92+
"r2_buckets": [
93+
{
94+
"binding": "NEXT_INC_CACHE_R2_BUCKET",
95+
"bucket_name": "gitbook-open-v2-cache-production"
96+
}
97+
],
98+
"tail_consumers": [
99+
{
100+
"service": "gitbook-x-prod-tail"
101+
}
102+
],
103+
"durable_objects": {
104+
"bindings": [
105+
{
106+
"name": "NEXT_CACHE_DO_QUEUE",
107+
"class_name": "DOQueueHandler"
108+
},
109+
{
110+
"name": "NEXT_TAG_CACHE_DO_SHARDED",
111+
"class_name": "DOShardedTagCache"
112+
},
113+
{
114+
"name": "WRITE_BUFFER",
115+
"class_name": "R2WriteBuffer"
116+
}
117+
]
118+
},
119+
"migrations": [
120+
{
121+
"tag": "v1",
122+
"new_sqlite_classes": ["DOQueueHandler", "DOShardedTagCache", "R2WriteBuffer"]
123+
}
124+
]
125+
}
126+
}
127+
}

‎packages/gitbook-v2/openNext/customWorkers/middlewareWrangler.jsonc

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,26 @@
6161
"binding": "DEFAULT_WORKER",
6262
"service": "gitbook-open-v2-server-preview"
6363
}
64-
]
65-
// No durable objects on preview, as they block the generation of preview URLs
66-
// and we don't need tags invalidation on preview
64+
],
65+
"durable_objects": {
66+
"bindings": [
67+
{
68+
"name": "WRITE_BUFFER",
69+
"class_name": "R2WriteBuffer",
70+
"script_name": "gitbook-open-v2-do-preview"
71+
},
72+
{
73+
"name": "NEXT_TAG_CACHE_DO_SHARDED",
74+
"class_name": "DOShardedTagCache",
75+
"script_name": "gitbook-open-v2-do-preview"
76+
},
77+
{
78+
"name": "NEXT_CACHE_DO_QUEUE",
79+
"class_name": "DOQueueHandler",
80+
"script_name": "gitbook-open-v2-do-preview"
81+
}
82+
]
83+
}
6784
},
6885
"staging": {
6986
"vars": {
@@ -104,12 +121,19 @@
104121
"durable_objects": {
105122
"bindings": [
106123
{
107-
"name": "NEXT_CACHE_DO_QUEUE",
108-
"class_name": "DOQueueHandler"
124+
"name": "WRITE_BUFFER",
125+
"class_name": "R2WriteBuffer",
126+
"script_name": "gitbook-open-v2-do-staging"
109127
},
110128
{
111129
"name": "NEXT_TAG_CACHE_DO_SHARDED",
112-
"class_name": "DOShardedTagCache"
130+
"class_name": "DOShardedTagCache",
131+
"script_name": "gitbook-open-v2-do-staging"
132+
},
133+
{
134+
"name": "NEXT_CACHE_DO_QUEUE",
135+
"class_name": "DOQueueHandler",
136+
"script_name": "gitbook-open-v2-do-staging"
113137
}
114138
]
115139
},
@@ -165,12 +189,19 @@
165189
"durable_objects": {
166190
"bindings": [
167191
{
168-
"name": "NEXT_CACHE_DO_QUEUE",
169-
"class_name": "DOQueueHandler"
192+
"name": "WRITE_BUFFER",
193+
"class_name": "R2WriteBuffer",
194+
"script_name": "gitbook-open-v2-do-production"
170195
},
171196
{
172197
"name": "NEXT_TAG_CACHE_DO_SHARDED",
173-
"class_name": "DOShardedTagCache"
198+
"class_name": "DOShardedTagCache",
199+
"script_name": "gitbook-open-v2-do-production"
200+
},
201+
{
202+
"name": "NEXT_CACHE_DO_QUEUE",
203+
"class_name": "DOQueueHandler",
204+
"script_name": "gitbook-open-v2-do-production"
174205
}
175206
]
176207
},

0 commit comments

Comments
 (0)