Skip to content

Commit a458468

Browse files
lukasholzerlukasholzer
authored
feat: implement stale through initialRevalidateSeconds (#43)
1 parent 5079abb commit a458468

File tree

6 files changed

+164
-82
lines changed

6 files changed

+164
-82
lines changed

‎src/build/content/prerendered.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { getDeployStore } from '@netlify/blobs'
22
import { NetlifyPluginConstants } from '@netlify/build'
33
import { globby } from 'globby'
44
import { readFile } from 'node:fs/promises'
5+
import { join } from 'node:path'
56
import { cpus } from 'os'
67
import pLimit from 'p-limit'
78
import { parse, ParsedPath } from 'path'
@@ -79,7 +80,7 @@ const buildPrerenderedContentEntries = async (cwd: string): Promise<Promise<Cach
7980
})
8081
.map(async (path: ParsedPath): Promise<CacheEntry> => {
8182
const { dir, name, ext } = path
82-
const key = `${dir}/${name}`
83+
const key = join(dir, name)
8384
let value
8485

8586
if (isPage(path, paths)) {

‎src/run/handlers/cache.cts

+73-8
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
//
44
import { getDeployStore } from '@netlify/blobs'
55
import { purgeCache } from '@netlify/functions'
6+
import { readFileSync } from 'node:fs'
67
import type {
78
CacheHandler,
89
CacheHandlerContext,
@@ -17,6 +18,16 @@ type TagManifest = { revalidatedAt: number }
1718
const tagsManifestPath = '_netlify-cache/tags'
1819
const blobStore = getDeployStore()
1920

21+
// load the prerender manifest
22+
const prerenderManifest = JSON.parse(
23+
readFileSync(join(process.cwd(), '.next/prerender-manifest.json'), 'utf-8'),
24+
)
25+
26+
/** Converts a cache key pathname to a route */
27+
function toRoute(pathname: string): string {
28+
return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/'
29+
}
30+
2031
export default class NetlifyCacheHandler implements CacheHandler {
2132
options: CacheHandlerContext
2233
revalidatedTags: string[]
@@ -31,15 +42,31 @@ export default class NetlifyCacheHandler implements CacheHandler {
3142

3243
async get(...args: Parameters<CacheHandler['get']>): ReturnType<CacheHandler['get']> {
3344
const [cacheKey, ctx = {}] = args
34-
console.log(`[NetlifyCacheHandler.get]: ${cacheKey}`)
45+
console.debug(`[NetlifyCacheHandler.get]: ${cacheKey}`)
3546
const blob = await this.getBlobKey(cacheKey, ctx.fetchCache)
3647

37-
switch (blob?.value?.kind) {
48+
if (!blob) {
49+
return null
50+
}
51+
52+
const revalidateAfter = this.calculateRevalidate(cacheKey, blob.lastModified)
53+
// first check if there is a tag manifest
54+
// if not => stale check with revalidateAfter
55+
// yes => check with manifest
56+
const isStale = revalidateAfter !== false && revalidateAfter < Date.now()
57+
console.debug(`!!! CHACHE KEY: ${cacheKey} - is stale: `, {
58+
isStale,
59+
revalidateAfter,
60+
})
61+
if (isStale) {
62+
return null
63+
}
64+
65+
switch (blob.value.kind) {
3866
// TODO:
3967
// case 'ROUTE':
4068
// case 'FETCH':
4169
case 'PAGE':
42-
// TODO: determine if the page is stale based on the blob.lastModified Date.now()
4370
return {
4471
lastModified: blob.lastModified,
4572
value: {
@@ -51,20 +78,37 @@ export default class NetlifyCacheHandler implements CacheHandler {
5178
},
5279
}
5380

54-
default:
55-
console.log('TODO: implmenet', blob)
81+
// default:
82+
// console.log('TODO: implmenet', blob)
5683
}
5784
return null
5885
}
5986

60-
// eslint-disable-next-line require-await, class-methods-use-this
6187
async set(...args: Parameters<IncrementalCache['set']>) {
6288
const [key, data, ctx] = args
63-
console.log('NetlifyCacheHandler.set', key)
89+
console.debug(`[NetlifyCacheHandler.set]: ${key}`)
90+
let cacheKey: string | null = null
91+
switch (data?.kind) {
92+
case 'FETCH':
93+
cacheKey = join('cache/fetch-cache', key)
94+
break
95+
case 'PAGE':
96+
cacheKey = join('server/app', key)
97+
break
98+
default:
99+
console.debug(`TODO: implement NetlifyCacheHandler.set for ${key}`, { data, ctx })
100+
}
101+
102+
if (cacheKey) {
103+
await blobStore.setJSON(cacheKey, {
104+
lastModified: Date.now(),
105+
value: data,
106+
})
107+
}
64108
}
65109

66110
async revalidateTag(tag: string) {
67-
console.log('NetlifyCacheHandler.revalidateTag', tag)
111+
console.debug('NetlifyCacheHandler.revalidateTag', tag)
68112

69113
const data: TagManifest = {
70114
revalidatedAt: Date.now(),
@@ -138,4 +182,25 @@ export default class NetlifyCacheHandler implements CacheHandler {
138182

139183
return cacheEntry || null
140184
}
185+
186+
/**
187+
* Retrieves the milliseconds since midnight, January 1, 1970 when it should revalidate for a path.
188+
*/
189+
private calculateRevalidate(pathname: string, fromTime: number, dev?: boolean): number | false {
190+
// in development we don't have a prerender-manifest
191+
// and default to always revalidating to allow easier debugging
192+
if (dev) return Date.now() - 1_000
193+
194+
// if an entry isn't present in routes we fallback to a default
195+
const { initialRevalidateSeconds } = prerenderManifest.routes[toRoute(pathname)] || {
196+
initialRevalidateSeconds: 0,
197+
}
198+
// the initialRevalidate can be either set to false or to a number (representing the seconds)
199+
const revalidateAfter: number | false =
200+
typeof initialRevalidateSeconds === 'number'
201+
? initialRevalidateSeconds * 1_000 + fromTime
202+
: initialRevalidateSeconds
203+
204+
return revalidateAfter
205+
}
141206
}

‎src/run/handlers/server.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ export default async (request: Request) => {
3232
setCacheTagsHeaders(headers)
3333
setVaryHeaders(headers, request, nextConfig)
3434
event.headers = Object.fromEntries(headers.entries())
35-
console.log('Modified response headers:', JSON.stringify(event.headers, null, 2))
35+
// console.log('Modified response headers:', JSON.stringify(event.headers, null, 2))
3636
})
3737

3838
try {
@@ -46,7 +46,7 @@ export default async (request: Request) => {
4646

4747
// log the response from Next.js
4848
const response = { headers: res.getHeaders(), statusCode: res.statusCode }
49-
console.log('Next server response:', JSON.stringify(response, null, 2))
49+
// console.log('Next server response:', JSON.stringify(response, null, 2))
5050

5151
return toComputeResponse(res)
5252
}

‎tests/integration/cache-handler.test.ts

+71-58
Original file line numberDiff line numberDiff line change
@@ -48,65 +48,78 @@ test<FixtureTestContext>('Test that the simple next app is working', async (ctx)
4848
expect(load(other.body)('h1').text()).toBe('Other')
4949
})
5050

51-
test<FixtureTestContext>(
52-
'should have two pages with fetch pre rendered and revalidated',
53-
async (ctx) => {
54-
await createFixture('revalidate-fetch', ctx)
55-
console.time('runPlugin')
56-
await runPlugin(ctx)
57-
console.timeEnd('runPlugin')
58-
// check if the blob entries where successful set on the build plugin
59-
const blobEntries = await getBlobEntries(ctx)
60-
expect(blobEntries).toEqual([
61-
{
62-
key: 'cache/fetch-cache/460ed46cd9a194efa197be9f2571e51b729a039d1cff9834297f416dce5ada29',
63-
etag: expect.any(String),
64-
},
65-
{
66-
key: 'cache/fetch-cache/ad74683e49684ff4fe3d01ba1bef627bc0e38b61fa6bd8244145fbaca87f3c49',
67-
etag: expect.any(String),
68-
},
69-
{ key: 'server/app/_not-found', etag: expect.any(String) },
70-
{ key: 'server/app/index', etag: expect.any(String) },
71-
{ key: 'server/app/posts/1', etag: expect.any(String) },
72-
{ key: 'server/app/posts/2', etag: expect.any(String) },
73-
])
51+
test<FixtureTestContext>('should have a page prerendered, then wait for it to get stale and on demand revalidate it', async (ctx) => {
52+
await createFixture('revalidate-fetch', ctx)
53+
console.time('runPlugin')
54+
await runPlugin(ctx)
55+
console.timeEnd('runPlugin')
56+
// check if the blob entries where successful set on the build plugin
57+
const blobEntries = await getBlobEntries(ctx)
58+
expect(blobEntries).toEqual([
59+
{
60+
key: 'cache/fetch-cache/460ed46cd9a194efa197be9f2571e51b729a039d1cff9834297f416dce5ada29',
61+
etag: expect.any(String),
62+
},
63+
{
64+
key: 'cache/fetch-cache/ad74683e49684ff4fe3d01ba1bef627bc0e38b61fa6bd8244145fbaca87f3c49',
65+
etag: expect.any(String),
66+
},
67+
{ key: 'server/app/_not-found', etag: expect.any(String) },
68+
{ key: 'server/app/index', etag: expect.any(String) },
69+
{ key: 'server/app/posts/1', etag: expect.any(String) },
70+
{ key: 'server/app/posts/2', etag: expect.any(String) },
71+
])
72+
73+
// test the function call
74+
const post1 = await invokeFunction(ctx, { url: 'posts/1' })
75+
const post1Date = load(post1.body)('[data-testid="date-now"]').text()
76+
expect(post1.statusCode).toBe(200)
77+
expect(load(post1.body)('h1').text()).toBe('Revalidate Fetch')
78+
expect(post1.headers).toEqual(
79+
expect.objectContaining({
80+
'x-nextjs-cache': 'HIT',
81+
'netlify-cdn-cache-control': 's-maxage=3, stale-while-revalidate',
82+
}),
83+
)
7484

75-
// test the function call
76-
const post1 = await invokeFunction(ctx, { url: 'posts/1' })
77-
const post1Date = load(post1.body)('[data-testid="date-now"]').text()
78-
expect(post1.statusCode).toBe(200)
79-
expect(load(post1.body)('h1').text()).toBe('Revalidate Fetch')
80-
expect(post1.headers).toEqual(
81-
expect.objectContaining({
82-
'x-nextjs-cache': 'HIT',
83-
'netlify-cdn-cache-control': 's-maxage=3, stale-while-revalidate',
84-
}),
85-
)
85+
expect(await ctx.blobStore.get('server/app/posts/3')).toBeNull()
86+
// this page is not pre-rendered and should result in a cache miss
87+
const post3 = await invokeFunction(ctx, { url: 'posts/3' })
88+
expect(post3.statusCode).toBe(200)
89+
expect(load(post3.body)('h1').text()).toBe('Revalidate Fetch')
90+
expect(post3.headers).toEqual(
91+
expect.objectContaining({
92+
'x-nextjs-cache': 'MISS',
93+
}),
94+
)
8695

87-
// this page is not pre-rendered and should result in a cache miss
88-
const post3 = await invokeFunction(ctx, { url: 'posts/3' })
89-
expect(post3.statusCode).toBe(200)
90-
expect(load(post3.body)('h1').text()).toBe('Revalidate Fetch')
91-
expect(post3.headers).toEqual(
92-
expect.objectContaining({
93-
'x-nextjs-cache': 'MISS',
94-
}),
95-
)
96+
// wait to have a stale page
97+
await new Promise<void>((resolve) => setTimeout(resolve, 1_000))
98+
// after the dynamic call of `posts/3` it should be in cache, not this is after the timout as the cache set happens async
99+
expect(await ctx.blobStore.get('server/app/posts/3')).not.toBeNull()
96100

97-
// TODO: uncomment once stale is implemented via the cache tags manifest
98-
// wait 500ms to have a stale page
99-
// await new Promise<void>((resolve) => setTimeout(resolve, 500))
101+
const stale = await invokeFunction(ctx, { url: 'posts/1' })
102+
const staleDate = load(stale.body)('[data-testid="date-now"]').text()
103+
expect(stale.statusCode).toBe(200)
104+
// it should have a new date rendered
105+
expect(staleDate, 'the date was cached and is matching the initial one').not.toBe(post1Date)
106+
expect(stale.headers).toEqual(
107+
expect.objectContaining({
108+
'x-nextjs-cache': 'MISS',
109+
}),
110+
)
100111

101-
// const stale = await invokeFunction(ctx, { url: 'posts/1' })
102-
// expect(stale.statusCode).toBe(200)
103-
// // it should have a new date rendered
104-
// expect(load(stale.body)('[data-testid="date-now"]').text()).not.toBe(post1Date)
105-
// expect(stale.headers).toEqual(
106-
// expect.objectContaining({
107-
// 'x-nextjs-cache': 'MISS',
108-
// }),
109-
// )
110-
},
111-
{ retry: 3 },
112-
)
112+
// it does not wait for the cache.set so we have to manually wait here until the blob storage got populated
113+
await new Promise<void>((resolve) => setTimeout(resolve, 100))
114+
115+
// now the page should be in cache again and we should get a cache hit
116+
const cached = await invokeFunction(ctx, { url: 'posts/1' })
117+
const cachedDate = load(cached.body)('[data-testid="date-now"]').text()
118+
expect(cached.statusCode).toBe(200)
119+
expect(staleDate, 'the date was not cached').toBe(cachedDate)
120+
expect(cached.headers).toEqual(
121+
expect.objectContaining({
122+
'x-nextjs-cache': 'HIT',
123+
}),
124+
)
125+
})

‎tests/utils/fixture.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { BlobsServer } from '@netlify/blobs'
1+
import { BlobsServer, type getStore } from '@netlify/blobs'
22
import { TestContext, assert, expect, vi } from 'vitest'
33

44
import type {
@@ -22,7 +22,8 @@ export interface FixtureTestContext extends TestContext {
2222
siteID: string
2323
deployID: string
2424
blobStoreHost: string
25-
blobStore: BlobsServer
25+
blobServer: BlobsServer
26+
blobStore: ReturnType<typeof getStore>
2627
functionDist: string
2728
cleanup?: () => Promise<void>
2829
}

‎tests/utils/helpers.ts

+13-11
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import getPort from 'get-port'
22
import { BLOB_TOKEN, type FixtureTestContext } from './fixture'
33

4-
import { cp, mkdtemp, rm } from 'node:fs/promises'
4+
import { BlobsServer, getDeployStore } from '@netlify/blobs'
5+
import { mkdtemp } from 'node:fs/promises'
56
import { tmpdir } from 'node:os'
67
import { join } from 'node:path'
7-
import { BlobsServer, getDeployStore } from '@netlify/blobs'
88

99
/**
1010
* Generates a 24char deploy ID (this is validated in the blob storage so we cant use a uuidv4)
@@ -38,26 +38,28 @@ export const createBlobContext = (ctx: FixtureTestContext) =>
3838
export const startMockBlobStore = async (ctx: FixtureTestContext) => {
3939
const port = await getPort()
4040
// create new blob store server
41-
ctx.blobStore = new BlobsServer({
41+
ctx.blobServer = new BlobsServer({
4242
port,
4343
token: BLOB_TOKEN,
4444
directory: await mkdtemp(join(tmpdir(), 'netlify-next-runtime-blob-')),
4545
})
46-
await ctx.blobStore.start()
46+
await ctx.blobServer.start()
4747
ctx.blobStoreHost = `localhost:${port}`
4848
}
4949

5050
/**
5151
* Retrieves an array of blob store entries
5252
*/
5353
export const getBlobEntries = async (ctx: FixtureTestContext) => {
54-
const store = getDeployStore({
55-
apiURL: `http://${ctx.blobStoreHost}`,
56-
deployID: ctx.deployID,
57-
siteID: ctx.siteID,
58-
token: BLOB_TOKEN,
59-
})
54+
ctx.blobStore = ctx.blobStore
55+
? ctx.blobStore
56+
: getDeployStore({
57+
apiURL: `http://${ctx.blobStoreHost}`,
58+
deployID: ctx.deployID,
59+
siteID: ctx.siteID,
60+
token: BLOB_TOKEN,
61+
})
6062

61-
const { blobs } = await store.list()
63+
const { blobs } = await ctx.blobStore.list()
6264
return blobs
6365
}

0 commit comments

Comments
 (0)