Skip to content

Commit a5e47c4

Browse files
committed
feat: improve cache cleanup
1 parent 1fbf884 commit a5e47c4

File tree

7 files changed

+146
-37
lines changed

7 files changed

+146
-37
lines changed

docs/content/1.getting-started/1.index.md

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,11 +102,15 @@ variant: subtle
102102
---
103103
::
104104

105-
#### `CACHE_CLEANUP_OLDER_THAN_DAYS`
105+
#### `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS`
106106

107-
- Default: `90`
107+
- Default: `30`
108108

109-
The number of days to keep stale cache data and metadata before deleting it. Set to `0` to disable cache cleanup.
109+
Cache entries which have not been accessed for `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` days will be automatically deleted. Set to `0` to disable.
110+
111+
#### `CACHE_CLEANUP_TTL_DAYS`
112+
113+
Cache entries which have been created `CACHE_CLEANUP_TTL_DAYS` days ago will be automatically deleted. Set to `0` to disable.
110114

111115
#### `CACHE_CLEANUP_CRON`
112116

lib/db/index.ts

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ export interface CacheKeysTable {
1818
version: string
1919
updated_at: string
2020
accessed_at: string
21+
created_at: string
2122
}
2223
export interface UploadsTable {
2324
created_at: string
@@ -191,17 +192,26 @@ export async function touchKey(
191192

192193
export async function findStaleKeys(
193194
db: DB,
194-
{ olderThanDays, date }: { olderThanDays?: number; date?: Date },
195+
opts?: {
196+
untouchedTTLDays?: number
197+
ttlDays?: number
198+
date?: Date
199+
},
195200
) {
196-
if (olderThanDays === undefined) return db.selectFrom('cache_keys').selectAll().execute()
201+
const now = opts?.date ?? new Date()
202+
let query = db.selectFrom('cache_keys')
197203

198-
const now = date ?? new Date()
199-
const threshold = new Date(now.getTime() - olderThanDays * 24 * 60 * 60 * 1000)
200-
return db
201-
.selectFrom('cache_keys')
202-
.where('accessed_at', '<', threshold.toISOString())
203-
.selectAll()
204-
.execute()
204+
if (opts?.ttlDays !== undefined) {
205+
const threshold = new Date(now.getTime() - opts.ttlDays * 24 * 60 * 60 * 1000)
206+
query = query.where('created_at', '<', threshold.toISOString())
207+
}
208+
209+
if (opts?.untouchedTTLDays !== undefined) {
210+
const untouchedThreshold = new Date(now.getTime() - opts.untouchedTTLDays * 24 * 60 * 60 * 1000)
211+
query = query.where('accessed_at', '<', untouchedThreshold.toISOString())
212+
}
213+
214+
return query.selectAll().execute()
205215
}
206216

207217
export async function createKey(
@@ -217,6 +227,7 @@ export async function createKey(
217227
version,
218228
updated_at: now.toISOString(),
219229
accessed_at: now.toISOString(),
230+
created_at: now.toISOString(),
220231
})
221232
.execute()
222233
}

lib/db/migrations.ts

Lines changed: 75 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,33 @@
1-
import type { Migration } from 'kysely'
1+
import type { Kysely, Migration } from 'kysely'
22

33
import type { DatabaseDriverName } from '~/lib/db/drivers'
44
import { sql } from 'kysely'
55

6+
async function createCacheKeysTable({
7+
db,
8+
dbType,
9+
}: {
10+
db: Kysely<any>
11+
dbType: DatabaseDriverName
12+
}) {
13+
let query = db.schema
14+
.createTable('cache_keys')
15+
.addColumn('id', 'varchar(255)', (col) => col.notNull().primaryKey())
16+
.addColumn('key', 'text', (col) => col.notNull())
17+
.addColumn('version', 'text', (col) => col.notNull())
18+
.addColumn('updated_at', 'text', (col) => col.notNull())
19+
.addColumn('accessed_at', 'text', (col) => col.notNull())
20+
21+
if (dbType === 'mysql') query = query.modifyEnd(sql`engine=InnoDB CHARSET=latin1`)
22+
23+
await query.ifNotExists().execute()
24+
}
25+
626
export function migrations(dbType: DatabaseDriverName) {
727
return {
828
$0_cache_keys_table: {
929
async up(db) {
10-
let query = db.schema
11-
.createTable('cache_keys')
12-
.addColumn('id', 'varchar(255)', (col) => col.notNull().primaryKey())
13-
.addColumn('key', 'text', (col) => col.notNull())
14-
.addColumn('version', 'text', (col) => col.notNull())
15-
.addColumn('updated_at', 'text', (col) => col.notNull())
16-
.addColumn('accessed_at', 'text', (col) => col.notNull())
17-
18-
if (dbType === 'mysql') query = query.modifyEnd(sql`engine=InnoDB CHARSET=latin1`)
19-
20-
await query.ifNotExists().execute()
30+
await createCacheKeysTable({ db, dbType })
2131
},
2232
async down(db) {
2333
await db.schema.dropTable('cache_keys').ifExists().execute()
@@ -76,5 +86,58 @@ export function migrations(dbType: DatabaseDriverName) {
7686
await db.schema.alterTable('upload_parts').dropColumn('e_tag').execute()
7787
},
7888
},
89+
$4_cache_entry_created_at: {
90+
async up(db) {
91+
await db
92+
.insertInto('cache_keys')
93+
.values({
94+
id: '',
95+
key: '',
96+
version: '',
97+
updated_at: new Date().toISOString(),
98+
accessed_at: new Date().toISOString(),
99+
})
100+
.execute()
101+
await db.schema.alterTable('cache_keys').addColumn('created_at', 'text').execute()
102+
103+
await db
104+
.updateTable('cache_keys')
105+
.set({
106+
created_at: new Date().toISOString(),
107+
})
108+
.execute()
109+
110+
if (dbType === 'mysql')
111+
await db.schema
112+
.alterTable('cache_keys')
113+
.modifyColumn('created_at', 'text', (c) => c.notNull())
114+
.execute()
115+
else if (dbType === 'postgres')
116+
await db.schema
117+
.alterTable('cache_keys')
118+
.alterColumn('created_at', (c) => c.setNotNull())
119+
.execute()
120+
else {
121+
// rename old table
122+
await db.schema.alterTable('cache_keys').renameTo('old_cache_keys').execute()
123+
// recreate table
124+
await createCacheKeysTable({ db, dbType })
125+
126+
// add not null column
127+
await db.schema
128+
.alterTable('cache_keys')
129+
.addColumn('created_at', 'text', (c) => c.notNull())
130+
.execute()
131+
132+
// migrate old data
133+
await db
134+
.insertInto('cache_keys')
135+
.expression((e) => e.selectFrom('old_cache_keys').selectAll())
136+
.execute()
137+
138+
await db.schema.dropTable('old_cache_keys').execute()
139+
}
140+
},
141+
},
79142
} satisfies Record<string, Migration>
80143
}

lib/env.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,12 @@ const portSchema = z.coerce.number().int().min(1).max(65_535)
66

77
const envSchema = z.object({
88
ENABLE_DIRECT_DOWNLOADS: booleanSchema.default('false'),
9+
/**
10+
* @deprecated use `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` or `CACHE_CLEANUP_TTL_DAYS` instead
11+
*/
912
CACHE_CLEANUP_OLDER_THAN_DAYS: z.coerce.number().int().min(0).default(90),
13+
CACHE_CLEANUP_UNTOUCHED_TTL_DAYS: z.coerce.number().int().min(0).default(30),
14+
CACHE_CLEANUP_TTL_DAYS: z.coerce.number().int().min(0).optional(),
1015
CACHE_CLEANUP_CRON: z.string().default('0 0 * * *'),
1116
UPLOAD_CLEANUP_CRON: z.string().default('*/10 * * * *'),
1217
API_BASE_URL: z.string().url(),

lib/storage/index.ts

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -184,12 +184,10 @@ export const useStorageAdapter = createSingletonPromise(async () => {
184184
logger.debug('Download:', objectName)
185185
return driver.createReadStream(objectName)
186186
},
187-
async pruneCaches(olderThanDays?: number) {
188-
logger.debug('Prune:', {
189-
olderThanDays,
190-
})
187+
async pruneCaches(opts?: { untouchedTTLDays?: number; ttlDays?: number }) {
188+
logger.debug('Prune:', opts)
191189

192-
const keys = await findStaleKeys(db, { olderThanDays })
190+
const keys = await findStaleKeys(db, opts)
193191
if (keys.length === 0) {
194192
logger.debug('Prune: No caches to prune')
195193
return
@@ -198,9 +196,7 @@ export const useStorageAdapter = createSingletonPromise(async () => {
198196
await driver.delete(keys.map((key) => getObjectNameFromKey(key.key, key.version)))
199197
await pruneKeys(db, keys)
200198

201-
logger.debug('Prune: Caches pruned', {
202-
olderThanDays,
203-
})
199+
logger.debug('Prune: Caches pruned', opts)
204200
},
205201
async pruneUploads(olderThanDate: Date) {
206202
logger.debug('Prune uploads')

plugins/cleanup.ts

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,22 @@ export default defineNitroPlugin(() => {
1111
if (!cluster.isPrimary) return
1212

1313
// cache cleanup
14-
if (ENV.CACHE_CLEANUP_OLDER_THAN_DAYS > 0) {
14+
if (
15+
ENV.CACHE_CLEANUP_OLDER_THAN_DAYS ||
16+
ENV.CACHE_CLEANUP_TTL_DAYS ||
17+
ENV.CACHE_CLEANUP_UNTOUCHED_TTL_DAYS
18+
) {
1519
const job = new Cron(ENV.CACHE_CLEANUP_CRON)
1620
const nextRun = job.nextRun()
1721
logger.info(
18-
`Cleaning up cache entries older than ${colorize('blue', `${ENV.CACHE_CLEANUP_OLDER_THAN_DAYS}d`)} with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`,
22+
`Cleaning up cache entries with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`,
1923
)
2024
job.schedule(async () => {
2125
const adapter = await useStorageAdapter()
22-
await adapter.pruneCaches(ENV.CACHE_CLEANUP_OLDER_THAN_DAYS)
26+
await adapter.pruneCaches({
27+
ttlDays: ENV.CACHE_CLEANUP_TTL_DAYS,
28+
untouchedTTLDays: ENV.CACHE_CLEANUP_UNTOUCHED_TTL_DAYS ?? ENV.CACHE_CLEANUP_OLDER_THAN_DAYS,
29+
})
2330
})
2431
}
2532

tests/cleanup.test.ts

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,14 +61,14 @@ describe('getting stale keys', async () => {
6161
beforeEach(() => pruneKeys(db))
6262

6363
const version = '0577ec58bee6d5415625'
64-
test('returns stale keys if threshold is passed', async () => {
64+
test('returns untouched stale keys if threshold is passed', async () => {
6565
const referenceDate = new Date('2024-04-01T00:00:00Z')
6666
await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') })
6767
await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') })
6868
await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') })
6969
await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') })
7070

71-
const match = await findStaleKeys(db, { olderThanDays: 30, date: referenceDate })
71+
const match = await findStaleKeys(db, { untouchedTTLDays: 30, date: referenceDate })
7272
expect(match.length).toBe(2)
7373

7474
const matchA = match.find((m) => m.key === 'cache-a')
@@ -80,6 +80,29 @@ describe('getting stale keys', async () => {
8080
expect(matchB?.accessed_at).toBe('2024-02-01T00:00:00.000Z')
8181
})
8282

83+
test('returns stale keys if threshold is passed', async () => {
84+
const referenceDate = new Date('2024-04-01T00:00:00Z')
85+
await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') })
86+
await touchKey(db, { key: 'cache-a', version, date: referenceDate })
87+
await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') })
88+
await touchKey(db, { key: 'cache-b', version, date: referenceDate })
89+
await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') })
90+
await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') })
91+
92+
const match = await findStaleKeys(db, { ttlDays: 30, date: referenceDate })
93+
expect(match.length).toBe(2)
94+
95+
const matchA = match.find((m) => m.key === 'cache-a')
96+
expect(matchA).toBeDefined()
97+
expect(matchA?.created_at).toBe('2024-01-01T00:00:00.000Z')
98+
expect(matchA?.accessed_at).toBe('2024-04-01T00:00:00.000Z')
99+
100+
const matchB = match.find((m) => m.key === 'cache-b')
101+
expect(matchB).toBeDefined()
102+
expect(matchB?.created_at).toBe('2024-02-01T00:00:00.000Z')
103+
expect(matchB?.accessed_at).toBe('2024-04-01T00:00:00.000Z')
104+
})
105+
83106
test('returns all keys if threshold is not passed', async () => {
84107
const referenceDate = new Date('2024-04-01T00:00:00Z')
85108
await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') })

0 commit comments

Comments
 (0)