Skip to content

Commit ec8b3bf

Browse files
committed
refactor: multi part uploads
1 parent 92d105d commit ec8b3bf

File tree

21 files changed

+2441
-1180
lines changed

21 files changed

+2441
-1180
lines changed

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,4 @@ WORKDIR /app
2828

2929
COPY --from=builder /app/.output ./
3030

31-
CMD node /app/server/index.mjs
31+
CMD ["node", "/app/server/index.mjs"]

lib/db/index.ts

Lines changed: 34 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ import type { DatabaseDriverName } from '~/lib/db/drivers'
44
import cluster from 'node:cluster'
55

66
import { hash } from 'node:crypto'
7+
import { createSingletonPromise } from '@antfu/utils'
78
import { Kysely, Migrator } from 'kysely'
89
import { getDatabaseDriver } from '~/lib/db/drivers'
910
import { migrations } from '~/lib/db/migrations'
10-
import { ENV } from '~/lib/env'
1111

12+
import { ENV } from '~/lib/env'
1213
import { logger } from '~/lib/logger'
1314

1415
export interface CacheKeysTable {
@@ -23,12 +24,10 @@ export interface UploadsTable {
2324
key: string
2425
version: string
2526
id: string
26-
driver_upload_id: string
2727
}
2828
export interface UploadPartsTable {
2929
upload_id: string
3030
part_number: number
31-
e_tag: string | null
3231
}
3332

3433
export interface MetaTable {
@@ -43,62 +42,46 @@ export interface Database {
4342
meta: MetaTable
4443
}
4544

46-
let _db: Kysely<Database>
47-
48-
let initializationPromise: Promise<void> | undefined
49-
export async function initializeDatabase() {
50-
if (initializationPromise) return initializationPromise
51-
52-
// eslint-disable-next-line unicorn/consistent-function-scoping
53-
const init = async () => {
54-
const driverName = ENV.DB_DRIVER
55-
const driverSetup = getDatabaseDriver(driverName)
56-
if (!driverSetup) {
57-
logger.error(`No database driver found for ${driverName}`)
58-
// eslint-disable-next-line unicorn/no-process-exit
59-
process.exit(1)
60-
}
61-
if (cluster.isPrimary) logger.info(`Using database driver: ${driverName}`)
45+
export const useDB = createSingletonPromise(async () => {
46+
const driverName = ENV.DB_DRIVER
47+
const driverSetup = getDatabaseDriver(driverName)
48+
if (!driverSetup) {
49+
logger.error(`No database driver found for ${driverName}`)
50+
// eslint-disable-next-line unicorn/no-process-exit
51+
process.exit(1)
52+
}
53+
if (cluster.isPrimary) logger.info(`Using database driver: ${driverName}`)
6254

63-
const driver = await driverSetup()
55+
const driver = await driverSetup()
6456

65-
_db = new Kysely<Database>({
66-
dialect: driver,
67-
})
57+
const db = new Kysely<Database>({
58+
dialect: driver,
59+
})
6860

69-
if (cluster.isPrimary) {
70-
logger.info('Migrating database...')
71-
const migrator = new Migrator({
72-
db: _db,
73-
provider: {
74-
async getMigrations() {
75-
return migrations(driverName as DatabaseDriverName)
76-
},
61+
if (cluster.isPrimary) {
62+
logger.info('Migrating database...')
63+
const migrator = new Migrator({
64+
db,
65+
provider: {
66+
async getMigrations() {
67+
return migrations(driverName as DatabaseDriverName)
7768
},
78-
})
79-
const { error, results } = await migrator.migrateToLatest()
80-
if (error) {
81-
logger.error('Database migration failed', error)
82-
// eslint-disable-next-line unicorn/no-process-exit
83-
process.exit(1)
84-
}
85-
logger.debug('Migration results', results)
86-
logger.success('Database migrated')
69+
},
70+
})
71+
const { error, results } = await migrator.migrateToLatest()
72+
if (error) {
73+
logger.error('Database migration failed', error)
74+
// eslint-disable-next-line unicorn/no-process-exit
75+
process.exit(1)
8776
}
77+
logger.debug('Migration results', results)
78+
logger.success('Database migrated')
8879
}
8980

90-
initializationPromise = init()
91-
return initializationPromise
92-
}
93-
94-
export async function useDB() {
95-
if (!_db) {
96-
await initializeDatabase()
97-
}
98-
return _db
99-
}
81+
return db
82+
})
10083

101-
type DB = typeof _db
84+
type DB = Awaited<ReturnType<typeof useDB>>
10285

10386
/**
10487
* @see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key

lib/db/migrations.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,5 +70,11 @@ export function migrations(dbType: DatabaseDriverName) {
7070
await db.schema.dropTable('meta').ifExists().execute()
7171
},
7272
},
73+
$3_remove_unused_columns: {
74+
async up(db) {
75+
await db.schema.alterTable('uploads').dropColumn('driver_upload_id').execute()
76+
await db.schema.alterTable('upload_parts').dropColumn('e_tag').execute()
77+
},
78+
},
7379
} satisfies Record<string, Migration>
7480
}

lib/storage/defineStorageDriver.ts

Lines changed: 50 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,53 +2,66 @@
22

33
import type { Readable } from 'node:stream'
44
import type { z } from 'zod'
5+
import path from 'node:path'
56

67
import { formatZodError } from '~/lib/env'
78
import { logger } from '~/lib/logger'
89

9-
interface PartDetails {
10-
partNumber: number
11-
eTag?: string
12-
}
10+
export abstract class StorageDriver {
11+
static baseFolder = 'gh-actions-cache'
12+
static uploadFolder = '.uploads'
13+
baseFolderPrefix: string | undefined
14+
15+
constructor(baseFolderPrefix?: string) {
16+
this.baseFolderPrefix = baseFolderPrefix
17+
}
18+
19+
addBaseFolderPrefix(objectName: string) {
20+
return path.join(this.baseFolderPrefix ?? '', StorageDriver.baseFolder, objectName)
21+
}
22+
23+
getUploadFolderPrefix(uploadId: string) {
24+
return path.join(
25+
this.baseFolderPrefix ?? '',
26+
StorageDriver.baseFolder,
27+
StorageDriver.uploadFolder,
28+
uploadId,
29+
)
30+
}
31+
32+
addUploadFolderPrefix(opts: { uploadId: string; objectName: string }) {
33+
return path.join(this.getUploadFolderPrefix(opts.uploadId), opts.objectName)
34+
}
1335

14-
export interface StorageDriver {
15-
initiateMultiPartUpload: (opts: { objectName: string; totalSize: number }) => Promise<string>
16-
uploadPart: (opts: {
17-
objectName: string
36+
getUploadPartObjectName(opts: { uploadId: string; partNumber: number }) {
37+
return this.addUploadFolderPrefix({
38+
uploadId: opts.uploadId,
39+
objectName: `part_${opts.partNumber}`,
40+
})
41+
}
42+
43+
abstract delete(objectNames: string[]): Promise<void>
44+
abstract createReadStream(objectName: string): Promise<ReadableStream | Readable>
45+
abstract createDownloadUrl?(objectName: string): Promise<string>
46+
abstract uploadPart(opts: {
1847
uploadId: string
1948
partNumber: number
2049
data: ReadableStream
21-
chunkStart: number
22-
}) => Promise<{
23-
eTag: string | null
24-
}>
25-
completeMultipartUpload: (opts: {
26-
objectName: string
50+
}): Promise<void>
51+
abstract completeMultipartUpload(opts: {
52+
finalOutputObjectName: string
2753
uploadId: string
28-
parts: PartDetails[]
29-
}) => Promise<void>
30-
abortMultipartUpload: (opts: { objectName: string; uploadId: string }) => Promise<void>
31-
download: (opts: { objectName: string }) => Promise<ReadableStream | Readable>
32-
createDownloadUrl?: (opts: { objectName: string }) => Promise<string>
33-
delete: (opts: { objectNames: string[] }) => Promise<void>
54+
partNumbers: number[]
55+
}): Promise<void>
56+
abstract cleanupMultipartUpload(uploadId: string): Promise<void>
3457
}
3558

36-
interface DefineStorageDriverOptions<EnvSchema extends z.ZodTypeAny> {
37-
envSchema: EnvSchema
38-
setup: (options: z.output<EnvSchema>) => Promise<StorageDriver> | StorageDriver
39-
}
40-
export function defineStorageDriver<EnvSchema extends z.ZodTypeAny>(
41-
options: DefineStorageDriverOptions<EnvSchema>,
42-
) {
43-
return () => {
44-
const env = options.envSchema.safeParse(process.env)
45-
if (!env.success) {
46-
logger.error(`Invalid environment variables:\n${formatZodError(env.error)}`)
47-
// eslint-disable-next-line unicorn/no-process-exit
48-
process.exit(1)
49-
}
50-
51-
const driver = options.setup(env.data)
52-
return driver instanceof Promise ? driver : Promise.resolve(driver)
59+
export function parseEnv<Schema extends z.ZodTypeAny>(schema: Schema) {
60+
const env = schema.safeParse(process.env)
61+
if (!env.success) {
62+
logger.error(`Invalid environment variables:\n${formatZodError(env.error)}`)
63+
// eslint-disable-next-line unicorn/no-process-exit
64+
process.exit(1)
5365
}
66+
return env.data as z.output<Schema>
5467
}

0 commit comments

Comments
 (0)