Use unnest everywhere

api/src/controllers/seed/images.ts

@@ -11,6 +11,7 @@ import { db, type Transaction } from "~/db";
 import { mqueue } from "~/db/schema/mqueue";
 import type { Image } from "~/models/utils";
 import { getFile } from "~/utils";
+import { unnestValues } from "~/db/utils";
 
 export const imageDir = process.env.IMAGES_PATH ?? "/images";
 export const defaultBlurhash = "000000";
@@ -83,9 +84,12 @@ export const flushImageQueue = async (
 ) => {
 	if (!imgQueue.length) return;
 	record("enqueue images", async () => {
-		await tx
-			.insert(mqueue)
-			.values(imgQueue.map((x) => ({ kind: "image", message: x, priority })));
+		await tx.insert(mqueue).select(
+			unnestValues(
+				imgQueue.map((x) => ({ kind: "image", message: x, priority })),
+				mqueue,
+			),
+		);
 		await tx.execute(sql`notify kyoo_image`);
 	});
 };

api/src/controllers/seed/insert/collection.ts

@@ -1,7 +1,7 @@
 import { sql } from "drizzle-orm";
 import { db } from "~/db";
 import { shows, showTranslations } from "~/db/schema";
-import { conflictUpdateAllExcept } from "~/db/utils";
+import { conflictUpdateAllExcept, unnestValues } from "~/db/utils";
 import type { SeedCollection } from "~/models/collections";
 import type { SeedMovie } from "~/models/movie";
 import type { SeedSerie } from "~/models/serie";
@@ -75,7 +75,7 @@ export const insertCollection = async (
 		await flushImageQueue(tx, imgQueue, 100);
 		await tx
 			.insert(showTranslations)
-			.values(trans)
+			.select(unnestValues(trans, showTranslations))
 			.onConflictDoUpdate({
 				target: [showTranslations.pk, showTranslations.language],
 				set: conflictUpdateAllExcept(showTranslations, ["pk", "language"]),

api/src/controllers/seed/insert/entries.ts

@@ -6,7 +6,7 @@ import {
 	entryVideoJoin,
 	videos,
 } from "~/db/schema";
-import { conflictUpdateAllExcept, unnestValues, values } from "~/db/utils";
+import { conflictUpdateAllExcept, unnest, unnestValues } from "~/db/utils";
 import type { SeedEntry as SEntry, SeedExtra as SExtra } from "~/models/entry";
 import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images";
 import { guessNextRefresh } from "../refresh";
@@ -169,11 +169,12 @@ export const insertEntries = async (
 						),
 					})
 					.from(
-						values(vids, {
+						unnest(vids, "vids", {
 							entryPk: "integer",
+							entrySlug: "string",
 							needRendering: "boolean",
 							videoId: "uuid",
-						}).as("vids"),
+						}),
 					)
 					.innerJoin(videos, eq(videos.id, sql`vids.videoId`)),
 			)

api/src/controllers/seed/insert/seasons.ts

@@ -1,6 +1,6 @@
 import { db } from "~/db";
 import { seasons, seasonTranslations } from "~/db/schema";
-import { conflictUpdateAllExcept } from "~/db/utils";
+import { conflictUpdateAllExcept, unnestValues } from "~/db/utils";
 import type { SeedSeason } from "~/models/season";
 import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images";
 import { guessNextRefresh } from "../refresh";
@@ -30,7 +30,7 @@ export const insertSeasons = async (
 		});
 		const ret = await tx
 			.insert(seasons)
-			.values(vals)
+			.select(unnestValues(vals, seasons))
 			.onConflictDoUpdate({
 				target: seasons.slug,
 				set: conflictUpdateAllExcept(seasons, [
@@ -66,7 +66,7 @@ export const insertSeasons = async (
 		await flushImageQueue(tx, imgQueue, -10);
 		await tx
 			.insert(seasonTranslations)
-			.values(trans)
+			.select(unnestValues(trans, seasonTranslations))
 			.onConflictDoUpdate({
 				target: [seasonTranslations.pk, seasonTranslations.language],
 				set: conflictUpdateAllExcept(seasonTranslations, ["pk", "language"]),

api/src/controllers/seed/insert/shows.ts

@@ -16,7 +16,7 @@ import {
 	shows,
 	showTranslations,
 } from "~/db/schema";
-import { conflictUpdateAllExcept, sqlarr } from "~/db/utils";
+import { conflictUpdateAllExcept, sqlarr, unnestValues } from "~/db/utils";
 import type { SeedCollection } from "~/models/collections";
 import type { SeedMovie } from "~/models/movie";
 import type { SeedSerie } from "~/models/serie";
@@ -95,7 +95,7 @@ export const insertShow = async (
 		await flushImageQueue(tx, imgQueue, 200);
 		await tx
 			.insert(showTranslations)
-			.values(trans)
+			.select(unnestValues(trans, showTranslations))
 			.onConflictDoUpdate({
 				target: [showTranslations.pk, showTranslations.language],
 				set: conflictUpdateAllExcept(showTranslations, ["pk", "language"]),

api/src/controllers/seed/insert/staff.ts

@@ -1,7 +1,7 @@
 import { eq, sql } from "drizzle-orm";
 import { db } from "~/db";
 import { roles, staff } from "~/db/schema";
-import { conflictUpdateAllExcept } from "~/db/utils";
+import { conflictUpdateAllExcept, unnestValues } from "~/db/utils";
 import type { SeedStaff } from "~/models/staff";
 import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images";
 
@@ -22,7 +22,7 @@ export const insertStaff = async (
 		}));
 		const ret = await tx
 			.insert(staff)
-			.values(people)
+			.select(unnestValues(people, staff))
 			.onConflictDoUpdate({
 				target: staff.slug,
 				set: conflictUpdateAllExcept(staff, ["pk", "id", "slug", "createdAt"]),
@@ -50,7 +50,7 @@ export const insertStaff = async (
 		//  - we want `order` to stay in sync (& without duplicates)
 		//  - we don't have ways to identify a role so we can't onConflict
 		await tx.delete(roles).where(eq(roles.showPk, showPk));
-		await tx.insert(roles).values(rval);
+		await tx.insert(roles).select(unnestValues(rval, roles));
 
 		return ret;
 	});

api/src/controllers/seed/insert/studios.ts

@@ -1,6 +1,7 @@
+import { sql } from "drizzle-orm";
 import { db } from "~/db";
 import { showStudioJoin, studios, studioTranslations } from "~/db/schema";
-import { conflictUpdateAllExcept } from "~/db/utils";
+import { conflictUpdateAllExcept, sqlarr, unnestValues } from "~/db/utils";
 import type { SeedStudio } from "~/models/studio";
 import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images";
 
@@ -21,7 +22,7 @@ export const insertStudios = async (
 
 		const ret = await tx
 			.insert(studios)
-			.values(vals)
+			.select(unnestValues(vals, studios))
 			.onConflictDoUpdate({
 				target: studios.slug,
 				set: conflictUpdateAllExcept(studios, [
@@ -48,7 +49,7 @@ export const insertStudios = async (
 		await flushImageQueue(tx, imgQueue, -100);
 		await tx
 			.insert(studioTranslations)
-			.values(trans)
+			.select(unnestValues(trans, studioTranslations))
 			.onConflictDoUpdate({
 				target: [studioTranslations.pk, studioTranslations.language],
 				set: conflictUpdateAllExcept(studioTranslations, ["pk", "language"]),
@@ -56,7 +57,14 @@ export const insertStudios = async (
 
 		await tx
 			.insert(showStudioJoin)
-			.values(ret.map((studio) => ({ showPk: showPk, studioPk: studio.pk })))
+			.select(
+				db
+					.select({
+						showPk: sql`${showPk}`.as("showPk"),
+						studioPk: sql`v.studioPk`.as("studioPk"),
+					})
+					.from(sql`unnest(${sqlarr(ret.map((x) => x.pk))}) as v("studioPk")`),
+			)
 			.onConflictDoNothing();
 		return ret;
 	});

api/src/controllers/videos.ts

@@ -35,7 +35,8 @@ import {
 	jsonbBuildObject,
 	jsonbObjectAgg,
 	sqlarr,
-	values,
+	unnest,
+	unnestValues,
 } from "~/db/utils";
 import { Entry } from "~/models/entry";
 import { KError } from "~/models/error";
@@ -129,10 +130,10 @@ async function linkVideos(
 					slug: computeVideoSlug(entriesQ.slug, hasRenderingQ),
 				})
 				.from(
-					values(links, {
+					unnest(links, "j", {
 						video: "integer",
 						entry: "jsonb",
-					}).as("j"),
+					}),
 				)
 				.innerJoin(videos, eq(videos.pk, sql`j.video`))
 				.innerJoin(
@@ -835,7 +836,7 @@ export const videosWriteH = new Elysia({ prefix: "/videos", tags: ["videos"] })
 				try {
 					vids = await tx
 						.insert(videos)
-						.values(body)
+						.select(unnestValues(body, videos))
 						.onConflictDoUpdate({
 							target: [videos.path],
 							set: conflictUpdateAllExcept(videos, ["pk", "id", "createdAt"]),

api/src/db/utils.ts

@@ -74,14 +74,16 @@ export function conflictUpdateAllExcept<
 }
 
 // drizzle is bugged and doesn't allow js arrays to be used in raw sql.
-export function sqlarr(array: unknown[]) {
+export function sqlarr(array: unknown[]): string {
 	return `{${array
 		.map((item) =>
 			!item || item === "null"
 				? "null"
-				: typeof item === "object"
-					? `"${JSON.stringify(item).replaceAll('"', '\\"')}"`
-					: `"${item}"`,
+				: Array.isArray(item)
+					? sqlarr(item)
+					: typeof item === "object"
+						? `"${JSON.stringify(item).replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`
+						: `"${item}"`,
 		)
 		.join(", ")}}`;
 }
@@ -137,6 +139,7 @@ export const unnestValues = <
 ) => {
 	if (values[0] === undefined)
 		throw new Error("Invalid values, expecting at least one items");
+
 	const columns = getTableColumns(typeInfo);
 	const keys = Object.keys(values[0]).filter((x) => x in columns);
 	// @ts-expect-error: drizzle internal
@@ -189,6 +192,28 @@ export const unnestValues = <
 		);
 };
 
+export const unnest = <T extends Record<string, unknown>>(
+	values: T[],
+	name: string,
+	typeInfo: Record<keyof T, string>,
+) => {
+	const keys = Object.keys(typeInfo);
+	const vals = values.reduce(
+		(acc, cur) => {
+			for (const k of keys) {
+				if (k in cur) acc[k].push(cur[k]);
+				else acc[k].push(null);
+			}
+			return acc;
+		},
+		Object.fromEntries(keys.map((x) => [x, [] as unknown[]])),
+	);
+	return sql`unnest(${sql.join(
+		keys.map((k) => sql`${sqlarr(vals[k])}${sql.raw(`::${typeInfo[k]}[]`)}`),
+		sql.raw(", "),
+	)}) as ${sql.raw(name)}(${sql.raw(keys.map((x) => `"${x}"`).join(", "))})`;
+};
+
 export const coalesce = <T>(val: SQL<T> | SQLWrapper, def: SQL<T> | Column) => {
 	return sql<T>`coalesce(${val}, ${def})`;
 };