6 Commits

Author SHA1 Message Date
7b38c12b85 Fix logout on deleted accounts 2025-12-04 13:04:08 +01:00
d374193ac2 Prevent duplicated staff members 2025-12-04 13:01:18 +01:00
eb4e3217a2 Remove well-known from otel 2025-12-04 12:18:21 +01:00
6a97c5319d Prevent all scanner slave to process requests 2025-12-04 12:18:05 +01:00
0f50c100cf Add requests errors in db and api 2025-12-04 12:17:47 +01:00
b11ac95d23 Fix shell.nix for sharp 2025-12-04 12:17:20 +01:00
17 changed files with 133 additions and 23 deletions

View File

@@ -13,4 +13,7 @@ pkgs.mkShell {
];
SHARP_FORCE_GLOBAL_LIBVIPS = 1;
shellHook = ''
export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH
'';
}

View File

@@ -52,8 +52,7 @@ export const base = new Elysia({ name: "base" })
console.error(code, error);
return {
status: 500,
message: "message" in error ? (error?.message ?? code) : code,
details: error,
message: "Internal server error",
} as KError;
})
.get("/health", () => ({ status: "healthy" }) as const, {

View File

@@ -4,6 +4,7 @@ import { roles, staff } from "~/db/schema";
import { conflictUpdateAllExcept, unnestValues } from "~/db/utils";
import type { SeedStaff } from "~/models/staff";
import { record } from "~/otel";
import { uniqBy } from "~/utils";
import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images";
export const insertStaff = record(
@@ -13,13 +14,16 @@ export const insertStaff = record(
return await db.transaction(async (tx) => {
const imgQueue: ImageTask[] = [];
const people = seed.map((x) => ({
...x.staff,
image: enqueueOptImage(imgQueue, {
url: x.staff.image,
column: staff.image,
}),
}));
const people = uniqBy(
seed.map((x) => ({
...x.staff,
image: enqueueOptImage(imgQueue, {
url: x.staff.image,
column: staff.image,
}),
})),
(x) => x.slug,
);
const ret = await tx
.insert(staff)
.select(unnestValues(people, staff))
@@ -36,7 +40,7 @@ export const insertStaff = record(
const rval = seed.map((x, i) => ({
showPk,
staffPk: ret[i].pk,
staffPk: ret.find(y => y.slug === x.staff.slug)!.pk,
kind: x.kind,
order: i,
character: {

View File

@@ -831,6 +831,9 @@ export const videosWriteH = new Elysia({ prefix: "/videos", tags: ["videos"] })
.post(
"",
async ({ body, status }) => {
if (body.length === 0) {
return status(422, { status: 422, message: "No videos" });
}
return await db.transaction(async (tx) => {
let vids: { pk: number; id: string; path: string; guess: Guess }[] = [];
try {
@@ -925,6 +928,7 @@ export const videosWriteH = new Elysia({ prefix: "/videos", tags: ["videos"] })
description:
"Invalid rendering specified. (conflicts with an existing video)",
},
422: KError,
},
},
)

View File

@@ -91,7 +91,7 @@ export const seasonRelations = relations(seasons, ({ one, many }) => ({
export const seasonTrRelations = relations(seasonTranslations, ({ one }) => ({
season: one(seasons, {
relationName: "season_translation",
relationName: "season_translations",
fields: [seasonTranslations.pk],
references: [seasons.pk],
}),

View File

@@ -28,3 +28,13 @@ export function getFile(path: string): BunFile | S3File {
return Bun.file(path);
}
export function uniqBy<T>(a: T[], key: (val: T) => string) {
const seen: Record<string, boolean> = {};
return a.filter((item) => {
const k = key(item);
if (seen[k]) return false;
seen[k] = true;
return true;
});
}

View File

@@ -104,4 +104,60 @@ describe("Serie seeding", () => {
],
});
});
it("Can create a serie with quotes", async () => {
const [resp, body] = await createSerie({
...madeInAbyss,
slug: "quote-test",
seasons: [
{
...madeInAbyss.seasons[0],
translations: {
en: {
...madeInAbyss.seasons[0].translations.en,
name: "Season'1",
},
},
},
{
...madeInAbyss.seasons[1],
translations: {
en: {
...madeInAbyss.seasons[0].translations.en,
name: 'Season"2',
description: `This's """""quote, idk'''''`,
},
},
},
],
});
expectStatus(resp, body).toBe(201);
expect(body.id).toBeString();
expect(body.slug).toBe("quote-test");
const ret = await db.query.shows.findFirst({
where: eq(shows.id, body.id),
with: {
seasons: {
orderBy: seasons.seasonNumber,
with: { translations: true },
},
entries: {
with: {
translations: true,
evj: { with: { video: true } },
},
},
},
});
expect(ret).not.toBeNull();
expect(ret!.seasons).toBeArrayOfSize(2);
expect(ret!.seasons[0].translations[0].name).toBe("Season'1");
expect(ret!.seasons[1].translations[0].name).toBe('Season"2');
expect(ret!.entries).toBeArrayOfSize(
madeInAbyss.entries.length + madeInAbyss.extras.length,
);
});
});

View File

@@ -1,6 +1,6 @@
{
"compilerOptions": {
"target": "ES2021",
"target": "ES2022",
"module": "ES2022",
"moduleResolution": "node",
"esModuleInterop": true,

View File

@@ -88,7 +88,9 @@ func setupOtel(e *echo.Echo) (func(), error) {
otel.SetTracerProvider(tp)
e.Use(otelecho.Middleware("kyoo.auth", otelecho.WithSkipper(func(c echo.Context) bool {
return c.Path() == "/auth/health" || c.Path() == "/auth/ready"
return (c.Path() == "/auth/health" ||
c.Path() == "/auth/ready" ||
strings.HasPrefix(c.Path(), "/.well-known/"))
})))
return func() {

View File

@@ -84,6 +84,7 @@ export const login = async (
export const logout = async () => {
const accounts = readAccounts();
const account = accounts.find((x) => x.selected);
removeAccounts((x) => x.selected);
if (account) {
await queryFn({
method: "DELETE",
@@ -92,7 +93,6 @@ export const logout = async () => {
parser: null,
});
}
removeAccounts((x) => x.selected);
};
export const deleteAccount = async () => {

View File

@@ -18,6 +18,7 @@ create table scanner.requests(
external_id jsonb not null default '{}'::jsonb,
videos jsonb not null default '[]'::jsonb,
status scanner.request_status not null default 'pending',
error jsonb,
started_at timestamptz,
created_at timestamptz not null default now()::timestamptz,
constraint unique_kty unique nulls not distinct (kind, title, year)

View File

@@ -24,6 +24,10 @@ async def lifespan(_):
):
# there's no way someone else used the same id, right?
is_master = await db.fetchval("select pg_try_advisory_lock(198347)")
is_http = not is_master and await db.fetchval("select pg_try_advisory_lock(645633)")
if is_http:
yield
return
if is_master:
await migrate()
processor = RequestProcessor(pool, client, tmdb)

View File

@@ -3,7 +3,7 @@ from logging import getLogger
from types import TracebackType
from typing import Literal
from aiohttp import ClientSession
from aiohttp import ClientResponse, ClientResponseError, ClientSession
from pydantic import TypeAdapter
from .models.movie import Movie
@@ -38,9 +38,19 @@ class KyooClient(metaclass=Singleton):
):
await self._client.close()
async def raise_for_status(self, r: ClientResponse):
if r.status >= 400:
raise ClientResponseError(
r.request_info,
r.history,
status=r.status,
message=await r.text(),
headers=r.headers,
)
async def get_videos_info(self) -> VideoInfo:
async with self._client.get("videos") as r:
r.raise_for_status()
await self.raise_for_status(r)
return VideoInfo(**await r.json())
async def create_videos(self, videos: list[Video]) -> list[VideoCreated]:
@@ -48,7 +58,7 @@ class KyooClient(metaclass=Singleton):
"videos",
data=TypeAdapter(list[Video]).dump_json(videos, by_alias=True),
) as r:
r.raise_for_status()
await self.raise_for_status(r)
return TypeAdapter(list[VideoCreated]).validate_json(await r.text())
async def delete_videos(self, videos: list[str] | set[str]):
@@ -56,14 +66,14 @@ class KyooClient(metaclass=Singleton):
"videos",
data=TypeAdapter(list[str] | set[str]).dump_json(videos, by_alias=True),
) as r:
r.raise_for_status()
await self.raise_for_status(r)
async def create_movie(self, movie: Movie) -> Resource:
async with self._client.post(
"movies",
data=movie.model_dump_json(by_alias=True),
) as r:
r.raise_for_status()
await self.raise_for_status(r)
return Resource.model_validate(await r.json())
async def create_serie(self, serie: Serie) -> Resource:
@@ -71,7 +81,7 @@ class KyooClient(metaclass=Singleton):
"series",
data=serie.model_dump_json(by_alias=True),
) as r:
r.raise_for_status()
await self.raise_for_status(r)
return Resource.model_validate(await r.json())
async def link_videos(
@@ -100,4 +110,4 @@ class KyooClient(metaclass=Singleton):
by_alias=True,
),
) as r:
r.raise_for_status()
await self.raise_for_status(r)

View File

@@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from typing import Any, Literal
from pydantic import Field
@@ -31,4 +31,5 @@ class RequestRet(Model):
"running",
"failed",
]
error: dict[str, Any] | None
started_at: datetime | None

View File

@@ -1,5 +1,6 @@
from asyncio import CancelledError, Event, TaskGroup
from logging import getLogger
from traceback import TracebackException
from typing import cast
from asyncpg import Connection, Pool
@@ -161,11 +162,22 @@ class RequestProcessor:
update
scanner.requests
set
status = 'failed'
status = 'failed',
error = $2
where
pk = $1
""",
request.pk,
{
"title": type(e).__name__,
"message": str(e),
"traceback": [
line
for part in TracebackException.from_exception(e).format()
for line in part.split("\n")
if line.strip()
],
},
)
return True

View File

@@ -28,6 +28,7 @@ class StatusService:
title,
year,
status,
error,
started_at
from
scanner.requests

View File

@@ -16,6 +16,9 @@ pkgs.mkShell {
# env vars aren't inherited from the `inputsFrom`
SHARP_FORCE_GLOBAL_LIBVIPS = 1;
shellHook = ''
export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH
'';
UV_PYTHON_PREFERENCE = "only-system";
UV_PYTHON = pkgs.python313;
}