Implementa o MVP do Adaptador de Planilhas no Elo Pro.
O módulo usa a identidade Supabase, isola cada workspace e registra storage, TTL, runner e biblioteca XLSX como decisões ainda abertas. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
camilafagundesandCursor committed
commit
6e23f1533b
144 files changed
+19060
No files matched your search
@@ -0,0 +1,79 @@
|
||||
import "server-only";
|
||||
import { cookies } from "next/headers";
|
||||
import { createServerClient } from "@supabase/ssr";
|
||||
import type { RequestContext, Role } from "@/domain/spreadsheet-adapter/types";
|
||||
import { getDb } from "@/infra/db/client";
|
||||
import { devAuthEnabled, verifyDevSession } from "./dev-session";
|
||||
|
||||
export type AuthState =
|
||||
| { kind: "anonymous" }
|
||||
| { kind: "no_workspace" }
|
||||
| { kind: "not_member" }
|
||||
| { kind: "ok"; ctx: RequestContext };
|
||||
|
||||
export async function resolveAuth(): Promise<AuthState> {
|
||||
const jar = await cookies();
|
||||
let userId: string | null = null;
|
||||
let workspaceId = jar.get("elo_pro_workspace_id")?.value ?? null;
|
||||
const devToken = jar.get("adapter_dev_session")?.value;
|
||||
if (devToken && devAuthEnabled()) {
|
||||
const dev = verifyDevSession(devToken);
|
||||
if (dev) {
|
||||
userId = dev.userId;
|
||||
workspaceId = dev.workspaceId;
|
||||
}
|
||||
}
|
||||
const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL;
|
||||
const supabaseKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY;
|
||||
if (!userId && supabaseUrl && supabaseKey) {
|
||||
const supabase = createServerClient(supabaseUrl, supabaseKey, {
|
||||
cookies: {
|
||||
getAll: () => jar.getAll(),
|
||||
setAll: (items) => {
|
||||
for (const item of items) jar.set(item.name, item.value, item.options);
|
||||
},
|
||||
},
|
||||
});
|
||||
const { data } = await supabase.auth.getUser();
|
||||
userId = data.user?.id ?? null;
|
||||
}
|
||||
if (!userId) return { kind: "anonymous" };
|
||||
if (!workspaceId) return { kind: "no_workspace" };
|
||||
const db = await getDb();
|
||||
const rows = await db.withUser(userId, (query) =>
|
||||
query.query<{
|
||||
display_name: string;
|
||||
is_superadmin: boolean;
|
||||
workspace_name: string;
|
||||
role: Role;
|
||||
module_enabled: boolean;
|
||||
allow_upload: boolean;
|
||||
}>(
|
||||
`select p.display_name, p.is_superadmin, w.name as workspace_name, m.role,
|
||||
coalesce(f.enabled, false) as module_enabled,
|
||||
coalesce(s.allow_non_admin_model_upload, false) as allow_upload
|
||||
from public.profiles p
|
||||
join public.workspace_members m on m.user_id = p.id and m.workspace_id = $2
|
||||
join public.workspaces w on w.id = m.workspace_id
|
||||
left join public.workspace_module_flags f on f.workspace_id = w.id and f.module_key = 'spreadsheet_adapter'
|
||||
left join public.spreadsheet_adapter_settings s on s.workspace_id = w.id
|
||||
where p.id = $1`,
|
||||
[userId, workspaceId],
|
||||
),
|
||||
);
|
||||
const row = rows[0];
|
||||
if (!row) return { kind: "not_member" };
|
||||
return {
|
||||
kind: "ok",
|
||||
ctx: {
|
||||
userId,
|
||||
displayName: row.display_name,
|
||||
isSuperadmin: row.is_superadmin,
|
||||
workspaceId,
|
||||
workspaceName: row.workspace_name,
|
||||
role: row.role,
|
||||
moduleEnabled: row.module_enabled,
|
||||
allowNonAdminModelUpload: row.allow_upload,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { createHmac, timingSafeEqual } from "node:crypto";
|
||||
|
||||
export type DevSession = { userId: string; workspaceId: string; exp: number };
|
||||
|
||||
export function devAuthEnabled(): boolean {
|
||||
return process.env.NODE_ENV !== "production" && process.env.ADAPTER_DEV_AUTH === "1" && Boolean(process.env.ADAPTER_DEV_AUTH_SECRET);
|
||||
}
|
||||
|
||||
function secret(): string {
|
||||
const value = process.env.ADAPTER_DEV_AUTH_SECRET;
|
||||
if (!value) throw new Error("ADAPTER_DEV_AUTH_SECRET ausente.");
|
||||
return value;
|
||||
}
|
||||
|
||||
export function signDevSession(userId: string, workspaceId: string, ttlSeconds = 60 * 60 * 8): string {
|
||||
const body = Buffer.from(JSON.stringify({ userId, workspaceId, exp: Date.now() + ttlSeconds * 1000 })).toString("base64url");
|
||||
const sig = createHmac("sha256", secret()).update(body).digest("base64url");
|
||||
return `${body}.${sig}`;
|
||||
}
|
||||
|
||||
export function verifyDevSession(token: string): DevSession | null {
|
||||
if (!devAuthEnabled()) return null;
|
||||
const [body, sig] = token.split(".");
|
||||
if (!body || !sig) return null;
|
||||
const expected = createHmac("sha256", secret()).update(body).digest("base64url");
|
||||
const left = Buffer.from(sig);
|
||||
const right = Buffer.from(expected);
|
||||
if (left.length !== right.length || !timingSafeEqual(left, right)) return null;
|
||||
const parsed = JSON.parse(Buffer.from(body, "base64url").toString("utf8")) as DevSession;
|
||||
if (!parsed.exp || parsed.exp < Date.now()) return null;
|
||||
return parsed;
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { AdapterError, isAdapterError } from "@/domain/spreadsheet-adapter/errors";
|
||||
|
||||
export function sha256(bytes: Buffer): string {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
export function asJson<T>(value: unknown, fallback: T): T {
|
||||
if (value == null) return fallback;
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
return JSON.parse(value) as T;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
return value as T;
|
||||
}
|
||||
|
||||
export function mapDbError(error: unknown): never {
|
||||
if (isAdapterError(error)) throw error;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (message.includes("MODEL_LIMIT")) {
|
||||
throw new AdapterError("TOO_MANY_MODELS", "Este workspace já possui 10 modelos ativos.", 409);
|
||||
}
|
||||
if (message.includes("SOURCE_LIMIT")) {
|
||||
throw new AdapterError("TOO_MANY_SOURCES", "A operação aceita no máximo 10 fontes.", 409);
|
||||
}
|
||||
if (message.includes("spreadsheet_models_active_name")) {
|
||||
throw new AdapterError("DUPLICATE_MODEL_NAME", "Já existe um modelo ativo com este nome neste workspace.", 409);
|
||||
}
|
||||
if (message.includes("spreadsheet_job_mappings") && message.toLowerCase().includes("unique")) {
|
||||
throw new AdapterError("MAPPING_CONFLICT", "O mapeamento precisa ser 1:1.", 409);
|
||||
}
|
||||
if (message.includes("MODEL_WORKSPACE_MISMATCH")) {
|
||||
throw new AdapterError("WORKSPACE_FORBIDDEN", "O modelo não pertence ao workspace ativo.", 403);
|
||||
}
|
||||
if (message.toLowerCase().includes("row-level security") || message.toLowerCase().includes("permission denied")) {
|
||||
throw new AdapterError("FORBIDDEN", "Operação não autorizada para este perfil.", 403);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { Db } from "@/infra/db/types";
|
||||
import type { Coordinator } from "@/infra/redis/memory";
|
||||
import type { ObjectStorage } from "@/infra/storage/memory";
|
||||
|
||||
export type AppDeps = {
|
||||
db: Db;
|
||||
storage: ObjectStorage;
|
||||
coordinator: Coordinator;
|
||||
now?: () => Date;
|
||||
ttlSeconds?: number;
|
||||
capture?: (error: unknown, tags: Record<string, string>) => void;
|
||||
};
|
||||
@@ -0,0 +1,67 @@
|
||||
import type { ColumnSchema, JobProgress, JobStatus, MappingInput, SourceStatus } from "@/domain/spreadsheet-adapter/types";
|
||||
|
||||
export type MappingDto = {
|
||||
sourceColumn: string;
|
||||
modelColumn: string;
|
||||
isRequired: boolean;
|
||||
position: number;
|
||||
};
|
||||
|
||||
export type ModelDto = {
|
||||
id: string;
|
||||
name: string;
|
||||
originalFilename: string;
|
||||
format: "xlsx" | "csv";
|
||||
sheetName: string | null;
|
||||
headerRow: number;
|
||||
columns: ColumnSchema[];
|
||||
createdAt: string;
|
||||
createdByName: string;
|
||||
};
|
||||
|
||||
export type SourceDto = {
|
||||
id: string;
|
||||
sourceName: string;
|
||||
status: SourceStatus;
|
||||
selectedSheet: string | null;
|
||||
sheets: string[];
|
||||
headerRow: number;
|
||||
rowCount: number;
|
||||
sizeBytes: number;
|
||||
errorCode: string | null;
|
||||
warnings: string[];
|
||||
columns: ColumnSchema[];
|
||||
mappings: MappingDto[];
|
||||
expiresAt: string;
|
||||
hasOutput: boolean;
|
||||
};
|
||||
|
||||
export type JobDto = {
|
||||
id: string;
|
||||
status: JobStatus;
|
||||
errorCode: string | null;
|
||||
progress: JobProgress | null;
|
||||
currentSource: number;
|
||||
model: { id: string; name: string; columns: ColumnSchema[] };
|
||||
sources: SourceDto[];
|
||||
};
|
||||
|
||||
export type SourceSchemaPayload = {
|
||||
columns: ColumnSchema[];
|
||||
sheets: string[];
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
export type UploadFile = {
|
||||
filename: string;
|
||||
bytes: Buffer;
|
||||
};
|
||||
|
||||
export type ModelCreateInput = UploadFile & {
|
||||
name: string;
|
||||
sheet?: string | null;
|
||||
headerRow?: number;
|
||||
requiredColumns?: string[];
|
||||
};
|
||||
|
||||
export type { MappingInput };
|
||||
@@ -0,0 +1,316 @@
|
||||
import { AdapterError, isAdapterError } from "@/domain/spreadsheet-adapter/errors";
|
||||
import { neutralizeRow, neutralizeSpreadsheetFormula } from "@/domain/spreadsheet-adapter/csv-injection";
|
||||
import { buildOutputFilename } from "@/domain/spreadsheet-adapter/filenames";
|
||||
import { validateMappings } from "@/domain/spreadsheet-adapter/mapping";
|
||||
import { canUseSpreadsheetAdapter } from "@/domain/spreadsheet-adapter/permissions";
|
||||
import { transformRows } from "@/domain/spreadsheet-adapter/transform";
|
||||
import type { JobProgress, OutputFormat, RequestContext } from "@/domain/spreadsheet-adapter/types";
|
||||
import { buildCsv, buildXlsx } from "@/infra/export/tabular";
|
||||
import { buildPdf } from "@/infra/export/pdf";
|
||||
import type { Query } from "@/infra/db/types";
|
||||
import { lockKey } from "@/infra/redis/redis";
|
||||
import { sanitizeAuditMetadata } from "@/infra/observability/scrub";
|
||||
import { parseCsvText } from "@/domain/spreadsheet-adapter/csv";
|
||||
import { readSpreadsheet } from "@/infra/spreadsheet/parse";
|
||||
import { asJson, mapDbError, sha256 } from "./db-utils";
|
||||
import type { AppDeps } from "./deps";
|
||||
import { readJob } from "./read-model";
|
||||
|
||||
function assertUse(ctx: RequestContext): void {
|
||||
if (!ctx.moduleEnabled || !canUseSpreadsheetAdapter(ctx)) {
|
||||
throw new AdapterError("FEATURE_DISABLED", "O Adaptador não está habilitado neste workspace.", 403);
|
||||
}
|
||||
}
|
||||
|
||||
async function audit(
|
||||
query: Query,
|
||||
ctx: RequestContext,
|
||||
action: string,
|
||||
resourceId: string,
|
||||
result: "success" | "error",
|
||||
metadata?: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
await query.query(
|
||||
`insert into public.audit_log (workspace_id, actor_id, action, resource_type, resource_id, result, metadata)
|
||||
values ($1,$2,$3,'job',$4,$5,$6::jsonb)`,
|
||||
[ctx.workspaceId, ctx.userId, action, resourceId, result, JSON.stringify(sanitizeAuditMetadata(metadata))],
|
||||
);
|
||||
}
|
||||
|
||||
function assertTempKey(key: string): void {
|
||||
if (!key.includes("/tmp/") || key.includes("/models/") || key.includes("..")) {
|
||||
throw new AdapterError("PATH_TRAVERSAL", "Chave temporária rejeitada.");
|
||||
}
|
||||
}
|
||||
|
||||
export async function runProcess(deps: AppDeps, ctx: RequestContext, jobId: string, sourceId: string): Promise<void> {
|
||||
const userLock = lockKey(ctx.userId);
|
||||
const written: string[] = [];
|
||||
try {
|
||||
const owns = await deps.coordinator.ownsLock(userLock, jobId);
|
||||
if (!owns) {
|
||||
const acquired = await deps.coordinator.acquireLock(userLock, jobId, 120);
|
||||
if (!acquired) throw new AdapterError("JOB_LOCKED", "Já existe uma transferência em andamento para este usuário.", 409);
|
||||
}
|
||||
const loaded = await deps.db.withUser(ctx.userId, async (query) => {
|
||||
const rows = await query.query<{
|
||||
storage_key: string;
|
||||
content_sha256: string;
|
||||
model_name: string;
|
||||
columns_schema: unknown;
|
||||
temp_storage_key: string;
|
||||
source_name: string;
|
||||
selected_sheet: string | null;
|
||||
header_row: number;
|
||||
source_count: number;
|
||||
}>(
|
||||
`select m.storage_key, m.content_sha256, m.name as model_name, m.columns_schema,
|
||||
s.temp_storage_key, s.source_name, s.selected_sheet, s.header_row, j.source_count
|
||||
from public.spreadsheet_jobs j
|
||||
join public.spreadsheet_models m on m.id = j.model_id
|
||||
join public.spreadsheet_job_sources s on s.job_id = j.id
|
||||
where j.id = $1 and j.workspace_id = $2 and j.user_id = $3 and s.id = $4`,
|
||||
[jobId, ctx.workspaceId, ctx.userId, sourceId],
|
||||
);
|
||||
const mappings = await query.query<{ source_column: string; model_column: string }>(
|
||||
`select source_column, model_column from public.spreadsheet_job_mappings where job_source_id = $1 order by position`,
|
||||
[sourceId],
|
||||
);
|
||||
return { row: rows[0], mappings };
|
||||
});
|
||||
if (!loaded.row) throw new AdapterError("NOT_FOUND", "Fonte da operação não encontrada.", 404);
|
||||
const columns = asJson<{ name: string; required: boolean }[]>(loaded.row.columns_schema, []);
|
||||
const validation = validateMappings(
|
||||
columns,
|
||||
[],
|
||||
loaded.mappings.map((mapping) => ({ sourceColumn: mapping.source_column, modelColumn: mapping.model_column })),
|
||||
);
|
||||
const modelBytes = await deps.storage.get(loaded.row.storage_key);
|
||||
if (sha256(modelBytes) !== loaded.row.content_sha256) {
|
||||
throw new AdapterError("MODEL_TAMPERED", "O arquivo do modelo não confere com o cadastro.", 409);
|
||||
}
|
||||
const sourceBytes = await deps.storage.get(loaded.row.temp_storage_key);
|
||||
const parsed = await readSpreadsheet(loaded.row.source_name, sourceBytes, {
|
||||
sheet: loaded.row.selected_sheet,
|
||||
headerRow: loaded.row.header_row,
|
||||
});
|
||||
const checked = validateMappings(
|
||||
columns,
|
||||
parsed.grid.headers,
|
||||
loaded.mappings.map((mapping) => ({ sourceColumn: mapping.source_column, modelColumn: mapping.model_column })),
|
||||
);
|
||||
if (!checked.ok || validation.missingRequired.length) {
|
||||
throw new AdapterError(checked.code, checked.missingRequired.length ? "Mapeamento obrigatório incompleto." : "Mapeamento inválido.", 422);
|
||||
}
|
||||
if (parsed.grid.columns.some((column) => column.inferredType === "mixed")) {
|
||||
throw new AdapterError("TYPE_INCONSISTENT", "A fonte mudou e agora tem tipos inconsistentes.", 422);
|
||||
}
|
||||
const sourceIndex = 1;
|
||||
const total = Number(loaded.row.source_count) || 1;
|
||||
const progress = async (value: JobProgress) => {
|
||||
await deps.coordinator.setProgress(jobId, value).catch(() => undefined);
|
||||
await deps.coordinator.renewLock(userLock, jobId, 120).catch(() => undefined);
|
||||
};
|
||||
await progress({ phase: "processing", label: "Processando", percent: 0, sourceIndex, sourceCount: total, sourceId });
|
||||
const transformed = transformRows(
|
||||
columns.map((column) => column.name),
|
||||
parsed.grid.headers,
|
||||
parsed.grid.rows,
|
||||
loaded.mappings.map((mapping) => ({ sourceColumn: mapping.source_column, modelColumn: mapping.model_column })),
|
||||
).map(neutralizeRow);
|
||||
const headers = columns.map((column) => neutralizeSpreadsheetFormula(column.name));
|
||||
let done = 0;
|
||||
for (const _row of transformed) {
|
||||
done += 1;
|
||||
if (done % 5000 === 0) {
|
||||
await progress({
|
||||
phase: "processing",
|
||||
label: "Processando",
|
||||
percent: Math.round((done / transformed.length) * 100),
|
||||
sourceIndex,
|
||||
sourceCount: total,
|
||||
sourceId,
|
||||
});
|
||||
}
|
||||
}
|
||||
await progress({ phase: "generating", label: "Gerando arquivo", percent: null, sourceIndex, sourceCount: total, sourceId });
|
||||
const csvKey = `workspaces/${ctx.workspaceId}/tmp/${jobId}/${sourceId}/out.csv`;
|
||||
const xlsxKey = `workspaces/${ctx.workspaceId}/tmp/${jobId}/${sourceId}/out.xlsx`;
|
||||
assertTempKey(csvKey);
|
||||
assertTempKey(xlsxKey);
|
||||
const csv = buildCsv(headers, transformed);
|
||||
const xlsx = await buildXlsx(headers, transformed);
|
||||
await deps.storage.put(csvKey, csv);
|
||||
written.push(csvKey);
|
||||
await deps.storage.put(xlsxKey, xlsx);
|
||||
written.push(xlsxKey);
|
||||
const after = await deps.storage.get(loaded.row.storage_key);
|
||||
if (!after.equals(modelBytes)) {
|
||||
throw new AdapterError("MODEL_TAMPERED", "O modelo original foi modificado durante a transferência.", 500);
|
||||
}
|
||||
const finished: JobProgress = {
|
||||
phase: "completed",
|
||||
label: "Concluído",
|
||||
percent: 100,
|
||||
sourceIndex,
|
||||
sourceCount: total,
|
||||
sourceId,
|
||||
};
|
||||
await deps.db.withUser(ctx.userId, async (query) => {
|
||||
await query.query(
|
||||
`update public.spreadsheet_job_sources
|
||||
set status = 'completed', output_csv_key = $2, output_xlsx_key = $3, processed_at = now(), error_code = null
|
||||
where id = $1`,
|
||||
[sourceId, csvKey, xlsxKey],
|
||||
);
|
||||
const pending = await query.query<{ total: number }>(
|
||||
`select count(*)::int as total from public.spreadsheet_job_sources where job_id = $1 and status <> 'completed'`,
|
||||
[jobId],
|
||||
);
|
||||
const status = Number(pending[0]?.total ?? 0) === 0 ? "completed" : "ready";
|
||||
await query.query(
|
||||
`update public.spreadsheet_jobs
|
||||
set status = $2, progress = $3::jsonb, error_code = null, finished_at = case when $2 = 'completed' then now() else finished_at end, updated_at = now()
|
||||
where id = $1`,
|
||||
[jobId, status, JSON.stringify(finished)],
|
||||
);
|
||||
await audit(query, ctx, "job.completed", jobId, "success", { source_id: sourceId, output_rows: transformed.length });
|
||||
});
|
||||
await deps.coordinator.setProgress(jobId, finished).catch(() => undefined);
|
||||
} catch (error) {
|
||||
for (const key of written) await deps.storage.delete(key).catch(() => undefined);
|
||||
const code = isAdapterError(error) ? error.code : "PROCESS_FAILED";
|
||||
const failed: JobProgress = {
|
||||
phase: "failed",
|
||||
label: "Falhou",
|
||||
percent: null,
|
||||
sourceIndex: 1,
|
||||
sourceCount: 1,
|
||||
sourceId,
|
||||
};
|
||||
await deps.db
|
||||
.withUser(ctx.userId, async (query) => {
|
||||
await query.query(`update public.spreadsheet_job_sources set status = 'failed', error_code = $2 where id = $1 and status <> 'completed'`, [
|
||||
sourceId,
|
||||
code,
|
||||
]);
|
||||
await query.query(
|
||||
`update public.spreadsheet_jobs set status = 'failed', error_code = $2, progress = $3::jsonb, finished_at = now(), updated_at = now() where id = $1`,
|
||||
[jobId, code, JSON.stringify(failed)],
|
||||
);
|
||||
await audit(query, ctx, "job.failed", jobId, "error", { error_code: code, source_id: sourceId });
|
||||
})
|
||||
.catch(() => undefined);
|
||||
await deps.coordinator.setProgress(jobId, failed).catch(() => undefined);
|
||||
if (!isAdapterError(error) || error.status >= 500) {
|
||||
deps.capture?.(error, { module: "spreadsheet-adapter", job_id: jobId, workspace_id: ctx.workspaceId, error_code: code });
|
||||
}
|
||||
if (isAdapterError(error)) throw error;
|
||||
throw new AdapterError("PROCESS_FAILED", "O processamento falhou. Você pode tentar novamente.", 500);
|
||||
} finally {
|
||||
await deps.coordinator.releaseLock(userLock, jobId).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
const CONTENT_TYPE: Record<OutputFormat, string> = {
|
||||
xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
csv: "text/csv; charset=utf-8",
|
||||
pdf: "application/pdf",
|
||||
};
|
||||
|
||||
export async function downloadOutput(deps: AppDeps, ctx: RequestContext, jobId: string, sourceId: string, format: OutputFormat) {
|
||||
assertUse(ctx);
|
||||
const file = await deps.db.withUser(ctx.userId, async (query) => {
|
||||
const rows = await query.query<{
|
||||
model_name: string;
|
||||
output_csv_key: string | null;
|
||||
output_xlsx_key: string | null;
|
||||
expires_at: string | Date;
|
||||
status: string;
|
||||
}>(
|
||||
`select m.name as model_name, s.output_csv_key, s.output_xlsx_key, s.expires_at, s.status
|
||||
from public.spreadsheet_job_sources s
|
||||
join public.spreadsheet_jobs j on j.id = s.job_id
|
||||
join public.spreadsheet_models m on m.id = j.model_id
|
||||
where s.id = $1 and j.id = $2 and j.workspace_id = $3 and j.user_id = $4 and s.expires_at > $5::timestamptz`,
|
||||
[sourceId, jobId, ctx.workspaceId, ctx.userId, (deps.now ?? (() => new Date()))().toISOString()],
|
||||
);
|
||||
if (!rows[0] || rows[0].status !== "completed") throw new AdapterError("NOT_FOUND", "Saída indisponível ou expirada.", 404);
|
||||
await audit(query, ctx, "job.download", jobId, "success", { format, source_id: sourceId });
|
||||
return rows[0];
|
||||
});
|
||||
const filename = buildOutputFilename(file.model_name, format, deps.now?.() ?? new Date());
|
||||
if (format === "pdf") {
|
||||
if (!file.output_csv_key) throw new AdapterError("NOT_FOUND", "Saída indisponível.", 404);
|
||||
assertTempKey(file.output_csv_key);
|
||||
const csv = (await deps.storage.get(file.output_csv_key)).toString("utf8");
|
||||
const grid = parseCsvText(csv);
|
||||
const headers = grid[0] ?? [];
|
||||
const body = await buildPdf(file.model_name, headers, grid.slice(1));
|
||||
return { filename, contentType: CONTENT_TYPE.pdf, body: body.body };
|
||||
}
|
||||
const key = format === "csv" ? file.output_csv_key : file.output_xlsx_key;
|
||||
if (!key) throw new AdapterError("NOT_FOUND", "Saída indisponível.", 404);
|
||||
assertTempKey(key);
|
||||
return { filename, contentType: CONTENT_TYPE[format], body: await deps.storage.get(key) };
|
||||
}
|
||||
|
||||
export async function cleanupExpired(deps: AppDeps): Promise<number> {
|
||||
const rows = await deps.db.withService((query) =>
|
||||
query.query<{ id: string; temp_storage_key: string; output_csv_key: string | null; output_xlsx_key: string | null }>(
|
||||
`select id, temp_storage_key, output_csv_key, output_xlsx_key
|
||||
from public.spreadsheet_job_sources
|
||||
where expires_at < $1::timestamptz and status <> 'expired'`,
|
||||
[(deps.now ?? (() => new Date()))().toISOString()],
|
||||
),
|
||||
);
|
||||
for (const row of rows) {
|
||||
for (const key of [row.temp_storage_key, row.output_csv_key, row.output_xlsx_key]) {
|
||||
if (!key || !key.includes("/tmp/") || key.includes("/models/")) continue;
|
||||
await deps.storage.delete(key).catch(() => undefined);
|
||||
}
|
||||
await deps.db.withService((query) =>
|
||||
query.query(`update public.spreadsheet_job_sources set status = 'expired', output_csv_key = null, output_xlsx_key = null where id = $1`, [row.id]),
|
||||
);
|
||||
}
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
export async function recoverStaleJobs(deps: AppDeps): Promise<number> {
|
||||
const rows = await deps.db.withService((query) =>
|
||||
query.query<{ id: string; user_id: string; workspace_id: string }>(
|
||||
`select id, user_id, workspace_id from public.spreadsheet_jobs where status = 'processing'`,
|
||||
),
|
||||
);
|
||||
let recovered = 0;
|
||||
for (const row of rows) {
|
||||
const owns = await deps.coordinator.ownsLock(lockKey(row.user_id), row.id).catch(() => false);
|
||||
if (owns) continue;
|
||||
await deps.db.withService((query) =>
|
||||
query.query(
|
||||
`update public.spreadsheet_jobs set status = 'failed', error_code = 'INTERRUPTED', updated_at = now() where id = $1 and status = 'processing'`,
|
||||
[row.id],
|
||||
),
|
||||
);
|
||||
recovered += 1;
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
export async function readJobForWorker(deps: AppDeps, jobId: string) {
|
||||
return deps.db.withService((query) =>
|
||||
query.query<{ user_id: string; workspace_id: string; progress: unknown }>(
|
||||
`select user_id, workspace_id, progress from public.spreadsheet_jobs where id = $1 and status = 'queued'`,
|
||||
[jobId],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export async function listQueuedJobIds(deps: AppDeps): Promise<string[]> {
|
||||
const rows = await deps.db.withService((query) =>
|
||||
query.query<{ id: string }>(`select id from public.spreadsheet_jobs where status = 'queued' order by created_at asc limit 5`),
|
||||
);
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
export { mapDbError, readJob };
|
||||
@@ -0,0 +1,163 @@
|
||||
import type { Query } from "@/infra/db/types";
|
||||
import { asJson } from "./db-utils";
|
||||
import type { RequestContext } from "@/domain/spreadsheet-adapter/types";
|
||||
import type { JobProgress } from "@/domain/spreadsheet-adapter/types";
|
||||
import { AdapterError } from "@/domain/spreadsheet-adapter/errors";
|
||||
import type { ColumnSchema, JobStatus, SourceStatus } from "@/domain/spreadsheet-adapter/types";
|
||||
import type { Coordinator } from "@/infra/redis/memory";
|
||||
import type { JobDto, MappingDto, ModelDto, SourceDto, SourceSchemaPayload } from "./dto";
|
||||
|
||||
function iso(value: unknown): string {
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
const date = new Date(String(value));
|
||||
return Number.isNaN(date.getTime()) ? String(value) : date.toISOString();
|
||||
}
|
||||
|
||||
function payloadOf(value: unknown): SourceSchemaPayload {
|
||||
const parsed = asJson<SourceSchemaPayload | ColumnSchema[]>(value, { columns: [], sheets: [], warnings: [] });
|
||||
if (Array.isArray(parsed)) return { columns: parsed, sheets: [], warnings: [] };
|
||||
return { columns: parsed.columns ?? [], sheets: parsed.sheets ?? [], warnings: parsed.warnings ?? [] };
|
||||
}
|
||||
|
||||
type JobRow = {
|
||||
id: string;
|
||||
status: JobStatus;
|
||||
error_code: string | null;
|
||||
progress: unknown;
|
||||
current_source: number;
|
||||
model_id: string;
|
||||
model_name: string;
|
||||
model_columns: unknown;
|
||||
};
|
||||
|
||||
type SourceRow = {
|
||||
id: string;
|
||||
source_name: string;
|
||||
status: SourceStatus;
|
||||
selected_sheet: string | null;
|
||||
header_row: number;
|
||||
row_count: number;
|
||||
size_bytes: number;
|
||||
error_code: string | null;
|
||||
columns_schema: unknown;
|
||||
output_csv_key: string | null;
|
||||
expires_at: unknown;
|
||||
};
|
||||
|
||||
export async function readJob(query: Query, coordinator: Coordinator, ctx: RequestContext, jobId: string): Promise<JobDto> {
|
||||
const jobs = await query.query<JobRow>(
|
||||
`select j.id, j.status, j.error_code, j.progress, j.current_source, j.model_id,
|
||||
m.name as model_name, m.columns_schema as model_columns
|
||||
from public.spreadsheet_jobs j
|
||||
join public.spreadsheet_models m on m.id = j.model_id
|
||||
where j.id = $1 and j.workspace_id = $2 and j.user_id = $3`,
|
||||
[jobId, ctx.workspaceId, ctx.userId],
|
||||
);
|
||||
const job = jobs[0];
|
||||
if (!job) throw new AdapterError("NOT_FOUND", "Operação não encontrada.", 404);
|
||||
const sources = await query.query<SourceRow>(
|
||||
`select id, source_name, status, selected_sheet, header_row, row_count, size_bytes, error_code,
|
||||
columns_schema, output_csv_key, expires_at
|
||||
from public.spreadsheet_job_sources
|
||||
where job_id = $1
|
||||
order by created_at asc`,
|
||||
[jobId],
|
||||
);
|
||||
const mappings = await query.query<{
|
||||
job_source_id: string;
|
||||
source_column: string;
|
||||
model_column: string;
|
||||
is_required: boolean;
|
||||
position: number;
|
||||
}>(
|
||||
`select mp.job_source_id, mp.source_column, mp.model_column, mp.is_required, mp.position
|
||||
from public.spreadsheet_job_mappings mp
|
||||
join public.spreadsheet_job_sources s on s.id = mp.job_source_id
|
||||
where s.job_id = $1
|
||||
order by mp.position asc`,
|
||||
[jobId],
|
||||
);
|
||||
const bySource = new Map<string, MappingDto[]>();
|
||||
for (const mapping of mappings) {
|
||||
const list = bySource.get(mapping.job_source_id) ?? [];
|
||||
list.push({
|
||||
sourceColumn: mapping.source_column,
|
||||
modelColumn: mapping.model_column,
|
||||
isRequired: mapping.is_required,
|
||||
position: mapping.position,
|
||||
});
|
||||
bySource.set(mapping.job_source_id, list);
|
||||
}
|
||||
let progress = asJson<JobProgress | null>(job.progress, null);
|
||||
if (job.status === "processing" || job.status === "queued") {
|
||||
try {
|
||||
progress = (await coordinator.getProgress(job.id)) ?? progress;
|
||||
} catch {
|
||||
progress = progress ?? null;
|
||||
}
|
||||
}
|
||||
return {
|
||||
id: job.id,
|
||||
status: job.status,
|
||||
errorCode: job.error_code,
|
||||
progress,
|
||||
currentSource: job.current_source,
|
||||
model: {
|
||||
id: job.model_id,
|
||||
name: job.model_name,
|
||||
columns: asJson<ColumnSchema[]>(job.model_columns, []),
|
||||
},
|
||||
sources: sources.map((source) => {
|
||||
const schema = payloadOf(source.columns_schema);
|
||||
return {
|
||||
id: source.id,
|
||||
sourceName: source.source_name,
|
||||
status: source.status,
|
||||
selectedSheet: source.selected_sheet,
|
||||
sheets: schema.sheets,
|
||||
headerRow: source.header_row,
|
||||
rowCount: source.row_count,
|
||||
sizeBytes: Number(source.size_bytes),
|
||||
errorCode: source.error_code,
|
||||
warnings: schema.warnings,
|
||||
columns: schema.columns,
|
||||
mappings: bySource.get(source.id) ?? [],
|
||||
expiresAt: iso(source.expires_at),
|
||||
hasOutput: Boolean(source.output_csv_key),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function readModels(query: Query, workspaceId: string): Promise<ModelDto[]> {
|
||||
const rows = await query.query<{
|
||||
id: string;
|
||||
name: string;
|
||||
original_filename: string;
|
||||
format: "xlsx" | "csv";
|
||||
sheet_name: string | null;
|
||||
header_row: number;
|
||||
columns_schema: unknown;
|
||||
created_at: unknown;
|
||||
created_by_name: string;
|
||||
}>(
|
||||
`select m.id, m.name, m.original_filename, m.format, m.sheet_name, m.header_row, m.columns_schema,
|
||||
m.created_at, p.display_name as created_by_name
|
||||
from public.spreadsheet_models m
|
||||
join public.profiles p on p.id = m.created_by
|
||||
where m.workspace_id = $1 and m.status = 'active'
|
||||
order by lower(m.name) asc`,
|
||||
[workspaceId],
|
||||
);
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
originalFilename: row.original_filename,
|
||||
format: row.format,
|
||||
sheetName: row.sheet_name,
|
||||
headerRow: row.header_row,
|
||||
columns: asJson<ColumnSchema[]>(row.columns_schema, []),
|
||||
createdAt: iso(row.created_at),
|
||||
createdByName: row.created_by_name,
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import "server-only";
|
||||
import { getDb } from "@/infra/db/client";
|
||||
import { createCoordinator } from "@/infra/redis";
|
||||
import { createStorage } from "@/infra/storage";
|
||||
import { captureAdapterException } from "@/infra/observability/capture";
|
||||
import type { AppDeps } from "./deps";
|
||||
|
||||
export async function runtimeDeps(): Promise<AppDeps> {
|
||||
return {
|
||||
db: await getDb(),
|
||||
storage: createStorage(),
|
||||
coordinator: createCoordinator(),
|
||||
capture: captureAdapterException,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { AdapterError, isAdapterError } from "@/domain/spreadsheet-adapter/errors";
|
||||
import { MAX_ACTIVE_MODELS, MAX_SOURCES_PER_JOB } from "@/domain/spreadsheet-adapter/limits";
|
||||
import { validateMappings } from "@/domain/spreadsheet-adapter/mapping";
|
||||
import { buildPreview } from "@/domain/spreadsheet-adapter/preview";
|
||||
import {
|
||||
canManageSpreadsheetModels,
|
||||
canOperateSpreadsheet,
|
||||
canUploadSpreadsheetModel,
|
||||
canUseSpreadsheetAdapter,
|
||||
} from "@/domain/spreadsheet-adapter/permissions";
|
||||
import { normalizeModelName, sanitizeDisplayName } from "@/domain/spreadsheet-adapter/filenames";
|
||||
import type { RequestContext } from "@/domain/spreadsheet-adapter/types";
|
||||
import type { Query } from "@/infra/db/types";
|
||||
import { lockKey } from "@/infra/redis/redis";
|
||||
import type { AppDeps } from "./deps";
|
||||
import { readSpreadsheet } from "@/infra/spreadsheet/parse";
|
||||
import { sanitizeAuditMetadata } from "@/infra/observability/scrub";
|
||||
import { tempTtlSeconds } from "@/config/retention";
|
||||
import type { JobDto, MappingInput, ModelCreateInput, ModelDto, UploadFile } from "./dto";
|
||||
import { mapDbError, sha256 } from "./db-utils";
|
||||
import { readJob, readModels } from "./read-model";
|
||||
import { runProcess } from "./process-job";
|
||||
|
||||
function ttl(deps: AppDeps): number {
|
||||
return deps.ttlSeconds ?? tempTtlSeconds();
|
||||
}
|
||||
|
||||
function assertUse(ctx: RequestContext): void {
|
||||
if (!ctx.moduleEnabled || !canUseSpreadsheetAdapter(ctx)) {
|
||||
throw new AdapterError("FEATURE_DISABLED", "O Adaptador não está habilitado neste workspace.", 403);
|
||||
}
|
||||
if (!canOperateSpreadsheet(ctx)) throw new AdapterError("FORBIDDEN", "Sem permissão para operar o Adaptador.", 403);
|
||||
}
|
||||
|
||||
async function audit(
|
||||
query: Query,
|
||||
ctx: RequestContext,
|
||||
action: string,
|
||||
resourceType: string,
|
||||
resourceId: string,
|
||||
result: "success" | "denied" | "error",
|
||||
metadata?: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
await query.query(
|
||||
`insert into public.audit_log (workspace_id, actor_id, action, resource_type, resource_id, result, metadata)
|
||||
values ($1, $2, $3, $4, $5, $6, $7::jsonb)`,
|
||||
[ctx.workspaceId, ctx.userId, action, resourceType, resourceId, result, JSON.stringify(sanitizeAuditMetadata(metadata))],
|
||||
);
|
||||
}
|
||||
|
||||
export async function rejectWorkspaceForgery(deps: AppDeps, ctx: RequestContext, incoming: unknown): Promise<void> {
|
||||
if (typeof incoming !== "string" || incoming.length === 0 || incoming === ctx.workspaceId) return;
|
||||
await deps.db.withUser(ctx.userId, (query) =>
|
||||
audit(query, ctx, "security.workspace_forgery", "workspace", ctx.workspaceId, "denied", {
|
||||
attempted_workspace_id: incoming.slice(0, 80),
|
||||
}),
|
||||
);
|
||||
throw new AdapterError("WORKSPACE_FORBIDDEN", "O workspace informado não é o workspace ativo.", 403);
|
||||
}
|
||||
|
||||
export async function listModels(deps: AppDeps, ctx: RequestContext): Promise<ModelDto[]> {
|
||||
assertUse(ctx);
|
||||
return deps.db.withUser(ctx.userId, (query) => readModels(query, ctx.workspaceId));
|
||||
}
|
||||
|
||||
export async function getSettings(deps: AppDeps, ctx: RequestContext): Promise<{ allowNonAdminModelUpload: boolean }> {
|
||||
assertUse(ctx);
|
||||
const rows = await deps.db.withUser(ctx.userId, (query) =>
|
||||
query.query<{ allow_non_admin_model_upload: boolean }>(
|
||||
`select allow_non_admin_model_upload from public.spreadsheet_adapter_settings where workspace_id = $1`,
|
||||
[ctx.workspaceId],
|
||||
),
|
||||
);
|
||||
return { allowNonAdminModelUpload: Boolean(rows[0]?.allow_non_admin_model_upload) };
|
||||
}
|
||||
|
||||
export async function updateSettings(deps: AppDeps, ctx: RequestContext, allowNonAdminModelUpload: boolean): Promise<void> {
|
||||
assertUse(ctx);
|
||||
if (!canManageSpreadsheetModels(ctx)) {
|
||||
throw new AdapterError("FORBIDDEN", "Somente owner ou admin alteram a configuração do módulo.", 403);
|
||||
}
|
||||
await deps.db.withUser(ctx.userId, async (query) => {
|
||||
await query.query(
|
||||
`insert into public.spreadsheet_adapter_settings (workspace_id, allow_non_admin_model_upload)
|
||||
values ($1, $2)
|
||||
on conflict (workspace_id) do update
|
||||
set allow_non_admin_model_upload = excluded.allow_non_admin_model_upload, updated_at = now()`,
|
||||
[ctx.workspaceId, allowNonAdminModelUpload],
|
||||
);
|
||||
await audit(query, ctx, "settings.upload_policy_changed", "settings", ctx.workspaceId, "success", {
|
||||
allow_non_admin_model_upload: allowNonAdminModelUpload,
|
||||
});
|
||||
}).catch(mapDbError);
|
||||
}
|
||||
|
||||
export async function createModel(deps: AppDeps, ctx: RequestContext, input: ModelCreateInput): Promise<ModelDto> {
|
||||
assertUse(ctx);
|
||||
if (!canUploadSpreadsheetModel(ctx)) {
|
||||
throw new AdapterError("FORBIDDEN", "Seu perfil não pode cadastrar modelos neste workspace.", 403);
|
||||
}
|
||||
const name = normalizeModelName(input.name);
|
||||
const filename = sanitizeDisplayName(input.filename);
|
||||
const parsed = await readSpreadsheet(filename, input.bytes, { sheet: input.sheet, headerRow: input.headerRow ?? 1 });
|
||||
if (parsed.grid.columns.some((column) => column.inferredType === "mixed")) {
|
||||
throw new AdapterError("TYPE_INCONSISTENT", "O modelo tem colunas com tipos inconsistentes. Ajuste a origem antes de cadastrar.", 422);
|
||||
}
|
||||
let columns = parsed.grid.columns;
|
||||
if (input.requiredColumns) {
|
||||
const known = new Set(columns.map((column) => column.name));
|
||||
for (const required of input.requiredColumns) {
|
||||
if (!known.has(required)) throw new AdapterError("UNKNOWN_COLUMN", "Uma coluna obrigatória não existe no modelo.");
|
||||
}
|
||||
const required = new Set(input.requiredColumns);
|
||||
columns = columns.map((column) => ({ ...column, required: required.has(column.name) }));
|
||||
}
|
||||
const id = randomUUID();
|
||||
const storageKey = `workspaces/${ctx.workspaceId}/models/${id}/original`;
|
||||
const digest = sha256(input.bytes);
|
||||
await deps.storage.put(storageKey, input.bytes);
|
||||
try {
|
||||
await deps.db.withUser(ctx.userId, async (query) => {
|
||||
const counts = await query.query<{ total: number }>(
|
||||
`select count(*)::int as total from public.spreadsheet_models where workspace_id = $1 and status = 'active'`,
|
||||
[ctx.workspaceId],
|
||||
);
|
||||
if (Number(counts[0]?.total ?? 0) >= MAX_ACTIVE_MODELS) {
|
||||
throw new AdapterError("TOO_MANY_MODELS", "Este workspace já possui 10 modelos ativos.", 409);
|
||||
}
|
||||
await query.query(
|
||||
`insert into public.spreadsheet_models
|
||||
(id, workspace_id, name, original_filename, format, storage_key, content_sha256, sheet_name, header_row, columns_schema, created_by, status)
|
||||
values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::jsonb,$11,'active')`,
|
||||
[
|
||||
id,
|
||||
ctx.workspaceId,
|
||||
name,
|
||||
filename,
|
||||
parsed.format,
|
||||
storageKey,
|
||||
digest,
|
||||
parsed.grid.selectedSheet,
|
||||
parsed.grid.headerRow,
|
||||
JSON.stringify(columns),
|
||||
ctx.userId,
|
||||
],
|
||||
);
|
||||
await audit(query, ctx, "model.uploaded", "model", id, "success", { format: parsed.format, column_count: columns.length });
|
||||
});
|
||||
} catch (error) {
|
||||
await deps.storage.delete(storageKey);
|
||||
mapDbError(error);
|
||||
}
|
||||
const models = await listModels(deps, ctx);
|
||||
const created = models.find((model) => model.id === id);
|
||||
if (!created) throw new AdapterError("NOT_FOUND", "Modelo não encontrado após o cadastro.", 500);
|
||||
return created;
|
||||
}
|
||||
|
||||
export async function deleteModel(deps: AppDeps, ctx: RequestContext, modelId: string): Promise<void> {
|
||||
assertUse(ctx);
|
||||
if (!canManageSpreadsheetModels(ctx)) {
|
||||
throw new AdapterError("FORBIDDEN", "Somente owner ou admin podem excluir modelos.", 403);
|
||||
}
|
||||
await deps.db.withUser(ctx.userId, async (query) => {
|
||||
const active = await query.query<{ busy: boolean }>(`select public.model_has_active_job($1::uuid) as busy`, [modelId]);
|
||||
if (active[0]?.busy) {
|
||||
throw new AdapterError("MODEL_IN_USE", "Este modelo está em uma operação ativa e não pode ser excluído.", 409);
|
||||
}
|
||||
const updated = await query.query<{ id: string }>(
|
||||
`update public.spreadsheet_models
|
||||
set status = 'deleted', deleted_at = now(), updated_at = now()
|
||||
where id = $1 and workspace_id = $2 and status = 'active'
|
||||
returning id`,
|
||||
[modelId, ctx.workspaceId],
|
||||
);
|
||||
if (!updated[0]) throw new AdapterError("NOT_FOUND", "Modelo não encontrado.", 404);
|
||||
await audit(query, ctx, "model.deleted", "model", modelId, "success");
|
||||
}).catch(mapDbError);
|
||||
}
|
||||
|
||||
export async function createJob(deps: AppDeps, ctx: RequestContext, modelId: string, idempotencyKey?: string | null): Promise<JobDto> {
|
||||
assertUse(ctx);
|
||||
const jobId = await deps.db.withUser(ctx.userId, async (query) => {
|
||||
if (idempotencyKey) {
|
||||
const existing = await query.query<{ id: string }>(
|
||||
`select id from public.spreadsheet_jobs where user_id = $1 and idempotency_key = $2`,
|
||||
[ctx.userId, idempotencyKey],
|
||||
);
|
||||
if (existing[0]) return existing[0].id;
|
||||
}
|
||||
const models = await query.query<{ id: string }>(
|
||||
`select id from public.spreadsheet_models where id = $1 and workspace_id = $2 and status = 'active'`,
|
||||
[modelId, ctx.workspaceId],
|
||||
);
|
||||
if (!models[0]) throw new AdapterError("NOT_FOUND", "Modelo não encontrado neste workspace.", 404);
|
||||
const id = randomUUID();
|
||||
await query.query(
|
||||
`insert into public.spreadsheet_jobs
|
||||
(id, workspace_id, user_id, model_id, status, source_count, current_source, idempotency_key)
|
||||
values ($1,$2,$3,$4,'draft',0,0,$5)`,
|
||||
[id, ctx.workspaceId, ctx.userId, modelId, idempotencyKey ?? null],
|
||||
);
|
||||
await audit(query, ctx, "job.created", "job", id, "success", { model_id: modelId });
|
||||
return id;
|
||||
}).catch(mapDbError);
|
||||
return deps.db.withUser(ctx.userId, (query) => readJob(query, deps.coordinator, ctx, jobId));
|
||||
}
|
||||
|
||||
async function ensureMutableJob(query: Query, ctx: RequestContext, jobId: string): Promise<void> {
|
||||
const rows = await query.query<{ status: string }>(
|
||||
`select status from public.spreadsheet_jobs where id = $1 and workspace_id = $2 and user_id = $3`,
|
||||
[jobId, ctx.workspaceId, ctx.userId],
|
||||
);
|
||||
if (!rows[0]) throw new AdapterError("NOT_FOUND", "Operação não encontrada.", 404);
|
||||
if (rows[0].status === "processing" || rows[0].status === "queued") {
|
||||
throw new AdapterError("JOB_LOCKED", "Aguarde o término da transferência em andamento.", 409);
|
||||
}
|
||||
}
|
||||
|
||||
export async function addSource(deps: AppDeps, ctx: RequestContext, jobId: string, file: UploadFile, sheet?: string | null, headerRow = 1) {
|
||||
assertUse(ctx);
|
||||
const filename = sanitizeDisplayName(file.filename);
|
||||
let parsed: Awaited<ReturnType<typeof readSpreadsheet>> | null = null;
|
||||
let sheetRequired: string[] | null = null;
|
||||
try {
|
||||
parsed = await readSpreadsheet(filename, file.bytes, { sheet, headerRow });
|
||||
} catch (error) {
|
||||
if (isAdapterError(error) && error.code === "SHEET_REQUIRED") sheetRequired = (error.details?.sheets as string[]) ?? [];
|
||||
else throw error;
|
||||
}
|
||||
const sourceId = randomUUID();
|
||||
const storageKey = `workspaces/${ctx.workspaceId}/tmp/${jobId}/${sourceId}/source`;
|
||||
const expires = new Date((deps.now ?? (() => new Date()))().getTime() + ttl(deps) * 1000);
|
||||
const mixed = parsed?.grid.columns.some((column) => column.inferredType === "mixed") ?? false;
|
||||
await deps.storage.put(storageKey, file.bytes);
|
||||
try {
|
||||
await deps.db.withUser(ctx.userId, async (query) => {
|
||||
await ensureMutableJob(query, ctx, jobId);
|
||||
const counts = await query.query<{ total: number }>(
|
||||
`select count(*)::int as total from public.spreadsheet_job_sources where job_id = $1`,
|
||||
[jobId],
|
||||
);
|
||||
if (Number(counts[0]?.total ?? 0) >= MAX_SOURCES_PER_JOB) {
|
||||
throw new AdapterError("TOO_MANY_SOURCES", "A operação aceita no máximo 10 fontes.", 409);
|
||||
}
|
||||
const status = sheetRequired ? "pending" : mixed ? "invalid" : "validated";
|
||||
const errorCode = sheetRequired ? "SHEET_REQUIRED" : mixed ? "TYPE_INCONSISTENT" : null;
|
||||
const schema = {
|
||||
columns: parsed?.grid.columns ?? [],
|
||||
sheets: sheetRequired ?? parsed?.grid.sheetNames ?? [],
|
||||
warnings: parsed?.grid.warnings ?? [],
|
||||
};
|
||||
await query.query(
|
||||
`insert into public.spreadsheet_job_sources
|
||||
(id, job_id, source_name, temp_storage_key, selected_sheet, header_row, row_count, size_bytes, status, columns_schema, error_code, expires_at)
|
||||
values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::jsonb,$11,$12)`,
|
||||
[
|
||||
sourceId,
|
||||
jobId,
|
||||
filename,
|
||||
storageKey,
|
||||
parsed?.grid.selectedSheet ?? null,
|
||||
parsed?.grid.headerRow ?? headerRow,
|
||||
parsed?.grid.rows.length ?? 0,
|
||||
file.bytes.length,
|
||||
status,
|
||||
JSON.stringify(schema),
|
||||
errorCode,
|
||||
expires.toISOString(),
|
||||
],
|
||||
);
|
||||
await query.query(
|
||||
`update public.spreadsheet_jobs set source_count = source_count + 1, updated_at = now() where id = $1`,
|
||||
[jobId],
|
||||
);
|
||||
});
|
||||
} catch (error) {
|
||||
await deps.storage.delete(storageKey);
|
||||
mapDbError(error);
|
||||
}
|
||||
const job = await deps.db.withUser(ctx.userId, (query) => readJob(query, deps.coordinator, ctx, jobId));
|
||||
if (sheetRequired) {
|
||||
return { httpStatus: 409, code: "SHEET_REQUIRED", message: "Selecione exatamente uma aba.", job };
|
||||
}
|
||||
if (mixed) {
|
||||
return { httpStatus: 422, code: "TYPE_INCONSISTENT", message: "Há colunas com tipos inconsistentes. Ajuste a linha de cabeçalho ou a fonte.", job };
|
||||
}
|
||||
return { httpStatus: 200, job };
|
||||
}
|
||||
|
||||
export async function updateSource(
|
||||
deps: AppDeps,
|
||||
ctx: RequestContext,
|
||||
jobId: string,
|
||||
sourceId: string,
|
||||
input: { sheet?: string | null; headerRow?: number },
|
||||
) {
|
||||
assertUse(ctx);
|
||||
await deps.db.withUser(ctx.userId, async (query) => {
|
||||
await ensureMutableJob(query, ctx, jobId);
|
||||
const rows = await query.query<{ temp_storage_key: string; source_name: string; header_row: number; selected_sheet: string | null }>(
|
||||
`select temp_storage_key, source_name, header_row, selected_sheet
|
||||
from public.spreadsheet_job_sources where id = $1 and job_id = $2`,
|
||||
[sourceId, jobId],
|
||||
);
|
||||
const source = rows[0];
|
||||
if (!source) throw new AdapterError("NOT_FOUND", "Fonte não encontrada.", 404);
|
||||
const bytes = await deps.storage.get(source.temp_storage_key);
|
||||
const parsed = await readSpreadsheet(source.source_name, bytes, {
|
||||
sheet: input.sheet === undefined ? source.selected_sheet : input.sheet,
|
||||
headerRow: input.headerRow ?? source.header_row,
|
||||
});
|
||||
const mixed = parsed.grid.columns.some((column) => column.inferredType === "mixed");
|
||||
await query.query(`delete from public.spreadsheet_job_mappings where job_source_id = $1`, [sourceId]);
|
||||
await query.query(
|
||||
`update public.spreadsheet_job_sources
|
||||
set selected_sheet = $2, header_row = $3, row_count = $4, status = $5, error_code = $6, columns_schema = $7::jsonb
|
||||
where id = $1`,
|
||||
[
|
||||
sourceId,
|
||||
parsed.grid.selectedSheet,
|
||||
parsed.grid.headerRow,
|
||||
parsed.grid.rows.length,
|
||||
mixed ? "invalid" : "validated",
|
||||
mixed ? "TYPE_INCONSISTENT" : null,
|
||||
JSON.stringify({ columns: parsed.grid.columns, sheets: parsed.grid.sheetNames, warnings: parsed.grid.warnings }),
|
||||
],
|
||||
);
|
||||
}).catch(mapDbError);
|
||||
const job = await deps.db.withUser(ctx.userId, (query) => readJob(query, deps.coordinator, ctx, jobId));
|
||||
const source = job.sources.find((item) => item.id === sourceId);
|
||||
if (source?.status === "invalid") {
|
||||
return { httpStatus: 422, code: source.errorCode ?? "TYPE_INCONSISTENT", message: "A fonte continua inválida para esta aba ou cabeçalho.", job };
|
||||
}
|
||||
return { httpStatus: 200, job };
|
||||
}
|
||||
|
||||
export async function removeSource(deps: AppDeps, ctx: RequestContext, jobId: string, sourceId: string): Promise<JobDto> {
|
||||
assertUse(ctx);
|
||||
const key = await deps.db.withUser(ctx.userId, async (query) => {
|
||||
await ensureMutableJob(query, ctx, jobId);
|
||||
const rows = await query.query<{ temp_storage_key: string; status: string }>(
|
||||
`delete from public.spreadsheet_job_sources
|
||||
where id = $1 and job_id = $2 and status not in ('processing', 'completed')
|
||||
returning temp_storage_key, status`,
|
||||
[sourceId, jobId],
|
||||
);
|
||||
if (!rows[0]) throw new AdapterError("NOT_FOUND", "Fonte não encontrada ou já processada.", 404);
|
||||
await query.query(`update public.spreadsheet_jobs set source_count = greatest(source_count - 1, 0), updated_at = now() where id = $1`, [jobId]);
|
||||
return rows[0].temp_storage_key;
|
||||
}).catch(mapDbError);
|
||||
if (key.includes("/tmp/")) await deps.storage.delete(key);
|
||||
return deps.db.withUser(ctx.userId, (query) => readJob(query, deps.coordinator, ctx, jobId));
|
||||
}
|
||||
|
||||
export async function saveMappings(deps: AppDeps, ctx: RequestContext, jobId: string, sourceId: string, mappings: MappingInput[]) {
|
||||
assertUse(ctx);
|
||||
await deps.db.withUser(ctx.userId, async (query) => {
|
||||
await ensureMutableJob(query, ctx, jobId);
|
||||
const job = await readJob(query, deps.coordinator, ctx, jobId);
|
||||
const source = job.sources.find((item) => item.id === sourceId);
|
||||
if (!source) throw new AdapterError("NOT_FOUND", "Fonte não encontrada.", 404);
|
||||
if (source.status === "invalid" || source.status === "pending") {
|
||||
throw new AdapterError(source.errorCode ?? "INVALID_STRUCTURE", "Corrija a fonte antes de mapear.", 422);
|
||||
}
|
||||
const result = validateMappings(job.model.columns, source.columns.map((column) => column.name), mappings);
|
||||
if (!result.ok && result.code !== "MAPPING_INCOMPLETE") {
|
||||
throw new AdapterError(result.code, result.errors[0] ?? "Mapeamento inválido.", 422);
|
||||
}
|
||||
await query.query(`delete from public.spreadsheet_job_mappings where job_source_id = $1`, [sourceId]);
|
||||
for (const [index, mapping] of mappings.entries()) {
|
||||
const required = job.model.columns.find((column) => column.name === mapping.modelColumn)?.required ?? false;
|
||||
await query.query(
|
||||
`insert into public.spreadsheet_job_mappings (job_source_id, source_column, model_column, is_required, position)
|
||||
values ($1,$2,$3,$4,$5)`,
|
||||
[sourceId, mapping.sourceColumn, mapping.modelColumn, required, index],
|
||||
);
|
||||
}
|
||||
await query.query(`update public.spreadsheet_job_sources set status = $2, error_code = null where id = $1`, [
|
||||
sourceId,
|
||||
result.ok ? "mapped" : "validated",
|
||||
]);
|
||||
if (result.ok) {
|
||||
await query.query(
|
||||
`update public.spreadsheet_jobs set status = 'ready', updated_at = now() where id = $1 and status in ('draft','failed','ready','completed')`,
|
||||
[jobId],
|
||||
);
|
||||
}
|
||||
}).catch(mapDbError);
|
||||
return deps.db.withUser(ctx.userId, (query) => readJob(query, deps.coordinator, ctx, jobId));
|
||||
}
|
||||
|
||||
export async function previewSource(deps: AppDeps, ctx: RequestContext, jobId: string, sourceId: string) {
|
||||
assertUse(ctx);
|
||||
return deps.db.withUser(ctx.userId, async (query) => {
|
||||
const job = await readJob(query, deps.coordinator, ctx, jobId);
|
||||
const source = job.sources.find((item) => item.id === sourceId);
|
||||
if (!source) throw new AdapterError("NOT_FOUND", "Fonte não encontrada.", 404);
|
||||
const validation = validateMappings(
|
||||
job.model.columns,
|
||||
source.columns.map((column) => column.name),
|
||||
source.mappings.map((mapping) => ({ sourceColumn: mapping.sourceColumn, modelColumn: mapping.modelColumn })),
|
||||
);
|
||||
if (!validation.ok) {
|
||||
const message = validation.missingRequired.length
|
||||
? `Mapeie as colunas obrigatórias: ${validation.missingRequired.join(", ")}.`
|
||||
: validation.errors[0] ?? "Mapeamento incompleto.";
|
||||
throw new AdapterError(validation.code, message, 422, { missingRequired: validation.missingRequired });
|
||||
}
|
||||
const stored = await query.query<{ temp_storage_key: string; source_name: string; selected_sheet: string | null; header_row: number }>(
|
||||
`select temp_storage_key, source_name, selected_sheet, header_row from public.spreadsheet_job_sources where id = $1`,
|
||||
[sourceId],
|
||||
);
|
||||
const bytes = await deps.storage.get(stored[0].temp_storage_key);
|
||||
const parsed = await readSpreadsheet(stored[0].source_name, bytes, {
|
||||
sheet: stored[0].selected_sheet,
|
||||
headerRow: stored[0].header_row,
|
||||
});
|
||||
return buildPreview(
|
||||
job.model.columns,
|
||||
parsed.grid.headers,
|
||||
parsed.grid.rows,
|
||||
source.mappings.map((mapping) => ({ sourceColumn: mapping.sourceColumn, modelColumn: mapping.modelColumn })),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function inlineMode(): boolean {
|
||||
if (process.env.ADAPTER_JOB_MODE === "worker") return false;
|
||||
if (process.env.ADAPTER_JOB_MODE === "inline") return true;
|
||||
return process.env.NODE_ENV !== "production";
|
||||
}
|
||||
|
||||
export async function startProcess(deps: AppDeps, ctx: RequestContext, jobId: string, sourceId: string, idempotencyKey: string): Promise<JobDto> {
|
||||
assertUse(ctx);
|
||||
if (!idempotencyKey || idempotencyKey.length > 120) {
|
||||
throw new AdapterError("IDEMPOTENCY_REQUIRED", "Informe uma chave de idempotência para processar.", 400);
|
||||
}
|
||||
const current = await deps.db.withUser(ctx.userId, (query) => readJob(query, deps.coordinator, ctx, jobId));
|
||||
const source = current.sources.find((item) => item.id === sourceId);
|
||||
if (!source) throw new AdapterError("NOT_FOUND", "Fonte não encontrada.", 404);
|
||||
const prior = await deps.db.withUser(ctx.userId, (query) =>
|
||||
query.query<{ process_idempotency_key: string | null; status: string }>(
|
||||
`select process_idempotency_key, status from public.spreadsheet_job_sources where id = $1 and job_id = $2`,
|
||||
[sourceId, jobId],
|
||||
),
|
||||
);
|
||||
if (
|
||||
prior[0]?.process_idempotency_key === idempotencyKey &&
|
||||
(prior[0].status === "completed" || prior[0].status === "processing")
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
const validation = validateMappings(
|
||||
current.model.columns,
|
||||
source.columns.map((column) => column.name),
|
||||
source.mappings.map((mapping) => ({ sourceColumn: mapping.sourceColumn, modelColumn: mapping.modelColumn })),
|
||||
);
|
||||
if (!validation.ok) {
|
||||
throw new AdapterError(
|
||||
validation.code,
|
||||
validation.missingRequired.length
|
||||
? `Mapeie as colunas obrigatórias: ${validation.missingRequired.join(", ")}.`
|
||||
: "Mapeamento inválido.",
|
||||
422,
|
||||
{ missingRequired: validation.missingRequired },
|
||||
);
|
||||
}
|
||||
let locked = false;
|
||||
try {
|
||||
locked = await deps.coordinator.acquireLock(lockKey(ctx.userId), jobId, 120);
|
||||
} catch (error) {
|
||||
if (isAdapterError(error) && error.code === "REDIS_UNAVAILABLE") throw error;
|
||||
throw new AdapterError("REDIS_UNAVAILABLE", "Coordenação indisponível. O processamento não foi iniciado.", 503);
|
||||
}
|
||||
if (!locked) {
|
||||
const owns = await deps.coordinator.ownsLock(lockKey(ctx.userId), jobId).catch(() => false);
|
||||
if (!owns) throw new AdapterError("JOB_LOCKED", "Já existe uma transferência em andamento para este usuário.", 409);
|
||||
}
|
||||
await deps.db.withUser(ctx.userId, async (query) => {
|
||||
const sourceIndex = current.sources.findIndex((item) => item.id === sourceId) + 1;
|
||||
const progress = {
|
||||
phase: "validating",
|
||||
label: "Validando",
|
||||
percent: null,
|
||||
sourceIndex,
|
||||
sourceCount: current.sources.length,
|
||||
sourceId,
|
||||
};
|
||||
await query.query(
|
||||
`update public.spreadsheet_jobs
|
||||
set status = 'queued', error_code = null, progress = $3::jsonb, current_source = $4,
|
||||
started_at = coalesce(started_at, now()), updated_at = now()
|
||||
where id = $1 and user_id = $2 and status in ('ready','draft','failed','completed')`,
|
||||
[jobId, ctx.userId, JSON.stringify(progress), sourceIndex],
|
||||
);
|
||||
await query.query(
|
||||
`update public.spreadsheet_job_sources set process_idempotency_key = $2 where id = $1`,
|
||||
[sourceId, idempotencyKey],
|
||||
);
|
||||
await audit(query, ctx, "job.started", "job", jobId, "success", { source_id: sourceId });
|
||||
});
|
||||
if (inlineMode()) await runProcess(deps, ctx, jobId, sourceId);
|
||||
return deps.db.withUser(ctx.userId, (query) => readJob(query, deps.coordinator, ctx, jobId));
|
||||
}
|
||||
|
||||
export async function getJob(deps: AppDeps, ctx: RequestContext, jobId: string): Promise<JobDto> {
|
||||
assertUse(ctx);
|
||||
return deps.db.withUser(ctx.userId, (query) => readJob(query, deps.coordinator, ctx, jobId));
|
||||
}
|
||||
|
||||
export { runProcess };
|
||||
Reference in new issue
Block a user