From f6dc617eae9ed6eda6609ec0eed8b5475a6a9b0f Mon Sep 17 00:00:00 2001 From: Vasily Zubarev Date: Fri, 21 Mar 2025 18:42:14 +0100 Subject: [PATCH] feat: use structured output, import CSV, bugfixes --- .env.example | 6 +- app/ai/analyze.ts | 104 +++------- app/ai/prompt.ts | 10 - app/ai/schema.ts | 16 ++ app/export/transactions/route.ts | 29 ++- app/import/csv/actions.tsx | 66 ++++++ app/import/csv/page.tsx | 11 + app/layout.tsx | 2 +- app/loading.tsx | 2 +- app/settings/backups/page.tsx | 3 +- app/settings/fields/page.tsx | 2 +- app/settings/layout.tsx | 2 +- app/settings/llm/page.tsx | 4 +- components/dashboard/drop-zone-widget.tsx | 3 +- components/dashboard/stats-widget.tsx | 9 +- components/forms/error.tsx | 3 + components/import/csv.tsx | 193 ++++++++++++++++++ components/settings/global-settings-form.tsx | 3 +- components/settings/llm-settings-form.tsx | 88 +++++--- components/settings/side-nav.tsx | 2 +- components/sidebar/sidebar.tsx | 51 +++-- components/transactions/create.tsx | 14 +- components/transactions/edit.tsx | 5 +- components/unsorted/analyze-form.tsx | 7 +- data/categories.ts | 6 + data/export.ts | 5 - data/export_and_import.ts | 155 ++++++++++++++ data/projects.ts | 6 + lib/files.ts | 1 + lib/stats.ts | 7 +- package-lock.json | 33 +++ package.json | 4 +- .../migration.sql | 42 ++++ prisma/schema.prisma | 5 +- prisma/seed.ts | 31 ++- 35 files changed, 735 insertions(+), 195 deletions(-) create mode 100644 app/ai/schema.ts create mode 100644 app/import/csv/actions.tsx create mode 100644 app/import/csv/page.tsx create mode 100644 components/forms/error.tsx create mode 100644 components/import/csv.tsx delete mode 100644 data/export.ts create mode 100644 data/export_and_import.ts create mode 100644 prisma/migrations/20250321141756_improve_fields/migration.sql diff --git a/.env.example b/.env.example index eb663ee..90bb454 100644 --- a/.env.example +++ b/.env.example @@ -13,8 +13,4 @@ And projects are: {projects} -If you can't find something leave it blank. Return only valid JSON with these fields: - -{json_structure} - -Do not include any other text in your response!" \ No newline at end of file +If you can't find something leave it blank. Return only one object. Do not include any other text in your response!" \ No newline at end of file diff --git a/app/ai/analyze.ts b/app/ai/analyze.ts index 84add0c..1da671f 100644 --- a/app/ai/analyze.ts +++ b/app/ai/analyze.ts @@ -1,11 +1,12 @@ import { Category, Field, File, Project } from "@prisma/client" import OpenAI from "openai" -import { ChatCompletion } from "openai/resources/index.mjs" import { buildLLMPrompt } from "./prompt" +import { fieldsToJsonSchema } from "./schema" -const MAX_PAGES_TO_ANALYZE = 3 +const MAX_PAGES_TO_ANALYZE = 4 type AnalyzeAttachment = { + filename: string contentType: string base64: string } @@ -32,7 +33,11 @@ export const retrieveFileContentForAI = async (file: File, page: number): Promis const buffer = await blob.arrayBuffer() const base64 = Buffer.from(buffer).toString("base64") - return { contentType: response.headers.get("Content-Type") || file.mimetype, base64: base64 } + return { + filename: file.filename, + contentType: response.headers.get("Content-Type") || file.mimetype, + base64: base64, + } } export async function analyzeTransaction( @@ -49,33 +54,41 @@ export async function analyzeTransaction( }) const prompt = buildLLMPrompt(promptTemplate, fields, categories, projects) + const schema = fieldsToJsonSchema(fields) console.log("PROMPT:", prompt) + console.log("SCHEMA:", schema) try { - const response = await openai.chat.completions.create({ - model: "gpt-4o-mini", - messages: [ + const response = await openai.responses.create({ + model: "gpt-4o-mini-2024-07-18", + input: [ { role: "user", - content: [ - { type: "text", text: prompt || "" }, - ...attachments.slice(0, MAX_PAGES_TO_ANALYZE).map((attachment) => ({ - type: "image_url" as const, - image_url: { - url: `data:${attachment.contentType};base64,${attachment.base64}`, - }, - })), - ], + content: prompt, + }, + { + role: "user", + content: attachments.map((attachment) => ({ + type: "input_image", + image_url: `data:${attachment.contentType};base64,${attachment.base64}`, + })), }, ], + text: { + format: { + type: "json_schema", + name: "transaction", + schema: schema, + strict: true, + }, + }, }) - console.log("ChatGPT response:", response.choices[0].message) + console.log("ChatGPT response:", response.output_text) - const cleanedJson = extractAndParseJSON(response) - - return { success: true, data: cleanedJson } + const result = JSON.parse(response.output_text) + return { success: true, data: result } } catch (error) { console.error("AI Analysis error:", error) return { @@ -84,56 +97,3 @@ export async function analyzeTransaction( } } } - -function extractAndParseJSON(response: ChatCompletion) { - try { - const content = response.choices?.[0]?.message?.content - - if (!content) { - throw new Error("No response content from AI") - } - - // Check for JSON in code blocks (handles ```json, ``` json, or just ```) - let jsonText = content.trim() - const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/ - const jsonMatch = content.match(codeBlockRegex) - - if (jsonMatch && jsonMatch[1]) { - jsonText = jsonMatch[1].trim() - } - - // Try to parse the JSON - try { - return JSON.parse(jsonText) - } catch (parseError) { - // JSON might have unescaped characters, try to fix them - const fixedJsonText = escapeJsonString(jsonText) - return JSON.parse(fixedJsonText) - } - } catch (error) { - console.error("Error processing AI response:", error) - throw new Error(`Failed to extract valid JSON: ${error instanceof Error ? error.message : "Unknown error"}`) - } -} - -function escapeJsonString(jsonStr: string) { - // This is a black magic to fix some AI-generated JSONs - if (jsonStr.trim().startsWith("{") && jsonStr.trim().endsWith("}")) { - return jsonStr.replace(/"([^"]*?)":(\s*)"(.*?)"/g, (match, key, space, value) => { - const escapedValue = value - .replace(/\\/g, "\\\\") // backslash - .replace(/"/g, '\\"') // double quotes - .replace(/\n/g, "\\n") // newline - .replace(/\r/g, "\\r") // carriage return - .replace(/\t/g, "\\t") // tab - .replace(/\f/g, "\\f") // form feed - .replace(/[\x00-\x1F\x7F-\x9F]/g, (c: string) => { - return "\\u" + ("0000" + c.charCodeAt(0).toString(16)).slice(-4) - }) - - return `"${key}":${space}"${escapedValue}"` - }) - } - - return jsonStr -} diff --git a/app/ai/prompt.ts b/app/ai/prompt.ts index a2df3cd..e326fa1 100644 --- a/app/ai/prompt.ts +++ b/app/ai/prompt.ts @@ -35,15 +35,5 @@ export function buildLLMPrompt( prompt = prompt.replace("{categories.code}", categories.map((category) => `${category.code}`).join(", ")) prompt = prompt.replace("{projects.code}", projects.map((project) => `${project.code}`).join(", ")) - prompt = prompt.replace( - "{json_structure}", - "{ " + - fields - .filter((field) => field.llm_prompt) - .map((field) => `${field.code}: ${field.type}`) - .join(", ") + - " }" - ) - return prompt } diff --git a/app/ai/schema.ts b/app/ai/schema.ts new file mode 100644 index 0000000..77ab074 --- /dev/null +++ b/app/ai/schema.ts @@ -0,0 +1,16 @@ +import { Field } from "@prisma/client" + +export const fieldsToJsonSchema = (fields: Field[]) => { + const fieldsWithPrompt = fields.filter((field) => field.llm_prompt) + const schema = { + type: "object", + properties: fieldsWithPrompt.reduce((acc, field) => { + acc[field.code] = { type: field.type, description: field.llm_prompt || "" } + return acc + }, {} as Record), + required: fieldsWithPrompt.map((field) => field.code), + additionalProperties: false, + } + + return schema +} diff --git a/app/export/transactions/route.ts b/app/export/transactions/route.ts index 772fc19..47f7c00 100644 --- a/app/export/transactions/route.ts +++ b/app/export/transactions/route.ts @@ -1,4 +1,4 @@ -import { ExportFields, ExportFilters } from "@/data/export" +import { ExportFields, ExportFilters, exportImportFieldsMapping } from "@/data/export_and_import" import { getFields } from "@/data/fields" import { getFilesByTransactionId } from "@/data/files" import { getTransactions } from "@/data/transactions" @@ -18,11 +18,10 @@ export async function GET(request: Request) { const transactions = await getTransactions(filters) const existingFields = await getFields() + // Generate CSV file with all transactions try { const fieldKeys = fields.filter((field) => existingFields.some((f) => f.code === field)) - const writeHeaders = fieldKeys.map((field) => existingFields.find((f) => f.code === field)?.name) - // Generate CSV file with all transactions let csvContent = "" const csvStream = format({ headers: fieldKeys, writeBOM: true, writeHeaders: false }) @@ -30,15 +29,25 @@ export async function GET(request: Request) { csvContent += chunk }) - csvStream.write(writeHeaders) - transactions.forEach((transaction) => { - const row = fieldKeys.reduce((acc, key) => { - acc[key] = transaction[key as keyof typeof transaction] ?? "" - return acc - }, {} as Record) + // Custom CSV headers + const headers = fieldKeys.map((field) => existingFields.find((f) => f.code === field)?.name ?? "UNKNOWN") + csvStream.write(headers) + + // CSV rows + for (const transaction of transactions) { + const row: Record = {} + for (const key of fieldKeys) { + const value = transaction[key as keyof typeof transaction] ?? "" + const exportFieldSettings = exportImportFieldsMapping[key] + if (exportFieldSettings && exportFieldSettings.export) { + row[key] = await exportFieldSettings.export(value) + } else { + row[key] = value + } + } csvStream.write(row) - }) + } csvStream.end() // Wait for CSV generation to complete diff --git a/app/import/csv/actions.tsx b/app/import/csv/actions.tsx new file mode 100644 index 0000000..72f42ba --- /dev/null +++ b/app/import/csv/actions.tsx @@ -0,0 +1,66 @@ +"use server" + +import { exportImportFieldsMapping } from "@/data/export_and_import" +import { createTransaction } from "@/data/transactions" +import { parse } from "@fast-csv/parse" +import { revalidatePath } from "next/cache" + +export async function parseCSVAction(prevState: any, formData: FormData) { + const file = formData.get("file") as File + if (!file) { + return { success: false, error: "No file uploaded" } + } + + if (!file.name.toLowerCase().endsWith(".csv")) { + return { success: false, error: "Only CSV files are allowed" } + } + + try { + const buffer = Buffer.from(await file.arrayBuffer()) + const rows: string[][] = [] + + const parser = parse() + .on("data", (row) => rows.push(row)) + .on("error", (error) => { + throw error + }) + parser.write(buffer) + parser.end() + + // Wait for parsing to complete + await new Promise((resolve) => parser.on("end", resolve)) + + return { success: true, data: rows } + } catch (error) { + console.error("Error parsing CSV:", error) + return { success: false, error: "Failed to parse CSV file" } + } +} + +export async function saveTransactionsAction(prevState: any, formData: FormData) { + try { + const rows = JSON.parse(formData.get("rows") as string) as Record[] + + for (const row of rows) { + const transactionData: Record = {} + for (const [fieldCode, value] of Object.entries(row)) { + const fieldDef = exportImportFieldsMapping[fieldCode] + if (fieldDef?.import) { + transactionData[fieldCode] = await fieldDef.import(value as string) + } else { + transactionData[fieldCode] = value as string + } + } + + await createTransaction(transactionData) + } + + revalidatePath("/import/csv") + revalidatePath("/transactions") + + return { success: true } + } catch (error) { + console.error("Error saving transactions:", error) + return { success: false, error: "Failed to save transactions: " + error } + } +} diff --git a/app/import/csv/page.tsx b/app/import/csv/page.tsx new file mode 100644 index 0000000..a6de58a --- /dev/null +++ b/app/import/csv/page.tsx @@ -0,0 +1,11 @@ +import { ImportCSVTable } from "@/components/import/csv" +import { getFields } from "@/data/fields" + +export default async function CSVImportPage() { + const fields = await getFields() + return ( +
+ +
+ ) +} diff --git a/app/layout.tsx b/app/layout.tsx index decdb2a..75ceccf 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -43,7 +43,7 @@ export default async function RootLayout({ children }: { children: React.ReactNo - {children} + {children} diff --git a/app/loading.tsx b/app/loading.tsx index 9a10c6b..deeb1b0 100644 --- a/app/loading.tsx +++ b/app/loading.tsx @@ -2,7 +2,7 @@ import { Loader2 } from "lucide-react" export default function AppLoading() { return ( -
+
) diff --git a/app/settings/backups/page.tsx b/app/settings/backups/page.tsx index a636c0f..617ee85 100644 --- a/app/settings/backups/page.tsx +++ b/app/settings/backups/page.tsx @@ -1,5 +1,6 @@ "use client" +import { FormError } from "@/components/forms/error" import { Button } from "@/components/ui/button" import { Card } from "@/components/ui/card" import { Download, Loader2 } from "lucide-react" @@ -51,7 +52,7 @@ export default function BackupSettingsPage() { )} - {restoreState?.error &&

{restoreState.error}

} + {restoreState?.error && {restoreState.error}}
) diff --git a/app/settings/fields/page.tsx b/app/settings/fields/page.tsx index 5c16f99..0f90dfb 100644 --- a/app/settings/fields/page.tsx +++ b/app/settings/fields/page.tsx @@ -17,7 +17,7 @@ export default async function FieldsSettingsPage() { items={fieldsWithActions} columns={[ { key: "name", label: "Name", editable: true }, - { key: "type", label: "Type", editable: true }, + { key: "type", label: "Type", defaultValue: "string", editable: true }, { key: "llm_prompt", label: "LLM Prompt", editable: true }, ]} onDelete={async (code) => { diff --git a/app/settings/layout.tsx b/app/settings/layout.tsx index bf0aac1..cee9ddd 100644 --- a/app/settings/layout.tsx +++ b/app/settings/layout.tsx @@ -41,7 +41,7 @@ const settingsCategories = [ export default function SettingsLayout({ children }: { children: React.ReactNode }) { return ( <> -
+

Settings

Customize your settings here

diff --git a/app/settings/llm/page.tsx b/app/settings/llm/page.tsx index 2422779..81b2d65 100644 --- a/app/settings/llm/page.tsx +++ b/app/settings/llm/page.tsx @@ -1,13 +1,15 @@ import LLMSettingsForm from "@/components/settings/llm-settings-form" +import { getFields } from "@/data/fields" import { getSettings } from "@/data/settings" export default async function LlmSettingsPage() { const settings = await getSettings() + const fields = await getFields() return ( <>
- +
) diff --git a/components/dashboard/drop-zone-widget.tsx b/components/dashboard/drop-zone-widget.tsx index 2bbf5b7..0c5933a 100644 --- a/components/dashboard/drop-zone-widget.tsx +++ b/components/dashboard/drop-zone-widget.tsx @@ -6,6 +6,7 @@ import { FILE_ACCEPTED_MIMETYPES } from "@/lib/files" import { Camera, Loader2 } from "lucide-react" import { useRouter } from "next/navigation" import { startTransition, useState } from "react" +import { FormError } from "../forms/error" export default function DashboardDropZoneWidget() { const router = useRouter() @@ -65,7 +66,7 @@ export default function DashboardDropZoneWidget() { upload receipts, invoices and any other documents for me to scan

)} - {uploadError &&

{uploadError}

} + {uploadError && {uploadError}}
diff --git a/components/dashboard/stats-widget.tsx b/components/dashboard/stats-widget.tsx index ccb1028..85c2d08 100644 --- a/components/dashboard/stats-widget.tsx +++ b/components/dashboard/stats-widget.tsx @@ -32,7 +32,7 @@ export async function StatsWidget({ filters }: { filters: TransactionFilters }) {Object.entries(stats.totalIncomePerCurrency).map(([currency, total]) => ( -
+
{formatCurrency(total, currency)}
))} @@ -46,12 +46,7 @@ export async function StatsWidget({ filters }: { filters: TransactionFilters }) {Object.entries(stats.totalExpensesPerCurrency).map(([currency, total]) => ( -
= 0 ? "text-green-500" : "text-red-500" - }`} - > +
{formatCurrency(total, currency)}
))} diff --git a/components/forms/error.tsx b/components/forms/error.tsx new file mode 100644 index 0000000..fceea9e --- /dev/null +++ b/components/forms/error.tsx @@ -0,0 +1,3 @@ +export function FormError({ children }: { children: React.ReactNode }) { + return

{children}

+} diff --git a/components/import/csv.tsx b/components/import/csv.tsx new file mode 100644 index 0000000..ff93bb6 --- /dev/null +++ b/components/import/csv.tsx @@ -0,0 +1,193 @@ +"use client" + +import { Button } from "@/components/ui/button" +import { Field } from "@prisma/client" +import { Upload } from "lucide-react" +import { useRouter } from "next/navigation" +import { startTransition, useActionState, useEffect, useState } from "react" +import { parseCSVAction, saveTransactionsAction } from "../../app/import/csv/actions" +import { FormError } from "../forms/error" + +const MAX_PREVIEW_ROWS = 100 + +export function ImportCSVTable({ fields }: { fields: Field[] }) { + const router = useRouter() + const [parseState, parseAction] = useActionState(parseCSVAction, null) + const [saveState, saveAction] = useActionState(saveTransactionsAction, null) + + const [csvSettings, setCSVSettings] = useState({ + skipHeader: true, + }) + const [csvData, setCSVData] = useState([]) + const [columnMappings, setColumnMappings] = useState([]) + + useEffect(() => { + if (parseState?.success && parseState.data) { + setCSVData(parseState.data) + if (parseState.data.length > 0) { + setColumnMappings( + parseState.data[0].map((value) => { + const field = fields.find((field) => field.code === value || field.name === value) + return field?.code || "" + }) + ) + } else { + setColumnMappings([]) + } + } + }, [parseState]) + + useEffect(() => { + if (saveState?.success) { + router.push("/transactions") + } + }, [saveState, router]) + + const handleFileChange = async (event: React.ChangeEvent) => { + const file = event.target.files?.[0] + if (!file) return + + const formData = new FormData() + formData.append("file", file) + + startTransition(async () => { + await parseAction(formData) + }) + } + + const handleMappingChange = (columnIndex: number, fieldCode: string) => { + setColumnMappings((prev) => { + const state = [...prev] + state[columnIndex] = fieldCode + return state + }) + } + + const handleSave = async () => { + if (csvData.length === 0) return + + if (!isAtLeastOneFieldMapped(columnMappings)) { + alert("Please map at least one column to a field") + return + } + + const startIndex = csvSettings.skipHeader ? 1 : 0 + const processedRows = csvData.slice(startIndex).map((row) => { + const processedRow: Record = {} + + columnMappings.forEach((fieldCode, columnIndex) => { + if (!fieldCode || !row[columnIndex]) return + processedRow[fieldCode] = row[columnIndex] + }) + + return processedRow + }) + + const formData = new FormData() + formData.append("rows", JSON.stringify(processedRows)) + + startTransition(async () => { + await saveAction(formData) + }) + } + + return ( + <> + {csvData.length === 0 && ( +
+

Upload your CSV file to import transactions

+
+
+ + +
+
+ {parseState?.error && {parseState.error}} +
+ )} + + {csvData.length > 0 && ( +
+
+

+ Import {csvData.length} items from CSV +

+
+ +
+
+ +
+ +
+ +
+
+ + + + {csvData[0].map((_, index) => ( + + ))} + + + + {csvData.slice(0, MAX_PREVIEW_ROWS).map((row, rowIndex) => ( + + {csvData[0].map((_, colIndex) => ( + + ))} + + ))} + +
+ +
+ {(row[colIndex] || "").toString().slice(0, 256)} +
+
+
+ + {csvData.length > MAX_PREVIEW_ROWS && ( +

and {csvData.length - MAX_PREVIEW_ROWS} more entries...

+ )} + + {saveState?.error && {saveState.error}} +
+ )} + + ) +} + +function isAtLeastOneFieldMapped(columnMappings: string[]) { + return columnMappings.some((mapping) => mapping !== "") +} diff --git a/components/settings/global-settings-form.tsx b/components/settings/global-settings-form.tsx index 7575f91..c6a197e 100644 --- a/components/settings/global-settings-form.tsx +++ b/components/settings/global-settings-form.tsx @@ -9,6 +9,7 @@ import { Button } from "@/components/ui/button" import { Category, Currency } from "@prisma/client" import { CircleCheckBig } from "lucide-react" import { useActionState } from "react" +import { FormError } from "../forms/error" export default function GlobalSettingsForm({ settings, @@ -53,7 +54,7 @@ export default function GlobalSettingsForm({ )}
- {saveState?.error &&

{saveState.error}

} + {saveState?.error && {saveState.error}} ) } diff --git a/components/settings/llm-settings-form.tsx b/components/settings/llm-settings-form.tsx index 8679133..80fc6dd 100644 --- a/components/settings/llm-settings-form.tsx +++ b/components/settings/llm-settings-form.tsx @@ -1,45 +1,75 @@ "use client" +import { fieldsToJsonSchema } from "@/app/ai/schema" import { saveSettingsAction } from "@/app/settings/actions" import { FormInput, FormTextarea } from "@/components/forms/simple" import { Button } from "@/components/ui/button" -import { CircleCheckBig } from "lucide-react" +import { CircleCheckBig, Edit } from "lucide-react" +import Link from "next/link" import { useActionState } from "react" +import { FormError } from "../forms/error" +import { Card, CardTitle } from "../ui/card" -export default function LLMSettingsForm({ settings }: { settings: Record }) { +export default function LLMSettingsForm({ settings, fields }: { settings: Record; fields: Field[] }) { const [saveState, saveAction, pending] = useActionState(saveSettingsAction, null) return ( -
- + <> + + - - Get your API key from{" "} - - OpenAI Platform Console - - + + Get your API key from{" "} + + OpenAI Platform Console + + - + -
- - {saveState?.success && ( -

- - Saved! -

- )} -
+
+ + {saveState?.success && ( +

+ + Saved! +

+ )} +
- {saveState?.error &&

{saveState.error}

} - + {saveState?.error && {saveState.error}} + + + + + + Current JSON Schema for{" "} + + structured output + + + + Edit Fields + + + 
+          {JSON.stringify(fieldsToJsonSchema(fields), null, 2)}
+
+ + ) } diff --git a/components/settings/side-nav.tsx b/components/settings/side-nav.tsx index 6679e0d..05f5987 100644 --- a/components/settings/side-nav.tsx +++ b/components/settings/side-nav.tsx @@ -16,7 +16,7 @@ export function SideNav({ className, items, ...props }: SidebarNavProps) { const pathname = usePathname() return ( -
diff --git a/data/categories.ts b/data/categories.ts index 0708ae0..a4b6183 100644 --- a/data/categories.ts +++ b/data/categories.ts @@ -11,6 +11,12 @@ export const getCategories = cache(async () => { }) }) +export const getCategoryByCode = cache(async (code: string) => { + return await prisma.category.findUnique({ + where: { code }, + }) +}) + export const createCategory = async (category: Prisma.CategoryCreateInput) => { if (!category.code) { category.code = codeFromName(category.name as string) diff --git a/data/export.ts b/data/export.ts deleted file mode 100644 index 2b13863..0000000 --- a/data/export.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { TransactionFilters } from "./transactions" - -export type ExportFilters = TransactionFilters - -export type ExportFields = string[] diff --git a/data/export_and_import.ts b/data/export_and_import.ts new file mode 100644 index 0000000..b55cee0 --- /dev/null +++ b/data/export_and_import.ts @@ -0,0 +1,155 @@ +import { prisma } from "@/lib/db" +import { codeFromName } from "@/lib/utils" +import { formatDate } from "date-fns" +import { createCategory, getCategoryByCode } from "./categories" +import { createProject, getProjectByCode } from "./projects" +import { TransactionFilters } from "./transactions" + +export type ExportFilters = TransactionFilters + +export type ExportFields = string[] + +export const exportImportFields = [ + { + code: "name", + type: "string", + }, + { + code: "description", + type: "string", + }, + { + code: "merchant", + type: "string", + }, + { + code: "total", + type: "number", + export: async function (value: number) { + return value / 100 + }, + import: async function (value: string) { + const num = parseFloat(value) + return isNaN(num) ? 0.0 : num * 100 + }, + }, + { + code: "currencyCode", + type: "string", + }, + { + code: "convertedTotal", + type: "number", + export: async function (value: number | null) { + if (!value) { + return null + } + return value / 100 + }, + import: async function (value: string) { + const num = parseFloat(value) + return isNaN(num) ? 0.0 : num * 100 + }, + }, + { + code: "convertedCurrencyCode", + type: "string", + }, + { + code: "type", + type: "string", + }, + { + code: "note", + type: "string", + }, + { + code: "categoryCode", + type: "string", + export: async function (value: string | null) { + if (!value) { + return null + } + const category = await getCategoryByCode(value) + return category?.name + }, + import: async function (value: string) { + const category = await importCategory(value) + return category?.code + }, + }, + { + code: "projectCode", + type: "string", + export: async function (value: string | null) { + if (!value) { + return null + } + const project = await getProjectByCode(value) + return project?.name + }, + import: async function (value: string) { + const project = await importProject(value) + return project?.code + }, + }, + { + code: "issuedAt", + type: "date", + export: async function (value: Date | null) { + if (!value || isNaN(value.getTime())) { + return null + } + + try { + return formatDate(value, "yyyy-MM-dd") + } catch (error) { + return null + } + }, + import: async function (value: string) { + try { + return new Date(value) + } catch (error) { + return null + } + }, + }, +] + +export const exportImportFieldsMapping = exportImportFields.reduce((acc, field) => { + acc[field.code] = field + return acc +}, {} as Record) + +export const importProject = async (name: string) => { + const code = codeFromName(name) + + const existingProject = await prisma.project.findFirst({ + where: { + OR: [{ code }, { name }], + }, + }) + + if (existingProject) { + return existingProject + } + + return await createProject({ code, name }) +} + +export const importCategory = async (name: string) => { + const code = codeFromName(name) + + const existingCategory = await prisma.category.findFirst({ + where: { + OR: [{ code }, { name }], + }, + }) + + if (existingCategory) { + return existingCategory + } + + return await createCategory({ code, name }) +} diff --git a/data/projects.ts b/data/projects.ts index 1ecee9d..db02645 100644 --- a/data/projects.ts +++ b/data/projects.ts @@ -11,6 +11,12 @@ export const getProjects = cache(async () => { }) }) +export const getProjectByCode = cache(async (code: string) => { + return await prisma.project.findUnique({ + where: { code }, + }) +}) + export const createProject = async (project: Prisma.ProjectCreateInput) => { if (!project.code) { project.code = codeFromName(project.name as string) diff --git a/lib/files.ts b/lib/files.ts index 30b7b40..e8dc1df 100644 --- a/lib/files.ts +++ b/lib/files.ts @@ -6,6 +6,7 @@ export const FILE_ACCEPTED_MIMETYPES = "image/*,.pdf,.doc,.docx,.xls,.xlsx" export const FILE_UPLOAD_PATH = path.resolve(process.env.UPLOAD_PATH || "./uploads") export const FILE_UNSORTED_UPLOAD_PATH = path.join(FILE_UPLOAD_PATH, "unsorted") export const FILE_PREVIEWS_PATH = path.join(FILE_UPLOAD_PATH, "previews") +export const FILE_IMPORT_CSV_UPLOAD_PATH = path.join(FILE_UPLOAD_PATH, "csv") export async function getUnsortedFileUploadPath(filename: string) { const fileUuid = randomUUID() diff --git a/lib/stats.ts b/lib/stats.ts index 14077de..e2497be 100644 --- a/lib/stats.ts +++ b/lib/stats.ts @@ -3,10 +3,11 @@ import { Transaction } from "@prisma/client" export function calcTotalPerCurrency(transactions: Transaction[]): Record { return transactions.reduce((acc, transaction) => { if (transaction.convertedCurrencyCode) { - acc[transaction.convertedCurrencyCode] = - (acc[transaction.convertedCurrencyCode] || 0) + (transaction.convertedTotal || 0) + acc[transaction.convertedCurrencyCode.toUpperCase()] = + (acc[transaction.convertedCurrencyCode.toUpperCase()] || 0) + (transaction.convertedTotal || 0) } else if (transaction.currencyCode) { - acc[transaction.currencyCode] = (acc[transaction.currencyCode] || 0) + (transaction.total || 0) + acc[transaction.currencyCode.toUpperCase()] = + (acc[transaction.currencyCode.toUpperCase()] || 0) + (transaction.total || 0) } return acc }, {} as Record) diff --git a/package-lock.json b/package-lock.json index cfb4bce..76ac910 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "dependencies": { "@fast-csv/format": "^5.0.2", + "@fast-csv/parse": "^5.0.2", "@hookform/resolvers": "^4.1.2", "@prisma/client": "^6.4.1", "@radix-ui/react-avatar": "^1.1.3", @@ -679,6 +680,20 @@ "lodash.isnil": "^4.0.0" } }, + "node_modules/@fast-csv/parse": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.2.tgz", + "integrity": "sha512-gMu1Btmm99TP+wc0tZnlH30E/F1Gw1Tah3oMDBHNPe9W8S68ixVHjt89Wg5lh7d9RuQMtwN+sGl5kxR891+fzw==", + "license": "MIT", + "dependencies": { + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, "node_modules/@floating-ui/core": { "version": "1.6.9", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.9.tgz", @@ -5597,6 +5612,12 @@ "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", "license": "MIT" }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -5622,6 +5643,12 @@ "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", "license": "MIT" }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -5629,6 +5656,12 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "license": "MIT" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", diff --git a/package.json b/package.json index 652459d..8af23b0 100644 --- a/package.json +++ b/package.json @@ -8,10 +8,12 @@ "build": "next build", "start": "next start", "lint": "next lint", - "seed": "ts-node prisma/seed.ts" + "seed": "ts-node prisma/seed.ts", + "postinstall": "prisma generate && prisma migrate deploy && npm run seed" }, "dependencies": { "@fast-csv/format": "^5.0.2", + "@fast-csv/parse": "^5.0.2", "@hookform/resolvers": "^4.1.2", "@prisma/client": "^6.4.1", "@radix-ui/react-avatar": "^1.1.3", diff --git a/prisma/migrations/20250321141756_improve_fields/migration.sql b/prisma/migrations/20250321141756_improve_fields/migration.sql new file mode 100644 index 0000000..5a7781f --- /dev/null +++ b/prisma/migrations/20250321141756_improve_fields/migration.sql @@ -0,0 +1,42 @@ +/* + Warnings: + + - The primary key for the `fields` table will be changed. If it partially fails, the table could be left without primary key constraint. + - You are about to drop the column `id` on the `fields` table. All the data in the column will be lost. + +*/ +-- RedefineTables +PRAGMA defer_foreign_keys=ON; +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_fields" ( + "code" TEXT NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL, + "type" TEXT NOT NULL DEFAULT 'string', + "llm_prompt" TEXT, + "options" JSONB, + "is_required" BOOLEAN NOT NULL DEFAULT false, + "is_extra" BOOLEAN NOT NULL DEFAULT true +); +INSERT INTO "new_fields" ("code", "is_extra", "is_required", "llm_prompt", "name", "options", "type") SELECT "code", "is_extra", "is_required", "llm_prompt", "name", "options", "type" FROM "fields"; +DROP TABLE "fields"; +ALTER TABLE "new_fields" RENAME TO "fields"; +PRAGMA foreign_keys=ON; +PRAGMA defer_foreign_keys=OFF; + +-- CreateIndex +CREATE INDEX "transactions_project_id_idx" ON "transactions"("project_id"); + +-- CreateIndex +CREATE INDEX "transactions_category_id_idx" ON "transactions"("category_id"); + +-- CreateIndex +CREATE INDEX "transactions_issued_at_idx" ON "transactions"("issued_at"); + +-- CreateIndex +CREATE INDEX "transactions_name_idx" ON "transactions"("name"); + +-- CreateIndex +CREATE INDEX "transactions_merchant_idx" ON "transactions"("merchant"); + +-- CreateIndex +CREATE INDEX "transactions_total_idx" ON "transactions"("total"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 1ba7062..1ed95fe 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -42,10 +42,9 @@ model Project { } model Field { - id String @id @default(uuid()) - code String @unique + code String @id name String - type String + type String @default("string") llm_prompt String? options Json? isRequired Boolean @default(false) @map("is_required") diff --git a/prisma/seed.ts b/prisma/seed.ts index b052cd7..7699eed 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -51,8 +51,7 @@ const settings = [ { code: "prompt_analyse_new_file", name: "Prompt for Analyze Transaction", - description: - "Allowed variables: {fields}, {categories}, {categories.code}, {projects}, {projects.code}, {json_structure}", + description: "Allowed variables: {fields}, {categories}, {categories.code}, {projects}, {projects.code}", value: `You are an accountant and invoice analysis assistant. Extract the following information from the given invoice: @@ -66,11 +65,7 @@ And projects are: {projects} -If you can't find something leave it blank. Return only valid JSON with these fields: - -{json_structure} - -Return only one object. Do not include any other text in your response!`, +If you can't find something leave it blank. Return only one object. Do not include any other text in your response!`, }, { code: "is_welcome_message_hidden", @@ -309,7 +304,7 @@ const fields = [ { code: "name", name: "Name", - type: "text", + type: "string", llm_prompt: "human readable name, summarize what is the invoice about", isRequired: true, isExtra: false, @@ -317,7 +312,7 @@ const fields = [ { code: "description", name: "Description", - type: "text", + type: "string", llm_prompt: "description of the transaction", isRequired: false, isExtra: false, @@ -325,7 +320,7 @@ const fields = [ { code: "merchant", name: "Merchant", - type: "text", + type: "string", llm_prompt: "merchant name", isRequired: false, isExtra: false, @@ -333,7 +328,7 @@ const fields = [ { code: "type", name: "Type", - type: "text", + type: "string", llm_prompt: "", isRequired: false, isExtra: false, @@ -349,7 +344,7 @@ const fields = [ { code: "currencyCode", name: "Currency", - type: "text", + type: "string", llm_prompt: "currency code, ISO 4217 three letter code like USD, EUR, including crypto codes like BTC, ETH, etc", isRequired: false, isExtra: false, @@ -365,7 +360,7 @@ const fields = [ { code: "convertedCurrencyCode", name: "Converted Currency Code", - type: "text", + type: "string", llm_prompt: "", isRequired: false, isExtra: false, @@ -373,7 +368,7 @@ const fields = [ { code: "note", name: "Note", - type: "text", + type: "string", llm_prompt: "", isRequired: false, isExtra: false, @@ -381,7 +376,7 @@ const fields = [ { code: "categoryCode", name: "Category", - type: "text", + type: "string", llm_prompt: "category code, one of: {categories.code}", isRequired: false, isExtra: false, @@ -389,7 +384,7 @@ const fields = [ { code: "projectCode", name: "Project", - type: "select", + type: "string", llm_prompt: "project code, one of: {projects.code}", isRequired: false, isExtra: false, @@ -397,7 +392,7 @@ const fields = [ { code: "issuedAt", name: "Issued At", - type: "date", + type: "string", llm_prompt: "issued at date (YYYY-MM-DD format)", isRequired: false, isExtra: false, @@ -405,7 +400,7 @@ const fields = [ { code: "text", name: "Extracted Text", - type: "text", + type: "string", llm_prompt: "extract all recognised text from the invoice", isRequired: false, isExtra: false,