feat: use structured output, import CSV, bugfixes

This commit is contained in:
Vasily Zubarev
2025-03-21 18:42:14 +01:00
parent 33727a431e
commit f6dc617eae
35 changed files with 735 additions and 195 deletions

View File

@@ -13,8 +13,4 @@ And projects are:
{projects}
If you can't find something leave it blank. Return only valid JSON with these fields:
{json_structure}
Do not include any other text in your response!"
If you can't find something leave it blank. Return only one object. Do not include any other text in your response!"

View File

@@ -1,11 +1,12 @@
import { Category, Field, File, Project } from "@prisma/client"
import OpenAI from "openai"
import { ChatCompletion } from "openai/resources/index.mjs"
import { buildLLMPrompt } from "./prompt"
import { fieldsToJsonSchema } from "./schema"
const MAX_PAGES_TO_ANALYZE = 3
const MAX_PAGES_TO_ANALYZE = 4
type AnalyzeAttachment = {
filename: string
contentType: string
base64: string
}
@@ -32,7 +33,11 @@ export const retrieveFileContentForAI = async (file: File, page: number): Promis
const buffer = await blob.arrayBuffer()
const base64 = Buffer.from(buffer).toString("base64")
return { contentType: response.headers.get("Content-Type") || file.mimetype, base64: base64 }
return {
filename: file.filename,
contentType: response.headers.get("Content-Type") || file.mimetype,
base64: base64,
}
}
export async function analyzeTransaction(
@@ -49,33 +54,41 @@ export async function analyzeTransaction(
})
const prompt = buildLLMPrompt(promptTemplate, fields, categories, projects)
const schema = fieldsToJsonSchema(fields)
console.log("PROMPT:", prompt)
console.log("SCHEMA:", schema)
try {
const response = await openai.chat.completions.create({
model: "gpt-4o-mini",
messages: [
const response = await openai.responses.create({
model: "gpt-4o-mini-2024-07-18",
input: [
{
role: "user",
content: [
{ type: "text", text: prompt || "" },
...attachments.slice(0, MAX_PAGES_TO_ANALYZE).map((attachment) => ({
type: "image_url" as const,
image_url: {
url: `data:${attachment.contentType};base64,${attachment.base64}`,
content: prompt,
},
{
role: "user",
content: attachments.map((attachment) => ({
type: "input_image",
image_url: `data:${attachment.contentType};base64,${attachment.base64}`,
})),
],
},
],
text: {
format: {
type: "json_schema",
name: "transaction",
schema: schema,
strict: true,
},
},
})
console.log("ChatGPT response:", response.choices[0].message)
console.log("ChatGPT response:", response.output_text)
const cleanedJson = extractAndParseJSON(response)
return { success: true, data: cleanedJson }
const result = JSON.parse(response.output_text)
return { success: true, data: result }
} catch (error) {
console.error("AI Analysis error:", error)
return {
@@ -84,56 +97,3 @@ export async function analyzeTransaction(
}
}
}
function extractAndParseJSON(response: ChatCompletion) {
try {
const content = response.choices?.[0]?.message?.content
if (!content) {
throw new Error("No response content from AI")
}
// Check for JSON in code blocks (handles ```json, ``` json, or just ```)
let jsonText = content.trim()
const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/
const jsonMatch = content.match(codeBlockRegex)
if (jsonMatch && jsonMatch[1]) {
jsonText = jsonMatch[1].trim()
}
// Try to parse the JSON
try {
return JSON.parse(jsonText)
} catch (parseError) {
// JSON might have unescaped characters, try to fix them
const fixedJsonText = escapeJsonString(jsonText)
return JSON.parse(fixedJsonText)
}
} catch (error) {
console.error("Error processing AI response:", error)
throw new Error(`Failed to extract valid JSON: ${error instanceof Error ? error.message : "Unknown error"}`)
}
}
function escapeJsonString(jsonStr: string) {
// This is a black magic to fix some AI-generated JSONs
if (jsonStr.trim().startsWith("{") && jsonStr.trim().endsWith("}")) {
return jsonStr.replace(/"([^"]*?)":(\s*)"(.*?)"/g, (match, key, space, value) => {
const escapedValue = value
.replace(/\\/g, "\\\\") // backslash
.replace(/"/g, '\\"') // double quotes
.replace(/\n/g, "\\n") // newline
.replace(/\r/g, "\\r") // carriage return
.replace(/\t/g, "\\t") // tab
.replace(/\f/g, "\\f") // form feed
.replace(/[\x00-\x1F\x7F-\x9F]/g, (c: string) => {
return "\\u" + ("0000" + c.charCodeAt(0).toString(16)).slice(-4)
})
return `"${key}":${space}"${escapedValue}"`
})
}
return jsonStr
}

View File

@@ -35,15 +35,5 @@ export function buildLLMPrompt(
prompt = prompt.replace("{categories.code}", categories.map((category) => `${category.code}`).join(", "))
prompt = prompt.replace("{projects.code}", projects.map((project) => `${project.code}`).join(", "))
prompt = prompt.replace(
"{json_structure}",
"{ " +
fields
.filter((field) => field.llm_prompt)
.map((field) => `${field.code}: ${field.type}`)
.join(", ") +
" }"
)
return prompt
}

16
app/ai/schema.ts Normal file
View File

@@ -0,0 +1,16 @@
import { Field } from "@prisma/client"
export const fieldsToJsonSchema = (fields: Field[]) => {
const fieldsWithPrompt = fields.filter((field) => field.llm_prompt)
const schema = {
type: "object",
properties: fieldsWithPrompt.reduce((acc, field) => {
acc[field.code] = { type: field.type, description: field.llm_prompt || "" }
return acc
}, {} as Record<string, { type: string; description: string }>),
required: fieldsWithPrompt.map((field) => field.code),
additionalProperties: false,
}
return schema
}

View File

@@ -1,4 +1,4 @@
import { ExportFields, ExportFilters } from "@/data/export"
import { ExportFields, ExportFilters, exportImportFieldsMapping } from "@/data/export_and_import"
import { getFields } from "@/data/fields"
import { getFilesByTransactionId } from "@/data/files"
import { getTransactions } from "@/data/transactions"
@@ -18,11 +18,10 @@ export async function GET(request: Request) {
const transactions = await getTransactions(filters)
const existingFields = await getFields()
// Generate CSV file with all transactions
try {
const fieldKeys = fields.filter((field) => existingFields.some((f) => f.code === field))
const writeHeaders = fieldKeys.map((field) => existingFields.find((f) => f.code === field)?.name)
// Generate CSV file with all transactions
let csvContent = ""
const csvStream = format({ headers: fieldKeys, writeBOM: true, writeHeaders: false })
@@ -30,15 +29,25 @@ export async function GET(request: Request) {
csvContent += chunk
})
csvStream.write(writeHeaders)
transactions.forEach((transaction) => {
const row = fieldKeys.reduce((acc, key) => {
acc[key] = transaction[key as keyof typeof transaction] ?? ""
return acc
}, {} as Record<string, any>)
// Custom CSV headers
const headers = fieldKeys.map((field) => existingFields.find((f) => f.code === field)?.name ?? "UNKNOWN")
csvStream.write(headers)
// CSV rows
for (const transaction of transactions) {
const row: Record<string, any> = {}
for (const key of fieldKeys) {
const value = transaction[key as keyof typeof transaction] ?? ""
const exportFieldSettings = exportImportFieldsMapping[key]
if (exportFieldSettings && exportFieldSettings.export) {
row[key] = await exportFieldSettings.export(value)
} else {
row[key] = value
}
}
csvStream.write(row)
})
}
csvStream.end()
// Wait for CSV generation to complete

View File

@@ -0,0 +1,66 @@
"use server"
import { exportImportFieldsMapping } from "@/data/export_and_import"
import { createTransaction } from "@/data/transactions"
import { parse } from "@fast-csv/parse"
import { revalidatePath } from "next/cache"
export async function parseCSVAction(prevState: any, formData: FormData) {
const file = formData.get("file") as File
if (!file) {
return { success: false, error: "No file uploaded" }
}
if (!file.name.toLowerCase().endsWith(".csv")) {
return { success: false, error: "Only CSV files are allowed" }
}
try {
const buffer = Buffer.from(await file.arrayBuffer())
const rows: string[][] = []
const parser = parse()
.on("data", (row) => rows.push(row))
.on("error", (error) => {
throw error
})
parser.write(buffer)
parser.end()
// Wait for parsing to complete
await new Promise((resolve) => parser.on("end", resolve))
return { success: true, data: rows }
} catch (error) {
console.error("Error parsing CSV:", error)
return { success: false, error: "Failed to parse CSV file" }
}
}
export async function saveTransactionsAction(prevState: any, formData: FormData) {
try {
const rows = JSON.parse(formData.get("rows") as string) as Record<string, unknown>[]
for (const row of rows) {
const transactionData: Record<string, unknown> = {}
for (const [fieldCode, value] of Object.entries(row)) {
const fieldDef = exportImportFieldsMapping[fieldCode]
if (fieldDef?.import) {
transactionData[fieldCode] = await fieldDef.import(value as string)
} else {
transactionData[fieldCode] = value as string
}
}
await createTransaction(transactionData)
}
revalidatePath("/import/csv")
revalidatePath("/transactions")
return { success: true }
} catch (error) {
console.error("Error saving transactions:", error)
return { success: false, error: "Failed to save transactions: " + error }
}
}

11
app/import/csv/page.tsx Normal file
View File

@@ -0,0 +1,11 @@
import { ImportCSVTable } from "@/components/import/csv"
import { getFields } from "@/data/fields"
export default async function CSVImportPage() {
const fields = await getFields()
return (
<div className="flex flex-col gap-4 p-4">
<ImportCSVTable fields={fields} />
</div>
)
}

View File

@@ -43,7 +43,7 @@ export default async function RootLayout({ children }: { children: React.ReactNo
<SidebarProvider>
<MobileMenu settings={settings} unsortedFilesCount={unsortedFilesCount} />
<AppSidebar settings={settings} unsortedFilesCount={unsortedFilesCount} />
<SidebarInset className="w-screen mt-[60px] md:mt-0">{children}</SidebarInset>
<SidebarInset className="w-full h-full mt-[60px] md:mt-0 overflow-auto">{children}</SidebarInset>
</SidebarProvider>
<Toaster />
</ScreenDropArea>

View File

@@ -2,7 +2,7 @@ import { Loader2 } from "lucide-react"
export default function AppLoading() {
return (
<div className="flex items-center justify-center h-full w-full">
<div className="absolute inset-0 flex items-center justify-center h-full w-full">
<Loader2 className="w-10 h-10 animate-spin text-muted-foreground" />
</div>
)

View File

@@ -1,5 +1,6 @@
"use client"
import { FormError } from "@/components/forms/error"
import { Button } from "@/components/ui/button"
import { Card } from "@/components/ui/card"
import { Download, Loader2 } from "lucide-react"
@@ -51,7 +52,7 @@ export default function BackupSettingsPage() {
)}
</Button>
</form>
{restoreState?.error && <p className="text-red-500">{restoreState.error}</p>}
{restoreState?.error && <FormError>{restoreState.error}</FormError>}
</Card>
</div>
)

View File

@@ -17,7 +17,7 @@ export default async function FieldsSettingsPage() {
items={fieldsWithActions}
columns={[
{ key: "name", label: "Name", editable: true },
{ key: "type", label: "Type", editable: true },
{ key: "type", label: "Type", defaultValue: "string", editable: true },
{ key: "llm_prompt", label: "LLM Prompt", editable: true },
]}
onDelete={async (code) => {

View File

@@ -41,7 +41,7 @@ const settingsCategories = [
export default function SettingsLayout({ children }: { children: React.ReactNode }) {
return (
<>
<div className="hidden space-y-6 p-10 pb-16 md:block">
<div className="space-y-6 p-10 pb-16">
<div className="space-y-0.5">
<h2 className="text-2xl font-bold tracking-tight">Settings</h2>
<p className="text-muted-foreground">Customize your settings here</p>

View File

@@ -1,13 +1,15 @@
import LLMSettingsForm from "@/components/settings/llm-settings-form"
import { getFields } from "@/data/fields"
import { getSettings } from "@/data/settings"
export default async function LlmSettingsPage() {
const settings = await getSettings()
const fields = await getFields()
return (
<>
<div className="w-full max-w-2xl">
<LLMSettingsForm settings={settings} />
<LLMSettingsForm settings={settings} fields={fields} />
</div>
</>
)

View File

@@ -6,6 +6,7 @@ import { FILE_ACCEPTED_MIMETYPES } from "@/lib/files"
import { Camera, Loader2 } from "lucide-react"
import { useRouter } from "next/navigation"
import { startTransition, useState } from "react"
import { FormError } from "../forms/error"
export default function DashboardDropZoneWidget() {
const router = useRouter()
@@ -65,7 +66,7 @@ export default function DashboardDropZoneWidget() {
upload receipts, invoices and any other documents for me to scan
</p>
)}
{uploadError && <p className="text-red-500">{uploadError}</p>}
{uploadError && <FormError>{uploadError}</FormError>}
</div>
</div>
</label>

View File

@@ -32,7 +32,7 @@ export async function StatsWidget({ filters }: { filters: TransactionFilters })
</CardHeader>
<CardContent>
{Object.entries(stats.totalIncomePerCurrency).map(([currency, total]) => (
<div key={currency} className="flex gap-2 items-center font-bold text-base first:text-2xl">
<div key={currency} className="flex gap-2 items-center font-bold text-base first:text-2xl text-green-500">
{formatCurrency(total, currency)}
</div>
))}
@@ -46,12 +46,7 @@ export async function StatsWidget({ filters }: { filters: TransactionFilters })
</CardHeader>
<CardContent>
{Object.entries(stats.totalExpensesPerCurrency).map(([currency, total]) => (
<div
key={currency}
className={`flex gap-2 items-center font-bold text-base first:text-2xl ${
total >= 0 ? "text-green-500" : "text-red-500"
}`}
>
<div key={currency} className="flex gap-2 items-center font-bold text-base first:text-2xl text-red-500">
{formatCurrency(total, currency)}
</div>
))}

View File

@@ -0,0 +1,3 @@
export function FormError({ children }: { children: React.ReactNode }) {
return <p className="text-red-500 mt-4 overflow-hidden">{children}</p>
}

193
components/import/csv.tsx Normal file
View File

@@ -0,0 +1,193 @@
"use client"
import { Button } from "@/components/ui/button"
import { Field } from "@prisma/client"
import { Upload } from "lucide-react"
import { useRouter } from "next/navigation"
import { startTransition, useActionState, useEffect, useState } from "react"
import { parseCSVAction, saveTransactionsAction } from "../../app/import/csv/actions"
import { FormError } from "../forms/error"
const MAX_PREVIEW_ROWS = 100
export function ImportCSVTable({ fields }: { fields: Field[] }) {
const router = useRouter()
const [parseState, parseAction] = useActionState(parseCSVAction, null)
const [saveState, saveAction] = useActionState(saveTransactionsAction, null)
const [csvSettings, setCSVSettings] = useState({
skipHeader: true,
})
const [csvData, setCSVData] = useState<string[][]>([])
const [columnMappings, setColumnMappings] = useState<string[]>([])
useEffect(() => {
if (parseState?.success && parseState.data) {
setCSVData(parseState.data)
if (parseState.data.length > 0) {
setColumnMappings(
parseState.data[0].map((value) => {
const field = fields.find((field) => field.code === value || field.name === value)
return field?.code || ""
})
)
} else {
setColumnMappings([])
}
}
}, [parseState])
useEffect(() => {
if (saveState?.success) {
router.push("/transactions")
}
}, [saveState, router])
const handleFileChange = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0]
if (!file) return
const formData = new FormData()
formData.append("file", file)
startTransition(async () => {
await parseAction(formData)
})
}
const handleMappingChange = (columnIndex: number, fieldCode: string) => {
setColumnMappings((prev) => {
const state = [...prev]
state[columnIndex] = fieldCode
return state
})
}
const handleSave = async () => {
if (csvData.length === 0) return
if (!isAtLeastOneFieldMapped(columnMappings)) {
alert("Please map at least one column to a field")
return
}
const startIndex = csvSettings.skipHeader ? 1 : 0
const processedRows = csvData.slice(startIndex).map((row) => {
const processedRow: Record<string, unknown> = {}
columnMappings.forEach((fieldCode, columnIndex) => {
if (!fieldCode || !row[columnIndex]) return
processedRow[fieldCode] = row[columnIndex]
})
return processedRow
})
const formData = new FormData()
formData.append("rows", JSON.stringify(processedRows))
startTransition(async () => {
await saveAction(formData)
})
}
return (
<>
{csvData.length === 0 && (
<div className="flex flex-col items-center justify-center gap-2 h-full min-h-[400px]">
<p className="text-muted-foreground">Upload your CSV file to import transactions</p>
<div className="flex flex-row gap-5 mt-8">
<div>
<input type="file" accept=".csv" className="hidden" id="csv-file" onChange={handleFileChange} />
<Button type="button" onClick={() => document.getElementById("csv-file")?.click()}>
<Upload className="mr-2" /> Import from CSV
</Button>
</div>
</div>
{parseState?.error && <FormError>{parseState.error}</FormError>}
</div>
)}
{csvData.length > 0 && (
<div>
<header className="flex flex-wrap items-center justify-between gap-2 mb-8">
<h2 className="flex flex-row gap-3 md:gap-5">
<span className="text-3xl font-bold tracking-tight">Import {csvData.length} items from CSV</span>
</h2>
<div className="flex gap-2">
<Button onClick={handleSave} disabled={saveState?.success}>
Import Transactions
</Button>
</div>
</header>
<div className="flex items-center gap-4 mb-4">
<label className="flex items-center gap-2 cursor-pointer">
<input
type="checkbox"
className="w-4 h-4"
id="skip-header"
defaultChecked={csvSettings.skipHeader}
onChange={(e) => setCSVSettings({ ...csvSettings, skipHeader: e.target.checked })}
/>
<span>First row is a header</span>
</label>
</div>
<div className="rounded-md border">
<div className="relative w-full overflow-auto">
<table className="w-full caption-bottom text-sm">
<thead className="[&_tr]:border-b">
<tr className="border-b transition-colors hover:bg-muted/50">
{csvData[0].map((_, index) => (
<th key={index} className="h-12 min-w-[200px] px-4 text-left align-middle font-medium">
<select
className="w-full p-2 border rounded-md"
value={columnMappings[index] || ""}
onChange={(e) => handleMappingChange(index, e.target.value)}
>
<option value="">Skip column</option>
{fields.map((field) => (
<option key={field.code} value={field.code}>
{field.name}
</option>
))}
</select>
</th>
))}
</tr>
</thead>
<tbody className="[&_tr:last-child]:border-0">
{csvData.slice(0, MAX_PREVIEW_ROWS).map((row, rowIndex) => (
<tr
key={rowIndex}
className={`border-b transition-colors hover:bg-muted/50 ${
rowIndex === 0 && csvSettings.skipHeader ? "line-through text-muted-foreground" : ""
}`}
>
{csvData[0].map((_, colIndex) => (
<td key={colIndex} className="p-4 align-middle">
{(row[colIndex] || "").toString().slice(0, 256)}
</td>
))}
</tr>
))}
</tbody>
</table>
</div>
</div>
{csvData.length > MAX_PREVIEW_ROWS && (
<p className="text-muted-foreground mt-4">and {csvData.length - MAX_PREVIEW_ROWS} more entries...</p>
)}
{saveState?.error && <FormError>{saveState.error}</FormError>}
</div>
)}
</>
)
}
function isAtLeastOneFieldMapped(columnMappings: string[]) {
return columnMappings.some((mapping) => mapping !== "")
}

View File

@@ -9,6 +9,7 @@ import { Button } from "@/components/ui/button"
import { Category, Currency } from "@prisma/client"
import { CircleCheckBig } from "lucide-react"
import { useActionState } from "react"
import { FormError } from "../forms/error"
export default function GlobalSettingsForm({
settings,
@@ -53,7 +54,7 @@ export default function GlobalSettingsForm({
)}
</div>
{saveState?.error && <p className="text-red-500">{saveState.error}</p>}
{saveState?.error && <FormError>{saveState.error}</FormError>}
</form>
)
}

View File

@@ -1,15 +1,20 @@
"use client"
import { fieldsToJsonSchema } from "@/app/ai/schema"
import { saveSettingsAction } from "@/app/settings/actions"
import { FormInput, FormTextarea } from "@/components/forms/simple"
import { Button } from "@/components/ui/button"
import { CircleCheckBig } from "lucide-react"
import { CircleCheckBig, Edit } from "lucide-react"
import Link from "next/link"
import { useActionState } from "react"
import { FormError } from "../forms/error"
import { Card, CardTitle } from "../ui/card"
export default function LLMSettingsForm({ settings }: { settings: Record<string, string> }) {
export default function LLMSettingsForm({ settings, fields }: { settings: Record<string, string>; fields: Field[] }) {
const [saveState, saveAction, pending] = useActionState(saveSettingsAction, null)
return (
<>
<form action={saveAction} className="space-y-4">
<FormInput title="OpenAI API Key" name="openai_api_key" defaultValue={settings.openai_api_key} />
@@ -39,7 +44,32 @@ export default function LLMSettingsForm({ settings }: { settings: Record<string,
)}
</div>
{saveState?.error && <p className="text-red-500">{saveState.error}</p>}
{saveState?.error && <FormError>{saveState.error}</FormError>}
</form>
<Card className="flex flex-col gap-4 p-4 bg-accent mt-20">
<CardTitle className="flex flex-row justify-between items-center gap-2">
<span className="text-md font-medium">
Current JSON Schema for{" "}
<a
href="https://platform.openai.com/docs/guides/structured-outputs?api-mode=responses&lang=javascript"
target="_blank"
className="underline"
>
structured output
</a>
</span>
<Link
href="/settings/fields"
className="text-xs underline inline-flex flex-row items-center gap-1 text-muted-foreground"
>
<Edit className="w-4 h-4" /> Edit Fields
</Link>
</CardTitle>
<pre className="text-xs overflow-hidden text-ellipsis">
{JSON.stringify(fieldsToJsonSchema(fields), null, 2)}
</pre>
</Card>
</>
)
}

View File

@@ -16,7 +16,7 @@ export function SideNav({ className, items, ...props }: SidebarNavProps) {
const pathname = usePathname()
return (
<nav className={cn("flex space-x-2 lg:flex-col lg:space-x-0 lg:space-y-1", className)} {...props}>
<nav className={cn("flex flex-wrap space-x-2 lg:flex-col lg:space-x-0 lg:space-y-1", className)} {...props}>
{items.map((item) => (
<Link
key={item.href}

View File

@@ -17,7 +17,7 @@ import {
SidebarTrigger,
useSidebar,
} from "@/components/ui/sidebar"
import { ClockArrowUp, FileText, LayoutDashboard, Settings, Sparkles, Upload } from "lucide-react"
import { ClockArrowUp, FileText, Import, LayoutDashboard, Settings, Sparkles, Upload } from "lucide-react"
import Link from "next/link"
import { usePathname } from "next/navigation"
import { useEffect } from "react"
@@ -31,7 +31,7 @@ export function AppSidebar({
settings: Record<string, string>
unsortedFilesCount: number
}) {
const { setOpenMobile } = useSidebar()
const { open, setOpenMobile } = useSidebar()
const pathname = usePathname()
const { notification } = useNotification()
@@ -44,6 +44,7 @@ export function AppSidebar({
<>
<Sidebar variant="inset" collapsible="icon">
<SidebarHeader>
{open ? (
<Link href="/" className="flex items-center gap-2 p-2">
<Avatar className="h-12 w-12 rounded-lg">
<AvatarImage src="/logo/256.png" />
@@ -53,14 +54,21 @@ export function AppSidebar({
<span className="truncate font-semibold">{settings.app_title}</span>
<span className="truncate text-xs">Beta</span>
</div>
<SidebarTrigger className="md:hidden" />
</Link>
) : (
<Link href="/">
<Avatar className="h-10 w-10 rounded-lg">
<AvatarImage src="/logo/256.png" />
<AvatarFallback className="rounded-lg">AI</AvatarFallback>
</Avatar>
</Link>
)}
</SidebarHeader>
<SidebarContent>
<SidebarGroup>
<UploadButton className="w-full mt-4 mb-2">
<Upload className="mr-2 h-4 w-4" />
<span>Upload</span>
<Upload className="h-4 w-4" />
{open ? <span>Upload</span> : ""}
</UploadButton>
</SidebarGroup>
<SidebarGroup>
@@ -121,6 +129,14 @@ export function AppSidebar({
<SidebarGroup>
<SidebarGroupContent>
<SidebarMenu>
<SidebarMenuItem>
<SidebarMenuButton asChild>
<Link href="/import/csv">
<Import />
Import from CSV
</Link>
</SidebarMenuButton>
</SidebarMenuItem>
<SidebarMenuItem>
<SidebarMenuButton asChild>
<Link href="https://vas3k.com/donate/" target="_blank">
@@ -129,6 +145,11 @@ export function AppSidebar({
</Link>
</SidebarMenuButton>
</SidebarMenuItem>
{!open && (
<SidebarMenuItem>
<SidebarTrigger />
</SidebarMenuItem>
)}
</SidebarMenu>
</SidebarGroupContent>
</SidebarGroup>

View File

@@ -9,9 +9,11 @@ import { FormInput, FormTextarea } from "@/components/forms/simple"
import { Button } from "@/components/ui/button"
import { Category, Currency, Project } from "@prisma/client"
import { format } from "date-fns"
import { Loader2 } from "lucide-react"
import { Import, Loader2 } from "lucide-react"
import Link from "next/link"
import { useRouter } from "next/navigation"
import { useActionState, useEffect, useState } from "react"
import { FormError } from "../forms/error"
export default function TransactionCreateForm({
categories,
@@ -110,7 +112,13 @@ export default function TransactionCreateForm({
<FormTextarea title="Note" name="note" defaultValue={formData.note} />
<div className="flex justify-end space-x-4 pt-6">
<div className="flex justify-between space-x-4 pt-6">
<Button type="button" variant="outline" className="aspect-square">
<Link href="/import/csv">
<Import className="h-4 w-4" />
</Link>
</Button>
<Button type="submit" disabled={isCreating}>
{isCreating ? (
<>
@@ -122,7 +130,7 @@ export default function TransactionCreateForm({
)}
</Button>
{createState?.error && <span className="text-red-500"> {createState.error}</span>}
{createState?.error && <FormError> {createState.error}</FormError>}
</div>
</form>
)

View File

@@ -12,6 +12,7 @@ import { format } from "date-fns"
import { Loader2 } from "lucide-react"
import { useRouter } from "next/navigation"
import { startTransition, useActionState, useEffect, useState } from "react"
import { FormError } from "../forms/error"
export default function TransactionEditForm({
transaction,
@@ -164,8 +165,8 @@ export default function TransactionEditForm({
)}
</Button>
{deleteState?.error && <span className="text-red-500"> {deleteState.error}</span>}
{saveState?.error && <span className="text-red-500"> {saveState.error}</span>}
{deleteState?.error && <FormError> {deleteState.error}</FormError>}
{saveState?.error && <FormError> {saveState.error}</FormError>}
</div>
</form>
)

View File

@@ -13,6 +13,7 @@ import { Button } from "@/components/ui/button"
import { Category, Currency, Field, File, Project } from "@prisma/client"
import { Brain, Loader2 } from "lucide-react"
import { startTransition, useActionState, useMemo, useState } from "react"
import { FormError } from "../forms/error"
export default function AnalyzeForm({
file,
@@ -133,7 +134,7 @@ export default function AnalyzeForm({
)}
</Button>
{analyzeError && <div className="mb-6 p-4 text-red-500 bg-red-50 rounded-md"> {analyzeError}</div>}
{analyzeError && <FormError> {analyzeError}</FormError>}
<form className="space-y-4" action={saveAsTransaction}>
<input type="hidden" name="fileId" value={file.id} />
@@ -288,8 +289,8 @@ export default function AnalyzeForm({
)}
</Button>
{deleteState?.error && <span className="text-red-500"> {deleteState.error}</span>}
{saveError && <span className="text-red-500"> {saveError}</span>}
{deleteState?.error && <FormError> {deleteState.error}</FormError>}
{saveError && <FormError> {saveError}</FormError>}
</div>
</form>
</>

View File

@@ -11,6 +11,12 @@ export const getCategories = cache(async () => {
})
})
export const getCategoryByCode = cache(async (code: string) => {
return await prisma.category.findUnique({
where: { code },
})
})
export const createCategory = async (category: Prisma.CategoryCreateInput) => {
if (!category.code) {
category.code = codeFromName(category.name as string)

View File

@@ -1,5 +0,0 @@
import { TransactionFilters } from "./transactions"
export type ExportFilters = TransactionFilters
export type ExportFields = string[]

155
data/export_and_import.ts Normal file
View File

@@ -0,0 +1,155 @@
import { prisma } from "@/lib/db"
import { codeFromName } from "@/lib/utils"
import { formatDate } from "date-fns"
import { createCategory, getCategoryByCode } from "./categories"
import { createProject, getProjectByCode } from "./projects"
import { TransactionFilters } from "./transactions"
export type ExportFilters = TransactionFilters
export type ExportFields = string[]
export const exportImportFields = [
{
code: "name",
type: "string",
},
{
code: "description",
type: "string",
},
{
code: "merchant",
type: "string",
},
{
code: "total",
type: "number",
export: async function (value: number) {
return value / 100
},
import: async function (value: string) {
const num = parseFloat(value)
return isNaN(num) ? 0.0 : num * 100
},
},
{
code: "currencyCode",
type: "string",
},
{
code: "convertedTotal",
type: "number",
export: async function (value: number | null) {
if (!value) {
return null
}
return value / 100
},
import: async function (value: string) {
const num = parseFloat(value)
return isNaN(num) ? 0.0 : num * 100
},
},
{
code: "convertedCurrencyCode",
type: "string",
},
{
code: "type",
type: "string",
},
{
code: "note",
type: "string",
},
{
code: "categoryCode",
type: "string",
export: async function (value: string | null) {
if (!value) {
return null
}
const category = await getCategoryByCode(value)
return category?.name
},
import: async function (value: string) {
const category = await importCategory(value)
return category?.code
},
},
{
code: "projectCode",
type: "string",
export: async function (value: string | null) {
if (!value) {
return null
}
const project = await getProjectByCode(value)
return project?.name
},
import: async function (value: string) {
const project = await importProject(value)
return project?.code
},
},
{
code: "issuedAt",
type: "date",
export: async function (value: Date | null) {
if (!value || isNaN(value.getTime())) {
return null
}
try {
return formatDate(value, "yyyy-MM-dd")
} catch (error) {
return null
}
},
import: async function (value: string) {
try {
return new Date(value)
} catch (error) {
return null
}
},
},
]
export const exportImportFieldsMapping = exportImportFields.reduce((acc, field) => {
acc[field.code] = field
return acc
}, {} as Record<string, any>)
export const importProject = async (name: string) => {
const code = codeFromName(name)
const existingProject = await prisma.project.findFirst({
where: {
OR: [{ code }, { name }],
},
})
if (existingProject) {
return existingProject
}
return await createProject({ code, name })
}
export const importCategory = async (name: string) => {
const code = codeFromName(name)
const existingCategory = await prisma.category.findFirst({
where: {
OR: [{ code }, { name }],
},
})
if (existingCategory) {
return existingCategory
}
return await createCategory({ code, name })
}

View File

@@ -11,6 +11,12 @@ export const getProjects = cache(async () => {
})
})
export const getProjectByCode = cache(async (code: string) => {
return await prisma.project.findUnique({
where: { code },
})
})
export const createProject = async (project: Prisma.ProjectCreateInput) => {
if (!project.code) {
project.code = codeFromName(project.name as string)

View File

@@ -6,6 +6,7 @@ export const FILE_ACCEPTED_MIMETYPES = "image/*,.pdf,.doc,.docx,.xls,.xlsx"
export const FILE_UPLOAD_PATH = path.resolve(process.env.UPLOAD_PATH || "./uploads")
export const FILE_UNSORTED_UPLOAD_PATH = path.join(FILE_UPLOAD_PATH, "unsorted")
export const FILE_PREVIEWS_PATH = path.join(FILE_UPLOAD_PATH, "previews")
export const FILE_IMPORT_CSV_UPLOAD_PATH = path.join(FILE_UPLOAD_PATH, "csv")
export async function getUnsortedFileUploadPath(filename: string) {
const fileUuid = randomUUID()

View File

@@ -3,10 +3,11 @@ import { Transaction } from "@prisma/client"
export function calcTotalPerCurrency(transactions: Transaction[]): Record<string, number> {
return transactions.reduce((acc, transaction) => {
if (transaction.convertedCurrencyCode) {
acc[transaction.convertedCurrencyCode] =
(acc[transaction.convertedCurrencyCode] || 0) + (transaction.convertedTotal || 0)
acc[transaction.convertedCurrencyCode.toUpperCase()] =
(acc[transaction.convertedCurrencyCode.toUpperCase()] || 0) + (transaction.convertedTotal || 0)
} else if (transaction.currencyCode) {
acc[transaction.currencyCode] = (acc[transaction.currencyCode] || 0) + (transaction.total || 0)
acc[transaction.currencyCode.toUpperCase()] =
(acc[transaction.currencyCode.toUpperCase()] || 0) + (transaction.total || 0)
}
return acc
}, {} as Record<string, number>)

33
package-lock.json generated
View File

@@ -9,6 +9,7 @@
"version": "0.1.0",
"dependencies": {
"@fast-csv/format": "^5.0.2",
"@fast-csv/parse": "^5.0.2",
"@hookform/resolvers": "^4.1.2",
"@prisma/client": "^6.4.1",
"@radix-ui/react-avatar": "^1.1.3",
@@ -679,6 +680,20 @@
"lodash.isnil": "^4.0.0"
}
},
"node_modules/@fast-csv/parse": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-5.0.2.tgz",
"integrity": "sha512-gMu1Btmm99TP+wc0tZnlH30E/F1Gw1Tah3oMDBHNPe9W8S68ixVHjt89Wg5lh7d9RuQMtwN+sGl5kxR891+fzw==",
"license": "MIT",
"dependencies": {
"lodash.escaperegexp": "^4.1.2",
"lodash.groupby": "^4.6.0",
"lodash.isfunction": "^3.0.9",
"lodash.isnil": "^4.0.0",
"lodash.isundefined": "^3.0.1",
"lodash.uniq": "^4.5.0"
}
},
"node_modules/@floating-ui/core": {
"version": "1.6.9",
"resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.9.tgz",
@@ -5597,6 +5612,12 @@
"integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==",
"license": "MIT"
},
"node_modules/lodash.groupby": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
"integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==",
"license": "MIT"
},
"node_modules/lodash.isboolean": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
@@ -5622,6 +5643,12 @@
"integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==",
"license": "MIT"
},
"node_modules/lodash.isundefined": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz",
"integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==",
"license": "MIT"
},
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -5629,6 +5656,12 @@
"dev": true,
"license": "MIT"
},
"node_modules/lodash.uniq": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
"integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==",
"license": "MIT"
},
"node_modules/loose-envify": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",

View File

@@ -8,10 +8,12 @@
"build": "next build",
"start": "next start",
"lint": "next lint",
"seed": "ts-node prisma/seed.ts"
"seed": "ts-node prisma/seed.ts",
"postinstall": "prisma generate && prisma migrate deploy && npm run seed"
},
"dependencies": {
"@fast-csv/format": "^5.0.2",
"@fast-csv/parse": "^5.0.2",
"@hookform/resolvers": "^4.1.2",
"@prisma/client": "^6.4.1",
"@radix-ui/react-avatar": "^1.1.3",

View File

@@ -0,0 +1,42 @@
/*
Warnings:
- The primary key for the `fields` table will be changed. If it partially fails, the table could be left without primary key constraint.
- You are about to drop the column `id` on the `fields` table. All the data in the column will be lost.
*/
-- RedefineTables
PRAGMA defer_foreign_keys=ON;
PRAGMA foreign_keys=OFF;
CREATE TABLE "new_fields" (
"code" TEXT NOT NULL PRIMARY KEY,
"name" TEXT NOT NULL,
"type" TEXT NOT NULL DEFAULT 'string',
"llm_prompt" TEXT,
"options" JSONB,
"is_required" BOOLEAN NOT NULL DEFAULT false,
"is_extra" BOOLEAN NOT NULL DEFAULT true
);
INSERT INTO "new_fields" ("code", "is_extra", "is_required", "llm_prompt", "name", "options", "type") SELECT "code", "is_extra", "is_required", "llm_prompt", "name", "options", "type" FROM "fields";
DROP TABLE "fields";
ALTER TABLE "new_fields" RENAME TO "fields";
PRAGMA foreign_keys=ON;
PRAGMA defer_foreign_keys=OFF;
-- CreateIndex
CREATE INDEX "transactions_project_id_idx" ON "transactions"("project_id");
-- CreateIndex
CREATE INDEX "transactions_category_id_idx" ON "transactions"("category_id");
-- CreateIndex
CREATE INDEX "transactions_issued_at_idx" ON "transactions"("issued_at");
-- CreateIndex
CREATE INDEX "transactions_name_idx" ON "transactions"("name");
-- CreateIndex
CREATE INDEX "transactions_merchant_idx" ON "transactions"("merchant");
-- CreateIndex
CREATE INDEX "transactions_total_idx" ON "transactions"("total");

View File

@@ -42,10 +42,9 @@ model Project {
}
model Field {
id String @id @default(uuid())
code String @unique
code String @id
name String
type String
type String @default("string")
llm_prompt String?
options Json?
isRequired Boolean @default(false) @map("is_required")

View File

@@ -51,8 +51,7 @@ const settings = [
{
code: "prompt_analyse_new_file",
name: "Prompt for Analyze Transaction",
description:
"Allowed variables: {fields}, {categories}, {categories.code}, {projects}, {projects.code}, {json_structure}",
description: "Allowed variables: {fields}, {categories}, {categories.code}, {projects}, {projects.code}",
value: `You are an accountant and invoice analysis assistant.
Extract the following information from the given invoice:
@@ -66,11 +65,7 @@ And projects are:
{projects}
If you can't find something leave it blank. Return only valid JSON with these fields:
{json_structure}
Return only one object. Do not include any other text in your response!`,
If you can't find something leave it blank. Return only one object. Do not include any other text in your response!`,
},
{
code: "is_welcome_message_hidden",
@@ -309,7 +304,7 @@ const fields = [
{
code: "name",
name: "Name",
type: "text",
type: "string",
llm_prompt: "human readable name, summarize what is the invoice about",
isRequired: true,
isExtra: false,
@@ -317,7 +312,7 @@ const fields = [
{
code: "description",
name: "Description",
type: "text",
type: "string",
llm_prompt: "description of the transaction",
isRequired: false,
isExtra: false,
@@ -325,7 +320,7 @@ const fields = [
{
code: "merchant",
name: "Merchant",
type: "text",
type: "string",
llm_prompt: "merchant name",
isRequired: false,
isExtra: false,
@@ -333,7 +328,7 @@ const fields = [
{
code: "type",
name: "Type",
type: "text",
type: "string",
llm_prompt: "",
isRequired: false,
isExtra: false,
@@ -349,7 +344,7 @@ const fields = [
{
code: "currencyCode",
name: "Currency",
type: "text",
type: "string",
llm_prompt: "currency code, ISO 4217 three letter code like USD, EUR, including crypto codes like BTC, ETH, etc",
isRequired: false,
isExtra: false,
@@ -365,7 +360,7 @@ const fields = [
{
code: "convertedCurrencyCode",
name: "Converted Currency Code",
type: "text",
type: "string",
llm_prompt: "",
isRequired: false,
isExtra: false,
@@ -373,7 +368,7 @@ const fields = [
{
code: "note",
name: "Note",
type: "text",
type: "string",
llm_prompt: "",
isRequired: false,
isExtra: false,
@@ -381,7 +376,7 @@ const fields = [
{
code: "categoryCode",
name: "Category",
type: "text",
type: "string",
llm_prompt: "category code, one of: {categories.code}",
isRequired: false,
isExtra: false,
@@ -389,7 +384,7 @@ const fields = [
{
code: "projectCode",
name: "Project",
type: "select",
type: "string",
llm_prompt: "project code, one of: {projects.code}",
isRequired: false,
isExtra: false,
@@ -397,7 +392,7 @@ const fields = [
{
code: "issuedAt",
name: "Issued At",
type: "date",
type: "string",
llm_prompt: "issued at date (YYYY-MM-DD format)",
isRequired: false,
isExtra: false,
@@ -405,7 +400,7 @@ const fields = [
{
code: "text",
name: "Extracted Text",
type: "text",
type: "string",
llm_prompt: "extract all recognised text from the invoice",
isRequired: false,
isExtra: false,