feat: use structured output, import CSV, bugfixes

app/ai/analyze.ts

@@ -1,11 +1,12 @@
 import { Category, Field, File, Project } from "@prisma/client"
 import OpenAI from "openai"
-import { ChatCompletion } from "openai/resources/index.mjs"
 import { buildLLMPrompt } from "./prompt"
+import { fieldsToJsonSchema } from "./schema"
 
-const MAX_PAGES_TO_ANALYZE = 3
+const MAX_PAGES_TO_ANALYZE = 4
 
 type AnalyzeAttachment = {
+  filename: string
   contentType: string
   base64: string
 }
@@ -32,7 +33,11 @@ export const retrieveFileContentForAI = async (file: File, page: number): Promis
   const buffer = await blob.arrayBuffer()
   const base64 = Buffer.from(buffer).toString("base64")
 
-  return { contentType: response.headers.get("Content-Type") || file.mimetype, base64: base64 }
+  return {
+    filename: file.filename,
+    contentType: response.headers.get("Content-Type") || file.mimetype,
+    base64: base64,
+  }
 }
 
 export async function analyzeTransaction(
@@ -49,33 +54,41 @@ export async function analyzeTransaction(
   })
 
   const prompt = buildLLMPrompt(promptTemplate, fields, categories, projects)
+  const schema = fieldsToJsonSchema(fields)
 
   console.log("PROMPT:", prompt)
+  console.log("SCHEMA:", schema)
 
   try {
-    const response = await openai.chat.completions.create({
-      model: "gpt-4o-mini",
-      messages: [
+    const response = await openai.responses.create({
+      model: "gpt-4o-mini-2024-07-18",
+      input: [
         {
           role: "user",
-          content: [
-            { type: "text", text: prompt || "" },
-            ...attachments.slice(0, MAX_PAGES_TO_ANALYZE).map((attachment) => ({
-              type: "image_url" as const,
-              image_url: {
-                url: `data:${attachment.contentType};base64,${attachment.base64}`,
-              },
-            })),
-          ],
+          content: prompt,
+        },
+        {
+          role: "user",
+          content: attachments.map((attachment) => ({
+            type: "input_image",
+            image_url: `data:${attachment.contentType};base64,${attachment.base64}`,
+          })),
         },
       ],
+      text: {
+        format: {
+          type: "json_schema",
+          name: "transaction",
+          schema: schema,
+          strict: true,
+        },
+      },
     })
 
-    console.log("ChatGPT response:", response.choices[0].message)
+    console.log("ChatGPT response:", response.output_text)
 
-    const cleanedJson = extractAndParseJSON(response)
-
-    return { success: true, data: cleanedJson }
+    const result = JSON.parse(response.output_text)
+    return { success: true, data: result }
   } catch (error) {
     console.error("AI Analysis error:", error)
     return {
@@ -84,56 +97,3 @@ export async function analyzeTransaction(
     }
   }
 }
-
-function extractAndParseJSON(response: ChatCompletion) {
-  try {
-    const content = response.choices?.[0]?.message?.content
-
-    if (!content) {
-      throw new Error("No response content from AI")
-    }
-
-    // Check for JSON in code blocks (handles ```json, ``` json, or just ```)
-    let jsonText = content.trim()
-    const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/
-    const jsonMatch = content.match(codeBlockRegex)
-
-    if (jsonMatch && jsonMatch[1]) {
-      jsonText = jsonMatch[1].trim()
-    }
-
-    // Try to parse the JSON
-    try {
-      return JSON.parse(jsonText)
-    } catch (parseError) {
-      // JSON might have unescaped characters, try to fix them
-      const fixedJsonText = escapeJsonString(jsonText)
-      return JSON.parse(fixedJsonText)
-    }
-  } catch (error) {
-    console.error("Error processing AI response:", error)
-    throw new Error(`Failed to extract valid JSON: ${error instanceof Error ? error.message : "Unknown error"}`)
-  }
-}
-
-function escapeJsonString(jsonStr: string) {
-  // This is a black magic to fix some AI-generated JSONs
-  if (jsonStr.trim().startsWith("{") && jsonStr.trim().endsWith("}")) {
-    return jsonStr.replace(/"([^"]*?)":(\s*)"(.*?)"/g, (match, key, space, value) => {
-      const escapedValue = value
-        .replace(/\\/g, "\\\\") // backslash
-        .replace(/"/g, '\\"') // double quotes
-        .replace(/\n/g, "\\n") // newline
-        .replace(/\r/g, "\\r") // carriage return
-        .replace(/\t/g, "\\t") // tab
-        .replace(/\f/g, "\\f") // form feed
-        .replace(/[\x00-\x1F\x7F-\x9F]/g, (c: string) => {
-          return "\\u" + ("0000" + c.charCodeAt(0).toString(16)).slice(-4)
-        })
-
-      return `"${key}":${space}"${escapedValue}"`
-    })
-  }
-
-  return jsonStr
-}

app/ai/prompt.ts

@@ -35,15 +35,5 @@ export function buildLLMPrompt(
   prompt = prompt.replace("{categories.code}", categories.map((category) => `${category.code}`).join(", "))
   prompt = prompt.replace("{projects.code}", projects.map((project) => `${project.code}`).join(", "))
 
-  prompt = prompt.replace(
-    "{json_structure}",
-    "{ " +
-      fields
-        .filter((field) => field.llm_prompt)
-        .map((field) => `${field.code}: ${field.type}`)
-        .join(", ") +
-      " }"
-  )
-
   return prompt
 }

app/ai/schema.ts

@@ -0,0 +1,16 @@
+import { Field } from "@prisma/client"
+
+export const fieldsToJsonSchema = (fields: Field[]) => {
+  const fieldsWithPrompt = fields.filter((field) => field.llm_prompt)
+  const schema = {
+    type: "object",
+    properties: fieldsWithPrompt.reduce((acc, field) => {
+      acc[field.code] = { type: field.type, description: field.llm_prompt || "" }
+      return acc
+    }, {} as Record<string, { type: string; description: string }>),
+    required: fieldsWithPrompt.map((field) => field.code),
+    additionalProperties: false,
+  }
+
+  return schema
+}

app/export/transactions/route.ts

@@ -1,4 +1,4 @@
-import { ExportFields, ExportFilters } from "@/data/export"
+import { ExportFields, ExportFilters, exportImportFieldsMapping } from "@/data/export_and_import"
 import { getFields } from "@/data/fields"
 import { getFilesByTransactionId } from "@/data/files"
 import { getTransactions } from "@/data/transactions"
@@ -18,11 +18,10 @@ export async function GET(request: Request) {
   const transactions = await getTransactions(filters)
   const existingFields = await getFields()
 
+  // Generate CSV file with all transactions
   try {
     const fieldKeys = fields.filter((field) => existingFields.some((f) => f.code === field))
-    const writeHeaders = fieldKeys.map((field) => existingFields.find((f) => f.code === field)?.name)
 
-    // Generate CSV file with all transactions
     let csvContent = ""
     const csvStream = format({ headers: fieldKeys, writeBOM: true, writeHeaders: false })
 
@@ -30,15 +29,25 @@ export async function GET(request: Request) {
       csvContent += chunk
     })
 
-    csvStream.write(writeHeaders)
-    transactions.forEach((transaction) => {
-      const row = fieldKeys.reduce((acc, key) => {
-        acc[key] = transaction[key as keyof typeof transaction] ?? ""
-        return acc
-      }, {} as Record<string, any>)
+    // Custom CSV headers
+    const headers = fieldKeys.map((field) => existingFields.find((f) => f.code === field)?.name ?? "UNKNOWN")
+    csvStream.write(headers)
+
+    // CSV rows
+    for (const transaction of transactions) {
+      const row: Record<string, any> = {}
+      for (const key of fieldKeys) {
+        const value = transaction[key as keyof typeof transaction] ?? ""
+        const exportFieldSettings = exportImportFieldsMapping[key]
+        if (exportFieldSettings && exportFieldSettings.export) {
+          row[key] = await exportFieldSettings.export(value)
+        } else {
+          row[key] = value
+        }
+      }
 
       csvStream.write(row)
-    })
+    }
     csvStream.end()
 
     // Wait for CSV generation to complete

app/import/csv/actions.tsx

@@ -0,0 +1,66 @@
+"use server"
+
+import { exportImportFieldsMapping } from "@/data/export_and_import"
+import { createTransaction } from "@/data/transactions"
+import { parse } from "@fast-csv/parse"
+import { revalidatePath } from "next/cache"
+
+export async function parseCSVAction(prevState: any, formData: FormData) {
+  const file = formData.get("file") as File
+  if (!file) {
+    return { success: false, error: "No file uploaded" }
+  }
+
+  if (!file.name.toLowerCase().endsWith(".csv")) {
+    return { success: false, error: "Only CSV files are allowed" }
+  }
+
+  try {
+    const buffer = Buffer.from(await file.arrayBuffer())
+    const rows: string[][] = []
+
+    const parser = parse()
+      .on("data", (row) => rows.push(row))
+      .on("error", (error) => {
+        throw error
+      })
+    parser.write(buffer)
+    parser.end()
+
+    // Wait for parsing to complete
+    await new Promise((resolve) => parser.on("end", resolve))
+
+    return { success: true, data: rows }
+  } catch (error) {
+    console.error("Error parsing CSV:", error)
+    return { success: false, error: "Failed to parse CSV file" }
+  }
+}
+
+export async function saveTransactionsAction(prevState: any, formData: FormData) {
+  try {
+    const rows = JSON.parse(formData.get("rows") as string) as Record<string, unknown>[]
+
+    for (const row of rows) {
+      const transactionData: Record<string, unknown> = {}
+      for (const [fieldCode, value] of Object.entries(row)) {
+        const fieldDef = exportImportFieldsMapping[fieldCode]
+        if (fieldDef?.import) {
+          transactionData[fieldCode] = await fieldDef.import(value as string)
+        } else {
+          transactionData[fieldCode] = value as string
+        }
+      }
+
+      await createTransaction(transactionData)
+    }
+
+    revalidatePath("/import/csv")
+    revalidatePath("/transactions")
+
+    return { success: true }
+  } catch (error) {
+    console.error("Error saving transactions:", error)
+    return { success: false, error: "Failed to save transactions: " + error }
+  }
+}

app/import/csv/page.tsx

@@ -0,0 +1,11 @@
+import { ImportCSVTable } from "@/components/import/csv"
+import { getFields } from "@/data/fields"
+
+export default async function CSVImportPage() {
+  const fields = await getFields()
+  return (
+    <div className="flex flex-col gap-4 p-4">
+      <ImportCSVTable fields={fields} />
+    </div>
+  )
+}

app/layout.tsx

@@ -43,7 +43,7 @@ export default async function RootLayout({ children }: { children: React.ReactNo
             <SidebarProvider>
               <MobileMenu settings={settings} unsortedFilesCount={unsortedFilesCount} />
               <AppSidebar settings={settings} unsortedFilesCount={unsortedFilesCount} />
-              <SidebarInset className="w-screen mt-[60px] md:mt-0">{children}</SidebarInset>
+              <SidebarInset className="w-full h-full mt-[60px] md:mt-0 overflow-auto">{children}</SidebarInset>
             </SidebarProvider>
             <Toaster />
           </ScreenDropArea>

app/loading.tsx

@@ -2,7 +2,7 @@ import { Loader2 } from "lucide-react"
 
 export default function AppLoading() {
   return (
-    <div className="flex items-center justify-center h-full w-full">
+    <div className="absolute inset-0 flex items-center justify-center h-full w-full">
       <Loader2 className="w-10 h-10 animate-spin text-muted-foreground" />
     </div>
   )

app/settings/backups/page.tsx

@@ -1,5 +1,6 @@
 "use client"
 
+import { FormError } from "@/components/forms/error"
 import { Button } from "@/components/ui/button"
 import { Card } from "@/components/ui/card"
 import { Download, Loader2 } from "lucide-react"
@@ -51,7 +52,7 @@ export default function BackupSettingsPage() {
             )}
           </Button>
         </form>
-        {restoreState?.error && <p className="text-red-500">{restoreState.error}</p>}
+        {restoreState?.error && <FormError>{restoreState.error}</FormError>}
       </Card>
     </div>
   )

app/settings/fields/page.tsx

@@ -17,7 +17,7 @@ export default async function FieldsSettingsPage() {
         items={fieldsWithActions}
         columns={[
           { key: "name", label: "Name", editable: true },
-          { key: "type", label: "Type", editable: true },
+          { key: "type", label: "Type", defaultValue: "string", editable: true },
           { key: "llm_prompt", label: "LLM Prompt", editable: true },
         ]}
         onDelete={async (code) => {

app/settings/layout.tsx

@@ -41,7 +41,7 @@ const settingsCategories = [
 export default function SettingsLayout({ children }: { children: React.ReactNode }) {
   return (
     <>
-      <div className="hidden space-y-6 p-10 pb-16 md:block">
+      <div className="space-y-6 p-10 pb-16">
         <div className="space-y-0.5">
           <h2 className="text-2xl font-bold tracking-tight">Settings</h2>
           <p className="text-muted-foreground">Customize your settings here</p>

app/settings/llm/page.tsx

@@ -1,13 +1,15 @@
 import LLMSettingsForm from "@/components/settings/llm-settings-form"
+import { getFields } from "@/data/fields"
 import { getSettings } from "@/data/settings"
 
 export default async function LlmSettingsPage() {
   const settings = await getSettings()
+  const fields = await getFields()
 
   return (
     <>
       <div className="w-full max-w-2xl">
-        <LLMSettingsForm settings={settings} />
+        <LLMSettingsForm settings={settings} fields={fields} />
       </div>
     </>
   )