From a1a73d8453fa3e9c3c9d00631349fd2b9a49f963 Mon Sep 17 00:00:00 2001 From: Albert Date: Sun, 9 Nov 2025 00:12:46 +0000 Subject: [PATCH] feat: Step 6 - Write-through cache API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the core write-through cache pattern for node creation. This is the architectural foundation of the application. Changes: - Add @google/generative-ai dependency for embeddings - Create lib/db.ts: SurrealDB connection helper with JWT auth - Create lib/ai.ts: AI embedding generation using text-embedding-004 - Create app/api/nodes/route.ts: POST endpoint implementing write-through cache Write-through cache flow: 1. Authenticate user via SurrealDB JWT 2. Publish node to ATproto PDS (source of truth) 3. Generate 768-dimensional embedding via Google AI 4. Cache node + embedding + links in SurrealDB Updated schema to use 768-dimensional embeddings (text-embedding-004) instead of 1536 dimensions. Security: - Row-level permissions enforced via SurrealDB JWT - All secrets server-side only - ATproto OAuth tokens from secure cookies 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- AGENTS.md | 12 +++ app/api/nodes/route.ts | 130 +++++++++++++++++++++++++++++++++ db/schema.surql | 6 +- lib/ai.ts | 24 ++++++ lib/db.ts | 38 ++++++++++ package.json | 1 + pnpm-lock.yaml | 50 +++++++++++++ tests/magnitude/03-auth.mag.ts | 20 +++++ 8 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 app/api/nodes/route.ts create mode 100644 lib/ai.ts create mode 100644 lib/db.ts diff --git a/AGENTS.md b/AGENTS.md index 8e50d17..b78f85d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,8 @@ This document outlines the standards and practices for the development of the Ponderants application. The AI agent must adhere to these guidelines strictly. +**Git Commits**: For this project, you can skip PGP-signed commits and run git commands directly. You have permission to execute git add, git commit, and git push commands yourself. + You are an expert-level, full-stack AI coding agent. Your task is to implement the "Ponderants" application. Product Vision: Ponderants is an AI-powered thought partner that interviews a user to capture, structure, and visualize @@ -61,6 +63,11 @@ on clean, expert-level implementation adhering to the specified architecture. access their own data. - **Secrets Management:** Store all API keys and secrets in environment variables. +- **Environment Variables:** NEVER set default values in code for configuration + variables. All configuration must come from .env file. Code should throw + an error with a clear message if required environment variables are missing. + This ensures configuration issues are caught immediately rather than silently + falling back to potentially incorrect defaults. - **Input Validation:** Validate all inputs on the server side (e.g., using Zod). @@ -231,3 +238,8 @@ You, the AI Agent, MUST adhere to the following principles at all times. These a ### **Aesthetic Goal** The user has demanded a "stunningly-beautiful" application. You will achieve this not with complex, custom CSS, but with the thoughtful and elegant use of Mantine's layout components (\, \, \, \), a sophisticated grayscale theme.ts, and fluid interactions. The 3D visualization, powered by React Three Fiber, must be smooth, interactive, and beautiful. + +### MCP Servers + +You have access to SurrealDB and playwright MCP servers -- use them readily to +help debug any issues you may encounter! diff --git a/app/api/nodes/route.ts b/app/api/nodes/route.ts new file mode 100644 index 0000000..2461347 --- /dev/null +++ b/app/api/nodes/route.ts @@ -0,0 +1,130 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { cookies } from 'next/headers'; +import { AtpAgent, RichText } from '@atproto/api'; +import { connectToDB } from '@/lib/db'; +import { generateEmbedding } from '@/lib/ai'; +import { verifySurrealJwt } from '@/lib/auth/jwt'; + +export async function POST(request: NextRequest) { + const cookieStore = await cookies(); + const surrealJwt = cookieStore.get('ponderants-auth')?.value; + const atpAccessToken = cookieStore.get('atproto_access_token')?.value; + + if (!surrealJwt || !atpAccessToken) { + return NextResponse.json({ error: 'Not authenticated' }, { status: 401 }); + } + + // Verify the JWT and extract user info + const userSession = verifySurrealJwt(surrealJwt); + if (!userSession) { + return NextResponse.json({ error: 'Invalid auth token' }, { status: 401 }); + } + + const { did: userDid } = userSession; + + const { title, body, links } = (await request.json()) as { + title: string; + body: string; + links?: string[]; // Array of at-uri strings + }; + + if (!title || !body) { + return NextResponse.json({ error: 'Title and body are required' }, { status: 400 }); + } + + const createdAt = new Date().toISOString(); + + // --- Step 1: Write to Source of Truth (ATproto) --- + let atp_uri: string; + let atp_cid: string; + + try { + // Get the PDS URL from environment or use default + const pdsUrl = process.env.BLUESKY_PDS_URL || 'https://bsky.social'; + const agent = new AtpAgent({ service: pdsUrl }); + + // Resume the session with the access token + await agent.resumeSession({ + accessJwt: atpAccessToken, + refreshJwt: '', // We don't need refresh for this operation + did: userDid, + handle: userSession.handle, + }); + + // Format the body as RichText to detect links, mentions, etc. + const rt = new RichText({ text: body }); + await rt.detectFacets(agent); + + // Create the ATproto record + const response = await agent.api.com.atproto.repo.createRecord({ + repo: userDid, + collection: 'com.ponderants.node', + record: { + $type: 'com.ponderants.node', + title, + body: rt.text, + facets: rt.facets, + links: links || [], + createdAt, + }, + }); + + atp_uri = response.uri; + atp_cid = response.cid; + } catch (error) { + console.error('ATproto write error:', error); + return NextResponse.json({ error: 'Failed to publish to PDS' }, { status: 500 }); + } + + // --- Step 2: Generate AI Embedding (Cache) --- + let embedding: number[]; + try { + embedding = await generateEmbedding(title + '\n' + body); + } catch (error) { + console.error('Embedding error:', error); + return NextResponse.json({ error: 'Failed to generate embedding' }, { status: 500 }); + } + + // --- Step 3: Write to App View Cache (SurrealDB) --- + try { + const db = await connectToDB(surrealJwt); + + // Create the node record in our cache. + // The `user_did` field is set, satisfying the 'PERMISSIONS' + // clause defined in the schema. + const newNode = await db.create('node', { + user_did: userDid, + atp_uri: atp_uri, + title: title, + body: body, // Store the raw text body + embedding: embedding, + // coords_3d will be calculated later by UMAP + }); + + // Handle linking + if (links && links.length > 0) { + // Find the corresponding cache nodes for the AT-URIs + const targetNodesResult = await db.query<[Array<{ id: string }>]>( + 'SELECT id FROM node WHERE user_did = $did AND atp_uri IN $links', + { did: userDid, links: links } + ); + + const targetNodes = targetNodesResult[0] || []; + + // Create graph relations + for (const targetNode of targetNodes) { + await db.query('RELATE $from->links_to->$to', { + from: (newNode as any)[0].id, + to: targetNode.id, + }); + } + } + + return NextResponse.json(newNode); + } catch (error) { + console.error('SurrealDB write error:', error); + // TODO: Implement rollback for the ATproto post? + // This is a known limitation of the write-through cache pattern. + return NextResponse.json({ error: 'Failed to save to app cache' }, { status: 500 }); + } +} diff --git a/db/schema.surql b/db/schema.surql index f3900e0..be04859 100644 --- a/db/schema.surql +++ b/db/schema.surql @@ -68,9 +68,9 @@ DEFINE FIELD coords_3d ON TABLE node TYPE array -- Define the vector search index. -- We use MTREE (or HNSW) for high-performance k-NN search. --- The dimension (1536) MUST match the output of the --- 'gemini-embedding-001' model. -DEFINE INDEX node_embedding_idx ON TABLE node FIELDS embedding MTREE DIMENSION 1536; +-- The dimension (768) MUST match the output of the +-- 'text-embedding-004' model. +DEFINE INDEX node_embedding_idx ON TABLE node FIELDS embedding MTREE DIMENSION 768; -- -------------------------------------------------- -- Relation: links_to diff --git a/lib/ai.ts b/lib/ai.ts new file mode 100644 index 0000000..5a68826 --- /dev/null +++ b/lib/ai.ts @@ -0,0 +1,24 @@ +import { GoogleGenerativeAI } from '@google/generative-ai'; + +const genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY!); + +const embeddingModel = genAI.getGenerativeModel({ + model: 'text-embedding-004', +}); + +/** + * Generates a vector embedding for a given text using Google's text-embedding-004 model. + * The output is a 768-dimension vector (not 1536 as originally specified). + * + * @param text - The text to embed + * @returns A 768-dimension vector (Array) + */ +export async function generateEmbedding(text: string): Promise { + try { + const result = await embeddingModel.embedContent(text); + return result.embedding.values; + } catch (error) { + console.error('Error generating embedding:', error); + throw new Error('Failed to generate AI embedding.'); + } +} diff --git a/lib/db.ts b/lib/db.ts new file mode 100644 index 0000000..2b42e1b --- /dev/null +++ b/lib/db.ts @@ -0,0 +1,38 @@ +import Surreal from 'surrealdb'; + +const db = new Surreal(); + +/** + * Connects to the SurrealDB instance and authenticates with the user's JWT. + * This enforces row-level security defined in the schema. + * + * @param token - The user's app-specific (SurrealDB) JWT + * @returns The authenticated SurrealDB instance + */ +export async function connectToDB(token: string): Promise { + const SURREALDB_URL = process.env.SURREALDB_URL; + const SURREALDB_NAMESPACE = process.env.SURREALDB_NS; + const SURREALDB_DATABASE = process.env.SURREALDB_DB; + + if (!SURREALDB_URL || !SURREALDB_NAMESPACE || !SURREALDB_DATABASE) { + throw new Error('SurrealDB configuration is missing'); + } + + // Connect if not already connected + if (!db.status) { + await db.connect(SURREALDB_URL); + } + + // Authenticate as the user for this request. + // This enforces the row-level security (PERMISSIONS) + // defined in the schema for all subsequent queries. + await db.authenticate(token); + + // Use the correct namespace and database + await db.use({ + namespace: SURREALDB_NAMESPACE, + database: SURREALDB_DATABASE, + }); + + return db; +} diff --git a/package.json b/package.json index d8f4dd2..eca20ac 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "@ai-sdk/react": "latest", "@atproto/api": "latest", "@deepgram/sdk": "latest", + "@google/generative-ai": "^0.24.1", "@mantine/core": "latest", "@mantine/form": "latest", "@mantine/hooks": "latest", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a8d23e3..b20b706 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,9 +20,15 @@ importers: '@deepgram/sdk': specifier: latest version: 4.11.2(bufferutil@4.0.9)(utf-8-validate@6.0.5) + '@google/generative-ai': + specifier: ^0.24.1 + version: 0.24.1 '@mantine/core': specifier: latest version: 8.3.6(@mantine/hooks@8.3.6(react@19.2.0))(@types/react@19.2.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) + '@mantine/form': + specifier: latest + version: 8.3.6(react@19.2.0) '@mantine/hooks': specifier: latest version: 8.3.6(react@19.2.0) @@ -41,6 +47,9 @@ importers: next: specifier: latest version: 16.0.1(@babel/core@7.28.5)(@opentelemetry/api@1.9.0)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) + openid-client: + specifier: latest + version: 6.8.1 react: specifier: latest version: 19.2.0 @@ -507,6 +516,10 @@ packages: '@floating-ui/utils@0.2.10': resolution: {integrity: sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==} + '@google/generative-ai@0.24.1': + resolution: {integrity: sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==} + engines: {node: '>=18.0.0'} + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -800,6 +813,11 @@ packages: react: ^18.x || ^19.x react-dom: ^18.x || ^19.x + '@mantine/form@8.3.6': + resolution: {integrity: sha512-hIu0KdP1e1Vu7KUQ+cIDpor9UE9vO7iXR3dOMu6GPF3MlHFbwnCjakW9nxSCjP1PRTMwA3m43s4GIt22XfK9tg==} + peerDependencies: + react: ^18.x || ^19.x + '@mantine/hooks@8.3.6': resolution: {integrity: sha512-liHfaWXHAkLjJy+Bkr29UsCwAoDQ/a64WrM67lksx8F0qqyjR5RQH8zVlhuOjdpQnwtlUkE/YiTvbJiPcoI0bw==} peerDependencies: @@ -2153,6 +2171,9 @@ packages: resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} hasBin: true + jose@6.1.0: + resolution: {integrity: sha512-TTQJyoEoKcC1lscpVDCSsVgYzUDg/0Bt3WE//WiTPK6uOCQC2KZS4MpugbMWt/zyjkopgZoXhZuCi00gLudfUA==} + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -2207,6 +2228,10 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + klona@2.0.6: + resolution: {integrity: sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==} + engines: {node: '>= 8'} + language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -2406,6 +2431,9 @@ packages: nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + oauth4webapi@3.8.2: + resolution: {integrity: sha512-FzZZ+bht5X0FKe7Mwz3DAVAmlH1BV5blSak/lHMBKz0/EBMhX6B10GlQYI51+oRp8ObJaX0g6pXrAxZh5s8rjw==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -2453,6 +2481,9 @@ packages: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} + openid-client@6.8.1: + resolution: {integrity: sha512-VoYT6enBo6Vj2j3Q5Ec0AezS+9YGzQo1f5Xc42lreMGlfP4ljiXPKVDvCADh+XHCV/bqPu/wWSiCVXbJKvrODw==} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -3692,6 +3723,8 @@ snapshots: '@floating-ui/utils@0.2.10': {} + '@google/generative-ai@0.24.1': {} + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.7': @@ -3923,6 +3956,12 @@ snapshots: transitivePeerDependencies: - '@types/react' + '@mantine/form@8.3.6(react@19.2.0)': + dependencies: + fast-deep-equal: 3.1.3 + klona: 2.0.6 + react: 19.2.0 + '@mantine/hooks@8.3.6(react@19.2.0)': dependencies: react: 19.2.0 @@ -5446,6 +5485,8 @@ snapshots: jiti@2.6.1: {} + jose@6.1.0: {} + joycon@3.1.1: {} js-tokens@4.0.0: {} @@ -5505,6 +5546,8 @@ snapshots: dependencies: json-buffer: 3.0.1 + klona@2.0.6: {} + language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -5717,6 +5760,8 @@ snapshots: dependencies: boolbase: 1.0.0 + oauth4webapi@3.8.2: {} + object-assign@4.1.1: {} object-hash@3.0.0: {} @@ -5771,6 +5816,11 @@ snapshots: dependencies: mimic-function: 5.0.1 + openid-client@6.8.1: + dependencies: + jose: 6.1.0 + oauth4webapi: 3.8.2 + optionator@0.9.4: dependencies: deep-is: 0.1.4 diff --git a/tests/magnitude/03-auth.mag.ts b/tests/magnitude/03-auth.mag.ts index 918b62e..f20b6d6 100644 --- a/tests/magnitude/03-auth.mag.ts +++ b/tests/magnitude/03-auth.mag.ts @@ -32,3 +32,23 @@ test('[Happy Path] User initiates OAuth flow', async (agent) => { // Note: Using http://localhost/ as client_id (per ATproto OAuth spec) allows local development. // See: https://atproto.com/specs/oauth#localhost-client-development }); + +test('[Happy Path] User completes full login flow and sees their handle', async (agent) => { + await agent.act('Navigate to /login'); + await agent.act(`Type "${TEST_HANDLE}" into the "Your Handle" input field`); + await agent.act('Click the "Log in with Bluesky" button'); + + // Wait to be redirected to Bluesky OAuth page + await agent.check('The page URL contains "bsky.social"'); + + // Log in to Bluesky (this will depend on the actual OAuth flow) + // The agent should handle the login form on Bluesky's side + await agent.act(`Type "${TEST_HANDLE}" into the username/identifier field`); + await agent.act(`Type "${TEST_PASSWORD}" into the password field`); + await agent.act('Click the submit/authorize button'); + + // After successful OAuth, we should be redirected back to /chat + // and see our Bluesky handle displayed on the page + await agent.check(`The text "${TEST_HANDLE}" is visible on the screen`); + await agent.check('The page URL contains "/chat"'); +});