diff --git a/.claude/skills/langchain/SKILL.md b/.claude/skills/langchain/SKILL.md deleted file mode 100644 index cf667923..00000000 --- a/.claude/skills/langchain/SKILL.md +++ /dev/null @@ -1,480 +0,0 @@ ---- -name: langchain -description: Framework for building LLM-powered applications with agents, chains, and RAG. Supports multiple providers (OpenAI, Anthropic, Google), 500+ integrations, ReAct agents, tool calling, memory management, and vector store retrieval. Use for building chatbots, question-answering systems, autonomous agents, or RAG applications. Best for rapid prototyping and production deployments. -version: 1.0.0 -author: Orchestra Research -license: MIT -tags: [Agents, LangChain, RAG, Tool Calling, ReAct, Memory Management, Vector Stores, LLM Applications, Chatbots, Production] -dependencies: [langchain, langchain-core, langchain-openai, langchain-anthropic] ---- - -# LangChain - Build LLM Applications with Agents & RAG - -The most popular framework for building LLM-powered applications. - -## When to use LangChain - -**Use LangChain when:** -- Building agents with tool calling and reasoning (ReAct pattern) -- Implementing RAG (retrieval-augmented generation) pipelines -- Need to swap LLM providers easily (OpenAI, Anthropic, Google) -- Creating chatbots with conversation memory -- Rapid prototyping of LLM applications -- Production deployments with LangSmith observability - -**Metrics**: -- **119,000+ GitHub stars** -- **272,000+ repositories** use LangChain -- **500+ integrations** (models, vector stores, tools) -- **3,800+ contributors** - -**Use alternatives instead**: -- **LlamaIndex**: RAG-focused, better for document Q&A -- **LangGraph**: Complex stateful workflows, more control -- **Haystack**: Production search pipelines -- **Semantic Kernel**: Microsoft ecosystem - -## Quick start - -### Installation - -```bash -# Core library (Python 3.10+) -pip install -U langchain - -# With OpenAI -pip install langchain-openai - -# With Anthropic -pip install langchain-anthropic - -# Common extras -pip install langchain-community # 500+ integrations -pip install langchain-chroma # Vector store -``` - -### Basic LLM usage - -```python -from langchain_anthropic import ChatAnthropic - -# Initialize model -llm = ChatAnthropic(model="claude-sonnet-4-5-20250929") - -# Simple completion -response = llm.invoke("Explain quantum computing in 2 sentences") -print(response.content) -``` - -### Create an agent (ReAct pattern) - -```python -from langchain.agents import create_agent -from langchain_anthropic import ChatAnthropic - -# Define tools -def get_weather(city: str) -> str: - """Get current weather for a city.""" - return f"It's sunny in {city}, 72°F" - -def search_web(query: str) -> str: - """Search the web for information.""" - return f"Search results for: {query}" - -# Create agent (<10 lines!) -agent = create_agent( - model=ChatAnthropic(model="claude-sonnet-4-5-20250929"), - tools=[get_weather, search_web], - system_prompt="You are a helpful assistant. Use tools when needed." -) - -# Run agent -result = agent.invoke({"messages": [{"role": "user", "content": "What's the weather in Paris?"}]}) -print(result["messages"][-1].content) -``` - -## Core concepts - -### 1. Models - LLM abstraction - -```python -from langchain_openai import ChatOpenAI -from langchain_anthropic import ChatAnthropic -from langchain_google_genai import ChatGoogleGenerativeAI - -# Swap providers easily -llm = ChatOpenAI(model="gpt-4o") -llm = ChatAnthropic(model="claude-sonnet-4-5-20250929") -llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash-exp") - -# Streaming -for chunk in llm.stream("Write a poem"): - print(chunk.content, end="", flush=True) -``` - -### 2. Chains - Sequential operations - -```python -from langchain.chains import LLMChain -from langchain.prompts import PromptTemplate - -# Define prompt template -prompt = PromptTemplate( - input_variables=["topic"], - template="Write a 3-sentence summary about {topic}" -) - -# Create chain -chain = LLMChain(llm=llm, prompt=prompt) - -# Run chain -result = chain.run(topic="machine learning") -``` - -### 3. Agents - Tool-using reasoning - -**ReAct (Reasoning + Acting) pattern:** - -```python -from langchain.agents import create_tool_calling_agent, AgentExecutor -from langchain.tools import Tool - -# Define custom tool -calculator = Tool( - name="Calculator", - func=lambda x: eval(x), - description="Useful for math calculations. Input: valid Python expression." -) - -# Create agent with tools -agent = create_tool_calling_agent( - llm=llm, - tools=[calculator, search_web], - prompt="Answer questions using available tools" -) - -# Create executor -agent_executor = AgentExecutor(agent=agent, tools=[calculator], verbose=True) - -# Run with reasoning -result = agent_executor.invoke({"input": "What is 25 * 17 + 142?"}) -``` - -### 4. Memory - Conversation history - -```python -from langchain.memory import ConversationBufferMemory -from langchain.chains import ConversationChain - -# Add memory to track conversation -memory = ConversationBufferMemory() - -conversation = ConversationChain( - llm=llm, - memory=memory, - verbose=True -) - -# Multi-turn conversation -conversation.predict(input="Hi, I'm Alice") -conversation.predict(input="What's my name?") # Remembers "Alice" -``` - -## RAG (Retrieval-Augmented Generation) - -### Basic RAG pipeline - -```python -from langchain_community.document_loaders import WebBaseLoader -from langchain.text_splitter import RecursiveCharacterTextSplitter -from langchain_openai import OpenAIEmbeddings -from langchain_chroma import Chroma -from langchain.chains import RetrievalQA - -# 1. Load documents -loader = WebBaseLoader("https://docs.python.org/3/tutorial/") -docs = loader.load() - -# 2. Split into chunks -text_splitter = RecursiveCharacterTextSplitter( - chunk_size=1000, - chunk_overlap=200 -) -splits = text_splitter.split_documents(docs) - -# 3. Create embeddings and vector store -vectorstore = Chroma.from_documents( - documents=splits, - embedding=OpenAIEmbeddings() -) - -# 4. Create retriever -retriever = vectorstore.as_retriever(search_kwargs={"k": 4}) - -# 5. Create QA chain -qa_chain = RetrievalQA.from_chain_type( - llm=llm, - retriever=retriever, - return_source_documents=True -) - -# 6. Query -result = qa_chain({"query": "What are Python decorators?"}) -print(result["result"]) -print(f"Sources: {result['source_documents']}") -``` - -### Conversational RAG with memory - -```python -from langchain.chains import ConversationalRetrievalChain - -# RAG with conversation memory -qa = ConversationalRetrievalChain.from_llm( - llm=llm, - retriever=retriever, - memory=ConversationBufferMemory( - memory_key="chat_history", - return_messages=True - ) -) - -# Multi-turn RAG -qa({"question": "What is Python used for?"}) -qa({"question": "Can you elaborate on web development?"}) # Remembers context -``` - -## Advanced agent patterns - -### Structured output - -```python -from langchain_core.pydantic_v1 import BaseModel, Field - -# Define schema -class WeatherReport(BaseModel): - city: str = Field(description="City name") - temperature: float = Field(description="Temperature in Fahrenheit") - condition: str = Field(description="Weather condition") - -# Get structured response -structured_llm = llm.with_structured_output(WeatherReport) -result = structured_llm.invoke("What's the weather in SF? It's 65F and sunny") -print(result.city, result.temperature, result.condition) -``` - -### Parallel tool execution - -```python -from langchain.agents import create_tool_calling_agent - -# Agent automatically parallelizes independent tool calls -agent = create_tool_calling_agent( - llm=llm, - tools=[get_weather, search_web, calculator] -) - -# This will call get_weather("Paris") and get_weather("London") in parallel -result = agent.invoke({ - "messages": [{"role": "user", "content": "Compare weather in Paris and London"}] -}) -``` - -### Streaming agent execution - -```python -# Stream agent steps -for step in agent_executor.stream({"input": "Research AI trends"}): - if "actions" in step: - print(f"Tool: {step['actions'][0].tool}") - if "output" in step: - print(f"Output: {step['output']}") -``` - -## Common patterns - -### Multi-document QA - -```python -from langchain.chains.qa_with_sources import load_qa_with_sources_chain - -# Load multiple documents -docs = [ - loader.load("https://docs.python.org"), - loader.load("https://docs.numpy.org") -] - -# QA with source citations -chain = load_qa_with_sources_chain(llm, chain_type="stuff") -result = chain({"input_documents": docs, "question": "How to use numpy arrays?"}) -print(result["output_text"]) # Includes source citations -``` - -### Custom tools with error handling - -```python -from langchain.tools import tool - -@tool -def risky_operation(query: str) -> str: - """Perform a risky operation that might fail.""" - try: - # Your operation here - result = perform_operation(query) - return f"Success: {result}" - except Exception as e: - return f"Error: {str(e)}" - -# Agent handles errors gracefully -agent = create_agent(model=llm, tools=[risky_operation]) -``` - -### LangSmith observability - -```python -import os - -# Enable tracing -os.environ["LANGCHAIN_TRACING_V2"] = "true" -os.environ["LANGCHAIN_API_KEY"] = "your-api-key" -os.environ["LANGCHAIN_PROJECT"] = "my-project" - -# All chains/agents automatically traced -agent = create_agent(model=llm, tools=[calculator]) -result = agent.invoke({"input": "Calculate 123 * 456"}) - -# View traces at smith.langchain.com -``` - -## Vector stores - -### Chroma (local) - -```python -from langchain_chroma import Chroma - -vectorstore = Chroma.from_documents( - documents=docs, - embedding=OpenAIEmbeddings(), - persist_directory="./chroma_db" -) -``` - -### Pinecone (cloud) - -```python -from langchain_pinecone import PineconeVectorStore - -vectorstore = PineconeVectorStore.from_documents( - documents=docs, - embedding=OpenAIEmbeddings(), - index_name="my-index" -) -``` - -### FAISS (similarity search) - -```python -from langchain_community.vectorstores import FAISS - -vectorstore = FAISS.from_documents(docs, OpenAIEmbeddings()) -vectorstore.save_local("faiss_index") - -# Load later -vectorstore = FAISS.load_local("faiss_index", OpenAIEmbeddings()) -``` - -## Document loaders - -```python -# Web pages -from langchain_community.document_loaders import WebBaseLoader -loader = WebBaseLoader("https://example.com") - -# PDFs -from langchain_community.document_loaders import PyPDFLoader -loader = PyPDFLoader("paper.pdf") - -# GitHub -from langchain_community.document_loaders import GithubFileLoader -loader = GithubFileLoader(repo="user/repo", file_filter=lambda x: x.endswith(".py")) - -# CSV -from langchain_community.document_loaders import CSVLoader -loader = CSVLoader("data.csv") -``` - -## Text splitters - -```python -# Recursive (recommended for general text) -from langchain.text_splitter import RecursiveCharacterTextSplitter -splitter = RecursiveCharacterTextSplitter( - chunk_size=1000, - chunk_overlap=200, - separators=["\n\n", "\n", " ", ""] -) - -# Code-aware -from langchain.text_splitter import PythonCodeTextSplitter -splitter = PythonCodeTextSplitter(chunk_size=500) - -# Semantic (by meaning) -from langchain_experimental.text_splitter import SemanticChunker -splitter = SemanticChunker(OpenAIEmbeddings()) -``` - -## Best practices - -1. **Start simple** - Use `create_agent()` for most cases -2. **Enable streaming** - Better UX for long responses -3. **Add error handling** - Tools can fail, handle gracefully -4. **Use LangSmith** - Essential for debugging agents -5. **Optimize chunk size** - 500-1000 chars for RAG -6. **Version prompts** - Track changes in production -7. **Cache embeddings** - Expensive, cache when possible -8. **Monitor costs** - Track token usage with LangSmith - -## Performance benchmarks - -| Operation | Latency | Notes | -|-----------|---------|-------| -| Simple LLM call | ~1-2s | Depends on provider | -| Agent with 1 tool | ~3-5s | ReAct reasoning overhead | -| RAG retrieval | ~0.5-1s | Vector search + LLM | -| Embedding 1000 docs | ~10-30s | Depends on model | - -## LangChain vs LangGraph - -| Feature | LangChain | LangGraph | -|---------|-----------|-----------| -| **Best for** | Quick agents, RAG | Complex workflows | -| **Abstraction level** | High | Low | -| **Code to start** | <10 lines | ~30 lines | -| **Control** | Simple | Full control | -| **Stateful workflows** | Limited | Native | -| **Cyclic graphs** | No | Yes | -| **Human-in-loop** | Basic | Advanced | - -**Use LangGraph when:** -- Need stateful workflows with cycles -- Require fine-grained control -- Building multi-agent systems -- Production apps with complex logic - -## References - -- **[Agents Guide](references/agents.md)** - ReAct, tool calling, streaming -- **[RAG Guide](references/rag.md)** - Document loaders, retrievers, QA chains -- **[Integration Guide](references/integration.md)** - Vector stores, LangSmith, deployment - -## Resources - -- **GitHub**: https://github.com/langchain-ai/langchain ⭐ 119,000+ -- **Docs**: https://docs.langchain.com -- **API Reference**: https://reference.langchain.com/python -- **LangSmith**: https://smith.langchain.com (observability) -- **Version**: 0.3+ (stable) -- **License**: MIT - - diff --git a/.claude/skills/langchain/references/agents.md b/.claude/skills/langchain/references/agents.md deleted file mode 100644 index fe8e6fee..00000000 --- a/.claude/skills/langchain/references/agents.md +++ /dev/null @@ -1,499 +0,0 @@ -# LangChain Agents Guide - -Complete guide to building agents with ReAct, tool calling, and streaming. - -## What are agents? - -Agents combine language models with tools to solve complex tasks through reasoning and action: - -1. **Reasoning**: LLM decides what to do -2. **Acting**: Execute tools based on reasoning -3. **Observation**: Receive tool results -4. **Loop**: Repeat until task complete - -This is the **ReAct pattern** (Reasoning + Acting). - -## Basic agent creation - -```python -from langchain.agents import create_agent -from langchain_anthropic import ChatAnthropic - -# Define tools -def calculator(expression: str) -> str: - """Evaluate a math expression.""" - return str(eval(expression)) - -def search(query: str) -> str: - """Search for information.""" - return f"Results for: {query}" - -# Create agent -agent = create_agent( - model=ChatAnthropic(model="claude-sonnet-4-5-20250929"), - tools=[calculator, search], - system_prompt="You are a helpful assistant. Use tools when needed." -) - -# Run agent -result = agent.invoke({ - "messages": [{"role": "user", "content": "What is 25 * 17?"}] -}) -print(result["messages"][-1].content) -``` - -## Agent components - -### 1. Model - The reasoning engine - -```python -from langchain_openai import ChatOpenAI -from langchain_anthropic import ChatAnthropic - -# OpenAI -model = ChatOpenAI(model="gpt-4o", temperature=0) - -# Anthropic (better for complex reasoning) -model = ChatAnthropic(model="claude-sonnet-4-5-20250929", temperature=0) - -# Dynamic model selection -def select_model(task_complexity: str): - if task_complexity == "high": - return ChatAnthropic(model="claude-sonnet-4-5-20250929") - else: - return ChatOpenAI(model="gpt-4o-mini") -``` - -### 2. Tools - Actions the agent can take - -```python -from langchain.tools import tool - -# Simple function tool -@tool -def get_current_time() -> str: - """Get the current time.""" - from datetime import datetime - return datetime.now().strftime("%H:%M:%S") - -# Tool with parameters -@tool -def fetch_weather(city: str, units: str = "fahrenheit") -> str: - """Fetch weather for a city. - - Args: - city: City name - units: Temperature units (fahrenheit or celsius) - """ - # Your weather API call here - return f"Weather in {city}: 72°{units[0].upper()}" - -# Tool with error handling -@tool -def risky_api_call(endpoint: str) -> str: - """Call an external API that might fail.""" - try: - response = requests.get(endpoint, timeout=5) - return response.text - except Exception as e: - return f"Error calling API: {str(e)}" -``` - -### 3. System prompt - Agent behavior - -```python -# General assistant -system_prompt = "You are a helpful assistant. Use tools when needed." - -# Domain expert -system_prompt = """You are a financial analyst assistant. -- Use the calculator for precise calculations -- Search for recent financial data -- Provide data-driven recommendations -- Always cite your sources""" - -# Constrained agent -system_prompt = """You are a customer support agent. -- Only use search_kb tool to find answers -- If answer not found, escalate to human -- Be concise and professional -- Never make up information""" -``` - -## Agent types - -### 1. Tool-calling agent (recommended) - -Uses native function calling for best performance: - -```python -from langchain.agents import create_tool_calling_agent, AgentExecutor -from langchain.prompts import ChatPromptTemplate - -# Create prompt -prompt = ChatPromptTemplate.from_messages([ - ("system", "You are a helpful assistant"), - ("human", "{input}"), - ("placeholder", "{agent_scratchpad}"), -]) - -# Create agent -agent = create_tool_calling_agent( - llm=model, - tools=[calculator, search], - prompt=prompt -) - -# Wrap in executor -agent_executor = AgentExecutor( - agent=agent, - tools=[calculator, search], - verbose=True, - max_iterations=5, - handle_parsing_errors=True -) - -# Run -result = agent_executor.invoke({"input": "What is the weather in Paris?"}) -``` - -### 2. ReAct agent (reasoning trace) - -Shows step-by-step reasoning: - -```python -from langchain.agents import create_react_agent - -# ReAct prompt shows thought process -react_prompt = """Answer the following questions as best you can. You have access to the following tools: - -{tools} - -Use the following format: - -Question: the input question you must answer -Thought: you should always think about what to do -Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... (this Thought/Action/Action Input/Observation can repeat N times) -Thought: I now know the final answer -Final Answer: the final answer to the original input question - -Begin! - -Question: {input} -Thought: {agent_scratchpad}""" - -agent = create_react_agent( - llm=model, - tools=[calculator, search], - prompt=ChatPromptTemplate.from_template(react_prompt) -) - -# Run with visible reasoning -result = agent_executor.invoke({"input": "What is 25 * 17 + 142?"}) -``` - -### 3. Conversational agent (with memory) - -Remembers conversation history: - -```python -from langchain.agents import create_conversational_retrieval_agent -from langchain.memory import ConversationBufferMemory - -# Add memory -memory = ConversationBufferMemory( - memory_key="chat_history", - return_messages=True -) - -# Conversational agent -agent_executor = AgentExecutor( - agent=agent, - tools=[calculator, search], - memory=memory, - verbose=True -) - -# Multi-turn conversation -agent_executor.invoke({"input": "My name is Alice"}) -agent_executor.invoke({"input": "What's my name?"}) # Remembers "Alice" -agent_executor.invoke({"input": "What is 25 * 17?"}) -``` - -## Tool execution patterns - -### Parallel tool execution - -```python -# Agent automatically parallelizes independent calls -agent = create_tool_calling_agent(llm=model, tools=[get_weather, search]) - -# This calls get_weather("Paris") and get_weather("London") in parallel -result = agent_executor.invoke({ - "input": "Compare weather in Paris and London" -}) -``` - -### Sequential tool chaining - -```python -# Agent chains tools automatically -@tool -def search_company(name: str) -> str: - """Search for company information.""" - return f"Company ID: 12345, Industry: Tech" - -@tool -def get_stock_price(company_id: str) -> str: - """Get stock price for a company.""" - return f"${150.00}" - -# Agent will: search_company → get_stock_price -result = agent_executor.invoke({ - "input": "What is Apple's current stock price?" -}) -``` - -### Conditional tool usage - -```python -# Agent decides when to use tools -@tool -def expensive_tool(query: str) -> str: - """Use only when necessary - costs $0.10 per call.""" - return perform_expensive_operation(query) - -# Agent uses tool only if needed -result = agent_executor.invoke({ - "input": "What is 2+2?" # Won't use expensive_tool -}) -``` - -## Streaming - -### Stream agent steps - -```python -# Stream intermediate steps -for step in agent_executor.stream({"input": "Research quantum computing"}): - if "actions" in step: - action = step["actions"][0] - print(f"Tool: {action.tool}, Input: {action.tool_input}") - if "steps" in step: - print(f"Observation: {step['steps'][0].observation}") - if "output" in step: - print(f"Final: {step['output']}") -``` - -### Stream LLM tokens - -```python -from langchain.callbacks import StreamingStdOutCallbackHandler - -# Stream model responses -agent_executor = AgentExecutor( - agent=agent, - tools=[calculator], - callbacks=[StreamingStdOutCallbackHandler()], - verbose=True -) - -result = agent_executor.invoke({"input": "Explain quantum computing"}) -``` - -## Error handling - -### Tool error handling - -```python -@tool -def fallible_tool(query: str) -> str: - """A tool that might fail.""" - try: - result = risky_operation(query) - return f"Success: {result}" - except Exception as e: - return f"Error: {str(e)}. Please try a different approach." - -# Agent adapts to errors -agent_executor = AgentExecutor( - agent=agent, - tools=[fallible_tool], - handle_parsing_errors=True, # Handle malformed tool calls - max_iterations=5 -) -``` - -### Timeout handling - -```python -from langchain.callbacks import TimeoutCallback - -# Set timeout -agent_executor = AgentExecutor( - agent=agent, - tools=[slow_tool], - callbacks=[TimeoutCallback(timeout=30)], # 30 second timeout - max_iterations=10 -) -``` - -### Retry logic - -```python -from langchain.callbacks import RetryCallback - -# Retry on failure -agent_executor = AgentExecutor( - agent=agent, - tools=[unreliable_tool], - callbacks=[RetryCallback(max_retries=3)], - max_execution_time=60 -) -``` - -## Advanced patterns - -### Dynamic tool selection - -```python -# Select tools based on context -def get_tools_for_user(user_role: str): - if user_role == "admin": - return [search, calculator, database_query, delete_data] - elif user_role == "analyst": - return [search, calculator, database_query] - else: - return [search, calculator] - -# Create agent with role-based tools -tools = get_tools_for_user(current_user.role) -agent = create_agent(model=model, tools=tools) -``` - -### Multi-step reasoning - -```python -# Agent plans multiple steps -system_prompt = """Break down complex tasks into steps: -1. Analyze the question -2. Determine required information -3. Use tools to gather data -4. Synthesize findings -5. Provide final answer""" - -agent = create_agent( - model=model, - tools=[search, calculator, database], - system_prompt=system_prompt -) - -result = agent.invoke({ - "input": "Compare revenue growth of top 3 tech companies over 5 years" -}) -``` - -### Structured output from agents - -```python -from langchain_core.pydantic_v1 import BaseModel, Field - -class ResearchReport(BaseModel): - summary: str = Field(description="Executive summary") - findings: list[str] = Field(description="Key findings") - sources: list[str] = Field(description="Source URLs") - -# Agent returns structured output -structured_agent = agent.with_structured_output(ResearchReport) -report = structured_agent.invoke({"input": "Research AI safety"}) -print(report.summary, report.findings) -``` - -## Middleware & customization - -### Custom agent middleware - -```python -from langchain.agents import AgentExecutor - -def logging_middleware(agent_executor): - """Log all agent actions.""" - original_invoke = agent_executor.invoke - - def wrapped_invoke(*args, **kwargs): - print(f"Agent invoked with: {args[0]}") - result = original_invoke(*args, **kwargs) - print(f"Agent result: {result}") - return result - - agent_executor.invoke = wrapped_invoke - return agent_executor - -# Apply middleware -agent_executor = logging_middleware(agent_executor) -``` - -### Custom stopping conditions - -```python -from langchain.agents import EarlyStoppingMethod - -# Stop early if confident -agent_executor = AgentExecutor( - agent=agent, - tools=[search], - early_stopping_method=EarlyStoppingMethod.GENERATE, # or FORCE - max_iterations=10 -) -``` - -## Best practices - -1. **Use tool-calling agents** - Fastest and most reliable -2. **Keep tool descriptions clear** - Agent needs to understand when to use each tool -3. **Add error handling** - Tools will fail, handle gracefully -4. **Set max_iterations** - Prevent infinite loops (default: 15) -5. **Enable streaming** - Better UX for long tasks -6. **Use verbose=True during dev** - See agent reasoning -7. **Test tool combinations** - Ensure tools work together -8. **Monitor with LangSmith** - Essential for production -9. **Cache tool results** - Avoid redundant API calls -10. **Version system prompts** - Track changes in behavior - -## Common pitfalls - -1. **Vague tool descriptions** - Agent won't know when to use tool -2. **Too many tools** - Agent gets confused (limit to 5-10) -3. **Tools without error handling** - One failure crashes agent -4. **Circular tool dependencies** - Agent gets stuck in loops -5. **Missing max_iterations** - Agent runs forever -6. **Poor system prompts** - Agent doesn't follow instructions - -## Debugging agents - -```python -# Enable verbose logging -agent_executor = AgentExecutor( - agent=agent, - tools=[calculator], - verbose=True, # See all steps - return_intermediate_steps=True # Get full trace -) - -result = agent_executor.invoke({"input": "Calculate 25 * 17"}) - -# Inspect intermediate steps -for step in result["intermediate_steps"]: - print(f"Action: {step[0].tool}") - print(f"Input: {step[0].tool_input}") - print(f"Output: {step[1]}") -``` - -## Resources - -- **ReAct Paper**: https://arxiv.org/abs/2210.03629 -- **LangChain Agents Docs**: https://docs.langchain.com/oss/python/langchain/agents -- **LangSmith Debugging**: https://smith.langchain.com diff --git a/.claude/skills/langchain/references/integration.md b/.claude/skills/langchain/references/integration.md deleted file mode 100644 index c06e1226..00000000 --- a/.claude/skills/langchain/references/integration.md +++ /dev/null @@ -1,562 +0,0 @@ -# LangChain Integration Guide - -Integration with vector stores, LangSmith observability, and deployment. - -## Vector store integrations - -### Chroma (local, open-source) - -```python -from langchain_chroma import Chroma -from langchain_openai import OpenAIEmbeddings - -# Create vector store -vectorstore = Chroma.from_documents( - documents=docs, - embedding=OpenAIEmbeddings(), - persist_directory="./chroma_db" -) - -# Load existing store -vectorstore = Chroma( - persist_directory="./chroma_db", - embedding_function=OpenAIEmbeddings() -) - -# Add documents incrementally -vectorstore.add_documents([new_doc1, new_doc2]) - -# Delete documents -vectorstore.delete(ids=["doc1", "doc2"]) -``` - -### Pinecone (cloud, scalable) - -```python -from langchain_pinecone import PineconeVectorStore -import pinecone - -# Initialize Pinecone -pinecone.init(api_key="your-api-key", environment="us-west1-gcp") - -# Create index (one-time) -pinecone.create_index("my-index", dimension=1536, metric="cosine") - -# Create vector store -vectorstore = PineconeVectorStore.from_documents( - documents=docs, - embedding=OpenAIEmbeddings(), - index_name="my-index" -) - -# Query with metadata filters -results = vectorstore.similarity_search( - "Python tutorials", - k=4, - filter={"category": "beginner"} -) -``` - -### FAISS (fast similarity search) - -```python -from langchain_community.vectorstores import FAISS - -# Create FAISS index -vectorstore = FAISS.from_documents(docs, OpenAIEmbeddings()) - -# Save to disk -vectorstore.save_local("./faiss_index") - -# Load from disk -vectorstore = FAISS.load_local( - "./faiss_index", - OpenAIEmbeddings(), - allow_dangerous_deserialization=True -) - -# Merge multiple indices -vectorstore1 = FAISS.load_local("./index1", embeddings) -vectorstore2 = FAISS.load_local("./index2", embeddings) -vectorstore1.merge_from(vectorstore2) -``` - -### Weaviate (production, ML-native) - -```python -from langchain_weaviate import WeaviateVectorStore -import weaviate - -# Connect to Weaviate -client = weaviate.Client("http://localhost:8080") - -# Create vector store -vectorstore = WeaviateVectorStore.from_documents( - documents=docs, - embedding=OpenAIEmbeddings(), - client=client, - index_name="LangChain" -) - -# Hybrid search (vector + keyword) -results = vectorstore.similarity_search( - "Python async", - k=4, - alpha=0.5 # 0=keyword, 1=vector, 0.5=hybrid -) -``` - -### Qdrant (fast, open-source) - -```python -from langchain_qdrant import QdrantVectorStore -from qdrant_client import QdrantClient - -# Connect to Qdrant -client = QdrantClient(host="localhost", port=6333) - -# Create vector store -vectorstore = QdrantVectorStore.from_documents( - documents=docs, - embedding=OpenAIEmbeddings(), - collection_name="my_documents", - client=client -) -``` - -## LangSmith observability - -### Enable tracing - -```python -import os - -# Set environment variables -os.environ["LANGCHAIN_TRACING_V2"] = "true" -os.environ["LANGCHAIN_API_KEY"] = "your-langsmith-api-key" -os.environ["LANGCHAIN_PROJECT"] = "my-project" - -# All chains/agents automatically traced -from langchain.agents import create_agent -from langchain_anthropic import ChatAnthropic - -agent = create_agent( - model=ChatAnthropic(model="claude-sonnet-4-5-20250929"), - tools=[calculator, search] -) - -# Run - automatically logged to LangSmith -result = agent.invoke({"input": "What is 25 * 17?"}) - -# View traces at https://smith.langchain.com -``` - -### Custom metadata - -```python -from langchain.callbacks import tracing_v2_enabled - -# Add custom metadata to traces -with tracing_v2_enabled( - project_name="my-project", - tags=["production", "customer-support"], - metadata={"user_id": "12345", "session_id": "abc"} -): - result = agent.invoke({"input": "Help me with Python"}) -``` - -### Evaluate runs - -```python -from langsmith import Client - -client = Client() - -# Create dataset -dataset = client.create_dataset("qa-eval") -client.create_example( - dataset_id=dataset.id, - inputs={"question": "What is Python?"}, - outputs={"answer": "Python is a programming language"} -) - -# Evaluate -from langchain.evaluation import load_evaluator - -evaluator = load_evaluator("qa") -results = client.evaluate( - lambda x: qa_chain(x), - data=dataset, - evaluators=[evaluator] -) -``` - -## Deployment patterns - -### FastAPI server - -```python -from fastapi import FastAPI -from pydantic import BaseModel -from langchain.agents import create_agent - -app = FastAPI() - -# Initialize agent once -agent = create_agent( - model=llm, - tools=[search, calculator] -) - -class Query(BaseModel): - input: str - -@app.post("/chat") -async def chat(query: Query): - result = agent.invoke({"input": query.input}) - return {"response": result["output"]} - -# Run: uvicorn main:app --reload -``` - -### Streaming responses - -```python -from fastapi.responses import StreamingResponse -from langchain.callbacks import AsyncIteratorCallbackHandler - -@app.post("/chat/stream") -async def chat_stream(query: Query): - callback = AsyncIteratorCallbackHandler() - - async def generate(): - async for token in agent.astream({"input": query.input}): - if "output" in token: - yield token["output"] - - return StreamingResponse(generate(), media_type="text/plain") -``` - -### Docker deployment - -```dockerfile -# Dockerfile -FROM python:3.11-slim - -WORKDIR /app - -COPY requirements.txt . -RUN pip install -r requirements.txt - -COPY . . - -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] -``` - -```bash -# Build and run -docker build -t langchain-app . -docker run -p 8000:8000 \ - -e OPENAI_API_KEY=your-key \ - -e LANGCHAIN_API_KEY=your-key \ - langchain-app -``` - -### Kubernetes deployment - -```yaml -# deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: langchain-app -spec: - replicas: 3 - selector: - matchLabels: - app: langchain - template: - metadata: - labels: - app: langchain - spec: - containers: - - name: langchain - image: your-registry/langchain-app:latest - ports: - - containerPort: 8000 - env: - - name: OPENAI_API_KEY - valueFrom: - secretKeyRef: - name: langchain-secrets - key: openai-api-key - resources: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "2000m" -``` - -## Model integrations - -### OpenAI - -```python -from langchain_openai import ChatOpenAI - -llm = ChatOpenAI( - model="gpt-4o", - temperature=0, - max_tokens=1000, - timeout=30, - max_retries=2 -) -``` - -### Anthropic - -```python -from langchain_anthropic import ChatAnthropic - -llm = ChatAnthropic( - model="claude-sonnet-4-5-20250929", - temperature=0, - max_tokens=4096, - timeout=60 -) -``` - -### Google - -```python -from langchain_google_genai import ChatGoogleGenerativeAI - -llm = ChatGoogleGenerativeAI( - model="gemini-2.0-flash-exp", - temperature=0 -) -``` - -### Local models (Ollama) - -```python -from langchain_community.llms import Ollama - -llm = Ollama( - model="llama3", - base_url="http://localhost:11434" -) -``` - -### Azure OpenAI - -```python -from langchain_openai import AzureChatOpenAI - -llm = AzureChatOpenAI( - azure_endpoint="https://your-endpoint.openai.azure.com/", - azure_deployment="gpt-4", - api_version="2024-02-15-preview" -) -``` - -## Tool integrations - -### Web search - -```python -from langchain_community.tools import DuckDuckGoSearchRun, TavilySearchResults - -# DuckDuckGo (free) -search = DuckDuckGoSearchRun() - -# Tavily (best quality) -search = TavilySearchResults(api_key="your-key") -``` - -### Wikipedia - -```python -from langchain_community.tools import WikipediaQueryRun -from langchain_community.utilities import WikipediaAPIWrapper - -wikipedia = WikipediaQueryRun(api_wrapper=WikipediaAPIWrapper()) -``` - -### Python REPL - -```python -from langchain_experimental.tools import PythonREPLTool - -python_repl = PythonREPLTool() - -# Agent can execute Python code -agent = create_agent(model=llm, tools=[python_repl]) -result = agent.invoke({"input": "Calculate the 10th Fibonacci number"}) -``` - -### Shell commands - -```python -from langchain_community.tools import ShellTool - -shell = ShellTool() - -# Agent can run shell commands -agent = create_agent(model=llm, tools=[shell]) -``` - -### SQL databases - -```python -from langchain_community.utilities import SQLDatabase -from langchain_community.agent_toolkits import create_sql_agent - -db = SQLDatabase.from_uri("sqlite:///mydatabase.db") - -agent = create_sql_agent( - llm=llm, - db=db, - agent_type="openai-tools", - verbose=True -) - -result = agent.run("How many users are in the database?") -``` - -## Memory integrations - -### Redis - -```python -from langchain.memory import RedisChatMessageHistory -from langchain.memory import ConversationBufferMemory - -# Redis-backed memory -message_history = RedisChatMessageHistory( - url="redis://localhost:6379", - session_id="user-123" -) - -memory = ConversationBufferMemory( - chat_memory=message_history, - return_messages=True -) -``` - -### PostgreSQL - -```python -from langchain_postgres import PostgresChatMessageHistory - -message_history = PostgresChatMessageHistory( - connection_string="postgresql://user:pass@localhost/db", - session_id="user-123" -) -``` - -### MongoDB - -```python -from langchain_mongodb import MongoDBChatMessageHistory - -message_history = MongoDBChatMessageHistory( - connection_string="mongodb://localhost:27017/", - session_id="user-123" -) -``` - -## Caching - -### In-memory cache - -```python -from langchain.cache import InMemoryCache -from langchain.globals import set_llm_cache - -set_llm_cache(InMemoryCache()) - -# Same query uses cache -response1 = llm.invoke("What is Python?") # API call -response2 = llm.invoke("What is Python?") # Cached -``` - -### SQLite cache - -```python -from langchain.cache import SQLiteCache - -set_llm_cache(SQLiteCache(database_path=".langchain.db")) -``` - -### Redis cache - -```python -from langchain.cache import RedisCache -from redis import Redis - -set_llm_cache(RedisCache(redis_=Redis(host="localhost", port=6379))) -``` - -## Monitoring & logging - -### Custom callbacks - -```python -from langchain.callbacks.base import BaseCallbackHandler - -class CustomCallback(BaseCallbackHandler): - def on_llm_start(self, serialized, prompts, **kwargs): - print(f"LLM started with prompts: {prompts}") - - def on_llm_end(self, response, **kwargs): - print(f"LLM finished with: {response}") - - def on_tool_start(self, serialized, input_str, **kwargs): - print(f"Tool {serialized['name']} started with: {input_str}") - - def on_tool_end(self, output, **kwargs): - print(f"Tool finished with: {output}") - -# Use callback -agent = create_agent( - model=llm, - tools=[calculator], - callbacks=[CustomCallback()] -) -``` - -### Token counting - -```python -from langchain.callbacks import get_openai_callback - -with get_openai_callback() as cb: - result = llm.invoke("Write a long story") - print(f"Tokens used: {cb.total_tokens}") - print(f"Cost: ${cb.total_cost:.4f}") -``` - -## Best practices - -1. **Use LangSmith in production** - Essential for debugging -2. **Cache aggressively** - LLM calls are expensive -3. **Set timeouts** - Prevent hanging requests -4. **Add retries** - Handle transient failures -5. **Monitor costs** - Track token usage -6. **Version your prompts** - Track changes -7. **Use async** - Better performance for I/O -8. **Persistent memory** - Don't lose conversation history -9. **Secure API keys** - Use environment variables -10. **Test integrations** - Verify connections before production - -## Resources - -- **LangSmith**: https://smith.langchain.com -- **Vector Stores**: https://python.langchain.com/docs/integrations/vectorstores -- **Model Providers**: https://python.langchain.com/docs/integrations/llms -- **Tools**: https://python.langchain.com/docs/integrations/tools -- **Deployment Guide**: https://docs.langchain.com/deploy diff --git a/.claude/skills/langchain/references/rag.md b/.claude/skills/langchain/references/rag.md deleted file mode 100644 index 294e7895..00000000 --- a/.claude/skills/langchain/references/rag.md +++ /dev/null @@ -1,600 +0,0 @@ -# LangChain RAG Guide - -Complete guide to Retrieval-Augmented Generation with LangChain. - -## What is RAG? - -**RAG (Retrieval-Augmented Generation)** combines: -1. **Retrieval**: Find relevant documents from knowledge base -2. **Generation**: LLM generates answer using retrieved context - -**Benefits**: -- Reduce hallucinations -- Up-to-date information -- Domain-specific knowledge -- Source citations - -## RAG pipeline components - -### 1. Document loading - -```python -from langchain_community.document_loaders import ( - WebBaseLoader, - PyPDFLoader, - TextLoader, - DirectoryLoader, - CSVLoader, - UnstructuredMarkdownLoader -) - -# Web pages -loader = WebBaseLoader("https://docs.python.org/3/tutorial/") -docs = loader.load() - -# PDF files -loader = PyPDFLoader("paper.pdf") -docs = loader.load() - -# Multiple PDFs -loader = DirectoryLoader("./papers/", glob="**/*.pdf", loader_cls=PyPDFLoader) -docs = loader.load() - -# Text files -loader = TextLoader("data.txt") -docs = loader.load() - -# CSV -loader = CSVLoader("data.csv") -docs = loader.load() - -# Markdown -loader = UnstructuredMarkdownLoader("README.md") -docs = loader.load() -``` - -### 2. Text splitting - -```python -from langchain.text_splitter import ( - RecursiveCharacterTextSplitter, - CharacterTextSplitter, - TokenTextSplitter -) - -# Recommended: Recursive (tries multiple separators) -text_splitter = RecursiveCharacterTextSplitter( - chunk_size=1000, # Characters per chunk - chunk_overlap=200, # Overlap between chunks - length_function=len, - separators=["\n\n", "\n", " ", ""] -) - -splits = text_splitter.split_documents(docs) - -# Token-based (for precise token limits) -text_splitter = TokenTextSplitter( - chunk_size=512, # Tokens per chunk - chunk_overlap=50 -) - -# Character-based (simple) -text_splitter = CharacterTextSplitter( - chunk_size=1000, - chunk_overlap=200, - separator="\n\n" -) -``` - -**Chunk size recommendations**: -- **Short answers**: 256-512 tokens -- **General Q&A**: 512-1024 tokens (recommended) -- **Long context**: 1024-2048 tokens -- **Overlap**: 10-20% of chunk_size - -### 3. Embeddings - -```python -from langchain_openai import OpenAIEmbeddings -from langchain_community.embeddings import ( - HuggingFaceEmbeddings, - CohereEmbeddings -) - -# OpenAI (fast, high quality) -embeddings = OpenAIEmbeddings(model="text-embedding-3-small") - -# HuggingFace (free, local) -embeddings = HuggingFaceEmbeddings( - model_name="sentence-transformers/all-mpnet-base-v2" -) - -# Cohere -embeddings = CohereEmbeddings(model="embed-english-v3.0") -``` - -### 4. Vector stores - -```python -from langchain_chroma import Chroma -from langchain_community.vectorstores import FAISS -from langchain_pinecone import PineconeVectorStore - -# Chroma (local, persistent) -vectorstore = Chroma.from_documents( - documents=splits, - embedding=embeddings, - persist_directory="./chroma_db" -) - -# FAISS (fast similarity search) -vectorstore = FAISS.from_documents(splits, embeddings) -vectorstore.save_local("./faiss_index") - -# Pinecone (cloud, scalable) -vectorstore = PineconeVectorStore.from_documents( - documents=splits, - embedding=embeddings, - index_name="my-index" -) -``` - -### 5. Retrieval - -```python -# Basic retriever (top-k similarity) -retriever = vectorstore.as_retriever( - search_type="similarity", - search_kwargs={"k": 4} # Return top 4 documents -) - -# MMR (Maximal Marginal Relevance) - diverse results -retriever = vectorstore.as_retriever( - search_type="mmr", - search_kwargs={ - "k": 4, - "fetch_k": 20, # Fetch 20, return diverse 4 - "lambda_mult": 0.5 # Diversity (0=diverse, 1=similar) - } -) - -# Similarity score threshold -retriever = vectorstore.as_retriever( - search_type="similarity_score_threshold", - search_kwargs={ - "score_threshold": 0.5 # Minimum similarity score - } -) - -# Query documents directly -docs = retriever.get_relevant_documents("What is Python?") -``` - -### 6. QA chain - -```python -from langchain.chains import RetrievalQA -from langchain_anthropic import ChatAnthropic - -llm = ChatAnthropic(model="claude-sonnet-4-5-20250929") - -# Basic QA chain -qa_chain = RetrievalQA.from_chain_type( - llm=llm, - retriever=retriever, - return_source_documents=True -) - -# Query -result = qa_chain({"query": "What are Python decorators?"}) -print(result["result"]) -print(f"Sources: {len(result['source_documents'])}") -``` - -## Advanced RAG patterns - -### Conversational RAG - -```python -from langchain.chains import ConversationalRetrievalChain -from langchain.memory import ConversationBufferMemory - -# Add memory -memory = ConversationBufferMemory( - memory_key="chat_history", - return_messages=True, - output_key="answer" -) - -# Conversational RAG chain -qa = ConversationalRetrievalChain.from_llm( - llm=llm, - retriever=retriever, - memory=memory, - return_source_documents=True -) - -# Multi-turn conversation -result1 = qa({"question": "What is Python used for?"}) -result2 = qa({"question": "Can you give examples?"}) # Remembers context -result3 = qa({"question": "What about web development?"}) -``` - -### Custom prompt template - -```python -from langchain.prompts import PromptTemplate - -# Custom QA prompt -template = """Use the following pieces of context to answer the question. -If you don't know the answer, say so - don't make it up. -Always cite your sources using [Source N] notation. - -Context: {context} - -Question: {question} - -Helpful Answer:""" - -prompt = PromptTemplate( - template=template, - input_variables=["context", "question"] -) - -qa_chain = RetrievalQA.from_chain_type( - llm=llm, - retriever=retriever, - chain_type_kwargs={"prompt": prompt} -) -``` - -### Chain types - -```python -# 1. Stuff (default) - Put all docs in context -qa_chain = RetrievalQA.from_chain_type( - llm=llm, - retriever=retriever, - chain_type="stuff" # Fast, works if docs fit in context -) - -# 2. Map-reduce - Summarize each doc, then combine -qa_chain = RetrievalQA.from_chain_type( - llm=llm, - retriever=retriever, - chain_type="map_reduce" # For many documents -) - -# 3. Refine - Iteratively refine answer -qa_chain = RetrievalQA.from_chain_type( - llm=llm, - retriever=retriever, - chain_type="refine" # Most thorough, slowest -) - -# 4. Map-rerank - Score answers, return best -qa_chain = RetrievalQA.from_chain_type( - llm=llm, - retriever=retriever, - chain_type="map_rerank" # Good for multiple perspectives -) -``` - -### Multi-query retrieval - -```python -from langchain.retrievers import MultiQueryRetriever - -# Generate multiple queries for better recall -retriever = MultiQueryRetriever.from_llm( - retriever=vectorstore.as_retriever(), - llm=llm -) - -# "What is Python?" becomes: -# - "What is Python programming language?" -# - "Python language definition" -# - "Overview of Python" -docs = retriever.get_relevant_documents("What is Python?") -``` - -### Contextual compression - -```python -from langchain.retrievers import ContextualCompressionRetriever -from langchain.retrievers.document_compressors import LLMChainExtractor - -# Compress retrieved docs to relevant parts only -compressor = LLMChainExtractor.from_llm(llm) - -compression_retriever = ContextualCompressionRetriever( - base_compressor=compressor, - base_retriever=vectorstore.as_retriever() -) - -# Returns only relevant excerpts -compressed_docs = compression_retriever.get_relevant_documents("Python decorators") -``` - -### Ensemble retrieval (hybrid search) - -```python -from langchain.retrievers import EnsembleRetriever -from langchain.retrievers import BM25Retriever - -# Vector search (semantic) -vector_retriever = vectorstore.as_retriever(search_kwargs={"k": 5}) - -# Keyword search (BM25) -keyword_retriever = BM25Retriever.from_documents(splits) -keyword_retriever.k = 5 - -# Combine both -ensemble_retriever = EnsembleRetriever( - retrievers=[vector_retriever, keyword_retriever], - weights=[0.5, 0.5] # Equal weight -) - -docs = ensemble_retriever.get_relevant_documents("Python async") -``` - -## RAG with agents - -### Agent-based RAG - -```python -from langchain.agents import create_tool_calling_agent -from langchain.tools.retriever import create_retriever_tool - -# Create retriever tool -retriever_tool = create_retriever_tool( - retriever=retriever, - name="python_docs", - description="Searches Python documentation for answers about Python programming" -) - -# Create agent with retriever tool -agent = create_tool_calling_agent( - llm=llm, - tools=[retriever_tool, calculator, search], - system_prompt="Use python_docs tool for Python questions" -) - -# Agent decides when to retrieve -from langchain.agents import AgentExecutor -agent_executor = AgentExecutor(agent=agent, tools=[retriever_tool]) - -result = agent_executor.invoke({"input": "What are Python generators?"}) -``` - -### Multi-document agents - -```python -# Multiple knowledge bases -python_retriever = create_retriever_tool( - retriever=python_vectorstore.as_retriever(), - name="python_docs", - description="Python programming documentation" -) - -numpy_retriever = create_retriever_tool( - retriever=numpy_vectorstore.as_retriever(), - name="numpy_docs", - description="NumPy library documentation" -) - -# Agent chooses which knowledge base to query -agent = create_agent( - model=llm, - tools=[python_retriever, numpy_retriever, search] -) - -result = agent.invoke({"input": "How do I create numpy arrays?"}) -``` - -## Metadata filtering - -### Add metadata to documents - -```python -from langchain.schema import Document - -# Documents with metadata -docs = [ - Document( - page_content="Python is a programming language", - metadata={"source": "tutorial.pdf", "page": 1, "category": "intro"} - ), - Document( - page_content="Python decorators modify functions", - metadata={"source": "advanced.pdf", "page": 42, "category": "advanced"} - ) -] - -vectorstore = Chroma.from_documents(docs, embeddings) -``` - -### Filter by metadata - -```python -# Retrieve only from specific source -retriever = vectorstore.as_retriever( - search_kwargs={ - "k": 4, - "filter": {"category": "intro"} # Only intro documents - } -) - -# Multiple filters -retriever = vectorstore.as_retriever( - search_kwargs={ - "k": 4, - "filter": { - "category": "advanced", - "source": "advanced.pdf" - } - } -) -``` - -## Document preprocessing - -### Clean documents - -```python -def preprocess_doc(doc): - """Clean and normalize document.""" - # Remove extra whitespace - doc.page_content = " ".join(doc.page_content.split()) - - # Remove special characters - doc.page_content = re.sub(r'[^\w\s]', '', doc.page_content) - - # Lowercase (optional) - doc.page_content = doc.page_content.lower() - - return doc - -# Apply preprocessing -clean_docs = [preprocess_doc(doc) for doc in docs] -``` - -### Extract structured data - -```python -from langchain.document_transformers import Html2TextTransformer - -# HTML to clean text -transformer = Html2TextTransformer() -clean_docs = transformer.transform_documents(html_docs) - -# Extract tables -from langchain.document_loaders import UnstructuredHTMLLoader - -loader = UnstructuredHTMLLoader("data.html") -docs = loader.load() # Extracts tables as structured data -``` - -## Evaluation & monitoring - -### Evaluate retrieval quality - -```python -from langchain.evaluation import load_evaluator - -# Relevance evaluator -evaluator = load_evaluator("relevance", llm=llm) - -# Test retrieval -query = "What are Python decorators?" -retrieved_docs = retriever.get_relevant_documents(query) - -for doc in retrieved_docs: - result = evaluator.evaluate_strings( - input=query, - prediction=doc.page_content - ) - print(f"Relevance score: {result['score']}") -``` - -### Track sources - -```python -# Always return sources -qa_chain = RetrievalQA.from_chain_type( - llm=llm, - retriever=retriever, - return_source_documents=True -) - -result = qa_chain({"query": "What is Python?"}) - -# Show sources to user -print(result["result"]) -print("\nSources:") -for i, doc in enumerate(result["source_documents"]): - print(f"[{i+1}] {doc.metadata.get('source', 'Unknown')}") - print(f" {doc.page_content[:100]}...") -``` - -## Best practices - -1. **Chunk size matters** - 512-1024 tokens is usually optimal -2. **Add overlap** - 10-20% overlap prevents context loss -3. **Use metadata** - Track sources for citations -4. **Test retrieval quality** - Evaluate before using in production -5. **Hybrid search** - Combine vector + keyword for best results -6. **Compress context** - Remove irrelevant parts before LLM -7. **Cache embeddings** - Expensive, cache when possible -8. **Version your index** - Track changes to knowledge base -9. **Monitor failures** - Log when retrieval doesn't find answers -10. **Update regularly** - Keep knowledge base current - -## Common pitfalls - -1. **Chunks too large** - Won't fit in context -2. **No overlap** - Important context lost at boundaries -3. **No metadata** - Can't cite sources -4. **Poor splitting** - Breaks mid-sentence or mid-paragraph -5. **Wrong embedding model** - Domain mismatch hurts retrieval -6. **No reranking** - Lower quality results -7. **Ignoring failures** - No handling when retrieval fails - -## Performance optimization - -### Caching - -```python -from langchain.cache import InMemoryCache, SQLiteCache -from langchain.globals import set_llm_cache - -# In-memory cache -set_llm_cache(InMemoryCache()) - -# Persistent cache -set_llm_cache(SQLiteCache(database_path=".langchain.db")) - -# Same query uses cache (faster + cheaper) -result1 = qa_chain({"query": "What is Python?"}) -result2 = qa_chain({"query": "What is Python?"}) # Cached -``` - -### Batch processing - -```python -# Process multiple queries efficiently -queries = [ - "What is Python?", - "What are decorators?", - "How do I use async?" -] - -# Batch retrieval -all_docs = vectorstore.similarity_search_batch(queries) - -# Batch QA -results = qa_chain.batch([{"query": q} for q in queries]) -``` - -### Async operations - -```python -# Async RAG for concurrent queries -import asyncio - -async def async_qa(query): - return await qa_chain.ainvoke({"query": query}) - -# Run multiple queries concurrently -results = await asyncio.gather( - async_qa("What is Python?"), - async_qa("What are decorators?") -) -``` - -## Resources - -- **LangChain RAG Docs**: https://docs.langchain.com/oss/python/langchain/rag -- **Vector Stores**: https://python.langchain.com/docs/integrations/vectorstores -- **Document Loaders**: https://python.langchain.com/docs/integrations/document_loaders -- **Retrievers**: https://python.langchain.com/docs/modules/data_connection/retrievers diff --git a/CLAUDE.md b/CLAUDE.md index f0cf77e0..ed5a07d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,13 +2,13 @@ ## Project Overview -gnubok is a Swedish-focused accounting SaaS for sole traders (enskild firma) and limited companies (aktiebolag). It implements double-entry bookkeeping compliant with Swedish accounting law (Bokforingslagen), including VAT handling, tax reporting, and 7-year document retention. +gnubok is a Swedish-focused accounting SaaS for sole traders (enskild firma) and limited companies (aktiebolag). It implements double-entry bookkeeping compliant with Swedish accounting law (Bokforingslagen), including VAT handling, tax reporting, and 7-year document retention. Multi-tenant: each user can own or be a member of multiple companies, optionally grouped into teams (for consultants). -**Tech stack**: Next.js 16 (App Router), React 19, TypeScript (strict), Supabase (PostgreSQL + RLS + email/password + TOTP MFA auth), Tailwind CSS 4 + shadcn/ui, Vercel hosting. +**Tech stack**: Next.js 16.1.5 (App Router), React 19.2.3, TypeScript 5 (strict), Zod 4, Supabase (PostgreSQL + RLS + email/password + TOTP MFA auth), Tailwind CSS 4 + shadcn/ui, Vercel hosting, Docker (self-hosted). -**Integrations**: Enable Banking (PSD2), Anthropic SDK, LangChain, OpenAI (embeddings), Resend (email), JSZip (archive export). +**Integrations**: Enable Banking (PSD2), TIC Identity (company lookup), Anthropic SDK, OpenAI (embeddings), Resend (email), Sentry (error tracking), Svix (webhooks), web-push (notifications), JSZip (archive export), sharp (image processing), Framer Motion (animations). -**Path alias**: `@/*` maps to the project root. **Language**: All code, comments, and commit messages in English. +**Path alias**: `@/*` maps to the project root. **Language**: All code, comments, and commit messages in English. **License**: AGPL-3.0-or-later. --- @@ -27,14 +27,42 @@ npm run setup:extensions # Regenerate extension registry from extensions.config. ## Key Architectural Relationships -- **All journal entry creation** routes through `lib/bookkeeping/engine.ts` via `createJournalEntry()`. -- **API routes** that emit events must call `ensureInitialized()` (from `lib/init.ts`) at module level. -- **Event bus** (`lib/events/bus.ts`) is a module-level singleton. Handlers run via `Promise.allSettled`. -- **Supabase clients**: browser (`lib/supabase/client.ts`), server with cookies (`createClient()` from `server.ts`), service role (`createServiceClient()`), cookieless service role for API key auth (`createServiceClientNoCookies()` from `lib/auth/api-keys.ts`). -- **Extension system**: Opt-in via `extensions.config.json`. Core builds and runs with zero extensions. -- **NE-bilaga, INK2 declaration, SRU export, and full archive export** are core reports (in `lib/reports/`), not extensions. -- **AI consent gate** (`lib/extensions/ai-consent.ts`): AI extensions (`receipt-ocr`, `ai-categorization`, `ai-chat`) require user consent before API calls. Returns `403 AI_CONSENT_REQUIRED` if missing. -- **Types**: All shared types in `types/index.ts` (single source of truth). Import via `import type { T } from '@/types'`. Event types live in `lib/events/types.ts`. +- **Multi-tenant model**: `companies` table owns all business data. `company_members` links users to companies with roles (owner/admin/member/viewer). `teams` group companies for consultants. Company context resolved via cookie (`gnubok-company-id`) in middleware (`lib/supabase/middleware.ts`). +- **All journal entry creation** routes through `lib/bookkeeping/engine.ts`. Lifecycle: `createDraftEntry()` → `commitEntry()` (atomic voucher assignment via `commit_journal_entry` DB RPC). Convenience: `createJournalEntry()` does both. Reversal via `reverseEntry()`. Correction via `correctEntry()` in `lib/core/bookkeeping/storno-service.ts`. +- **API routes** that emit events must call `ensureInitialized()` (from `lib/init.ts`) at module level. This loads extensions, wires event handlers, and registers the supplier invoice handler + event log handler. +- **Event bus** (`lib/events/bus.ts`) is a module-level singleton. Handlers run via `Promise.allSettled` — failing handlers never crash the emitter. 30+ event types defined in `lib/events/types.ts`. The event log handler persists actionable events to `event_log` table for external automation. +- **Supabase clients**: browser (`lib/supabase/client.ts`), server with cookies (`createClient()` from `server.ts`), service role (`createServiceClient()`), cookieless service role for API key auth (`createServiceClientNoCookies()` from `lib/auth/api-keys.ts`). Pagination helper: `fetchAllRows()` in `lib/supabase/fetch-all.ts`. +- **Extension system**: Opt-in via `extensions.config.json`. Core builds and runs with zero extensions. Currently enabled: `enable-banking`, `email`, `arcim-migration`, `tic`, `mcp-server`. +- **Core reports** (in `lib/reports/`, not extensions): balance sheet, income statement, trial balance, general ledger, AR/supplier ledger, AR/supplier reconciliation, bank reconciliation status, VAT declaration, journal register, monthly breakdown, continuity check, opening balances, KPI, NE-bilaga, INK2 declaration, SIE export, full archive export. +- **Types**: All shared types in `types/index.ts` (~2,200 lines, single source of truth). Import via `import type { T } from '@/types'`. Event types live in `lib/events/types.ts`. Extension types in `lib/extensions/types.ts`. +- **Error messages**: `lib/errors/get-error-message.ts` maps technical errors to Swedish user messages (Zod → Postgres → HTTP → context fallback). + +--- + +## Multi-Tenant Architecture + +### Data Model + +- **companies**: Business unit (name, org_number, entity_type, created_by, team_id). All business data (journal entries, invoices, transactions, etc.) has a `company_id` column. +- **company_members**: Links users to companies (company_id, user_id, role, source='direct'|'team'). Roles: `owner`, `admin`, `member`, `viewer`. +- **teams**: Consultant grouping (name, created_by). A company can belong to one team. Team members auto-sync to company_members via DB triggers. +- **team_members**: Links users to teams (team_id, user_id, role='owner'|'admin'|'member'). +- **user_preferences**: Stores `active_company_id` per user. + +### Company Context Resolution + +Middleware (`lib/supabase/middleware.ts`) resolves the active company on every request: +1. Check `gnubok-company-id` cookie +2. Fall back to `user_preferences.active_company_id` +3. Fall back to first company membership + +RLS policies use `user_company_ids()` DB helper function to filter by companies the user has access to. + +### Invitations + +- **company_invitations**: Email-based invites with `gnubok_inv_` prefixed tokens (SHA-256 hashed, 7-day TTL). +- **team_invitations**: Same pattern for team invites. +- Token generation: `lib/auth/invite-tokens.ts`. --- @@ -47,13 +75,32 @@ MFA is enforced **application-side** (middleware + API routes), **not** in RLS p - `NEXT_PUBLIC_SELF_HOSTED=true` → MFA never enforced (users can enable voluntarily) - `NEXT_PUBLIC_REQUIRE_MFA=true` (hosted/Vercel) → middleware redirects to `/mfa/enroll` or `/mfa/verify` until AAL2 +**API route auth** (`lib/auth/require-auth.ts`): `requireAuth()` returns `{ user, supabase, error }` discriminated union, enforces MFA on hosted. + +**API keys** (`lib/auth/api-keys.ts`): SHA-256 hashed with `gnubok_sk_` prefix. Scoped permissions (`TOOL_SCOPE_MAP`). Rate limited at 100 RPM via atomic DB RPC (`validate_and_increment_api_key`). + +**Cron auth** (`lib/auth/cron.ts`): `verifyCronSecret()` with constant-time comparison. + --- ## Core Bookkeeping Engine The engine (`lib/bookkeeping/engine.ts`) is the most critical system. All accounting flows route through it. -**Lifecycle**: `createDraftEntry()` → `commitEntry()` (assigns voucher number via DB RPC). Convenience: `createJournalEntry()` does both in one call. Reversal via `reverseEntry()` (storno). Correction via `correctEntry()` in `lib/core/bookkeeping/storno-service.ts`. +**Lifecycle**: `createDraftEntry()` → `commitEntry()` (atomic voucher assignment via `commit_journal_entry` DB RPC). Convenience: `createJournalEntry()` does both in one call. Reversal via `reverseEntry()` (storno). Correction via `correctEntry()` in `lib/core/bookkeeping/storno-service.ts`. + +**Key engine files**: +- `transaction-entries.ts` — Journal entries from bank transactions +- `invoice-entries.ts` — Journal entries from customer invoices (`generatePerRateLines()` for mixed-rate) +- `supplier-invoice-entries.ts` — Journal entries from supplier invoices +- `vat-entries.ts` — VAT-related entries +- `currency-revaluation.ts` — Multi-currency revaluation +- `mapping-engine.ts` — Account mapping rules evaluation +- `booking-templates.ts` / `counterparty-templates.ts` — Reusable templates +- `propose-payment-lines.ts` / `propose-send-lines.ts` — AI-powered matching proposals +- `handlers/supplier-invoice-handler.ts` — Event handler creating registration entries on confirmation + +**BAS data** (`bookkeeping/bas-data/`): Full BAS 2026 chart organized by class (1–8) + SRU mapping. ### Key BAS Accounts @@ -63,7 +110,7 @@ The engine (`lib/bookkeeping/engine.ts`) is the most critical system. All accoun `standard_25`, `reduced_12`, `reduced_6`, `reverse_charge`, `export`, `exempt` -Invoice items support individual `vat_rate` values (mixed-rate invoices). `generatePerRateLines()` in `lib/bookkeeping/invoice-entries.ts` groups by rate. Use `getAvailableVatRates(customerType, vatNumberValidated)` from `lib/invoices/vat-rules.ts`. +Invoice items support individual `vat_rate` values (mixed-rate invoices). Use `getAvailableVatRates(customerType, vatNumberValidated)` from `lib/invoices/vat-rules.ts`. VIES validation via `lib/vat/vies-client.ts`. ### VAT Declaration Rutor (SKV 4700) @@ -76,7 +123,16 @@ The `VatDeclarationRutor` type maps to the Swedish tax authority's momsdeklarati - **Ruta 48**: Ingående moms — input VAT (from 2641/2645) - **Ruta 49**: Moms att betala/återfå = (ruta 10 + 11 + 12 + 30 + 31 + 32 + 60 + 61 + 62) - ruta 48 -`VatDeclaration.breakdown.invoices` also includes `base25`/`base12`/`base6` for per-rate revenue breakdown in the UI. +--- + +## Core Services (`lib/core/`) + +- `bookkeeping/period-service.ts` — Fiscal period lifecycle management (open, close, lock) +- `bookkeeping/year-end-service.ts` — Year-end closing procedures +- `bookkeeping/storno-service.ts` — Reversal/correction entry generation +- `tax/tax-code-service.ts` — Tax code definitions and rates +- `audit/audit-service.ts` — Audit trail and compliance logging +- `documents/document-service.ts` — Document attachment lifecycle (WORM storage with version chains) --- @@ -87,13 +143,14 @@ These rules exist for legal compliance, enforced by database triggers. **Never v 1. **Committed entries are immutable.** Once `status: 'posted'`, cannot be edited or deleted (DB trigger). 2. **Never delete posted entries.** Use `reverseEntry()` (storno) to cancel. 3. **Every entry must balance.** `sum(debits) === sum(credits)`, both `> 0`. -4. **Voucher numbers are sequential.** Assigned via DB RPC. Never set manually. -5. **Period lock enforcement.** DB trigger blocks writes to closed/locked periods. -6. **7-year document retention.** DB triggers prevent deletion of documents linked to posted entries. -7. **Storno, never edit.** Use `correctEntry()` from `lib/core/bookkeeping/storno-service.ts`. -8. **Use `Math.round(x * 100) / 100`** for monetary calculations. Never `toFixed()`. -9. **Always use engine functions.** Never insert directly into journal tables. -10. **Account numbers are strings.** `'1930'`, never `1930`. +4. **Voucher numbers are sequential.** Assigned atomically via `commit_journal_entry` DB RPC. Never set manually. +5. **Voucher gap documentation.** BFNAR 2013:2 requires documented explanations for gaps (`voucher_gap_explanations` table, `detect_voucher_gaps` RPC). +6. **Period lock enforcement.** DB trigger blocks writes to closed/locked periods. Company-wide lock date enforced via `enforce_company_lock_date()` trigger. +7. **7-year document retention.** DB triggers prevent deletion of documents linked to posted entries. +8. **Storno, never edit.** Use `correctEntry()` from `lib/core/bookkeeping/storno-service.ts`. +9. **Use `Math.round(x * 100) / 100`** for monetary calculations. Never `toFixed()`. +10. **Always use engine functions.** Never insert directly into journal tables. +11. **Account numbers are strings.** `'1930'`, never `1930`. --- @@ -101,6 +158,29 @@ These rules exist for legal compliance, enforced by database triggers. **Never v Extensions are opt-in plugins in `extensions/general//`, controlled by `extensions.config.json`. Core builds and runs with zero extensions. `npm run setup:extensions` generates static imports in `lib/extensions/_generated/` (runs automatically via `predev`/`prebuild`). Extensions **cannot** use dynamic imports (Next.js bundling). +### Available Extensions (12) + +| Extension | Purpose | Currently Enabled | +|-----------|---------|:-:| +| `enable-banking` | PSD2 bank sync via Enable Banking | Yes | +| `email` | Email delivery via Resend | Yes | +| `arcim-migration` | Legacy ARCIM system data migration | Yes | +| `tic` | TIC Identity company lookup (org number → name, VAT, address) | Yes | +| `mcp-server` | MCP server for Claude Desktop/Code | Yes | +| `receipt-ocr` | AI receipt scanning and extraction | No | +| `ai-categorization` | AI transaction categorization | No | +| `ai-chat` | AI assistant for bookkeeping questions | No | +| `push-notifications` | Web push notifications for events | No | +| `invoice-inbox` | Email-based invoice document processing | No | +| `calendar` | Payment calendar with iCal feed | No | +| `skatteverket` | Skatteverket VAT declaration submission | No | + +### Extension Architecture + +**Registration** (`lib/extensions/registry.ts`): Singleton registry. `register()` wires event handlers to the bus. `get(id)`, `getAll()`, `getByCapability(key)`. + +**Context** (`lib/extensions/context-factory.ts`): Every handler receives `ExtensionContext` with: `userId`, `companyId`, `extensionId`, `supabase`, `emit()`, `settings` (key-value in `extension_data` table), `storage` (Supabase Storage), `log` (prefixed logger), `services` (e.g., `ingestTransactions`). + **API routes**: Dispatched via catch-all at `app/api/extensions/ext/[...path]/route.ts`. URL: `/api/extensions/ext/{extensionId}/{routePath}`. Path params extracted as `_paramName` search params. **Service provider patterns**: @@ -117,9 +197,9 @@ gnubok exposes its bookkeeping engine as an MCP (Model Context Protocol) server, **MCP extension** (`extensions/general/mcp-server/`): 26 tools — transactions, categorization, customers, suppliers, invoices, supplier invoices, accounts, fiscal periods, trial balance, general ledger, balance sheet, income statement, AR/supplier ledger, reconciliation, VAT report, KPI report, receipt matching, invoice payments/sending. JSON-RPC 2.0 protocol implemented directly (no SDK dependency). Endpoint: `/api/extensions/ext/mcp-server/mcp`. -**API key infrastructure** (`lib/auth/api-keys.ts`, `api_keys` table): SHA-256 hashed keys with `gnubok_sk_` prefix. Rate limited at 100 RPM via atomic DB RPC (`validate_and_increment_api_key`). `createServiceClientNoCookies()` creates a Supabase service client without cookies for API key auth — all queries filter by `user_id` (defense in depth). +**API key infrastructure** (`lib/auth/api-keys.ts`, `api_keys` table): SHA-256 hashed keys with `gnubok_sk_` prefix. Scoped permissions mapped via `TOOL_SCOPE_MAP`. Rate limited at 100 RPM via atomic DB RPC (`validate_and_increment_api_key`). `createServiceClientNoCookies()` creates a Supabase service client without cookies for API key auth — all queries filter by `company_id` (defense in depth). -**OAuth 2.1** for Claude Desktop connectors (beta — Claude's callback has a known issue, see #78): +**OAuth 2.1** for Claude Desktop connectors: - `.well-known/oauth-protected-resource` and `.well-known/oauth-authorization-server` — discovery endpoints (excluded from auth middleware) - `/api/mcp-oauth/authorize` — consent page + auth code generation - `/api/mcp-oauth/token` — PKCE verification + API key creation @@ -130,8 +210,6 @@ gnubok exposes its bookkeeping engine as an MCP (Model Context Protocol) server, **npm package** (`packages/gnubok-mcp`): Published as `gnubok-mcp` on npm. Stdio-to-HTTP bridge for Claude Desktop. Users configure `npx gnubok-mcp` with their API key. -**KPI page** (`/kpi`): 4 metrics (Resultat, Kassa, Kundfordringar, Moms) + monthly trend chart. API at `/api/reports/kpi`. - --- ## API Route Pattern @@ -153,14 +231,106 @@ export async function POST(request: Request) { const result = await validateBody(request, MySchema) if (!result.success) return result.response - // Business logic... always filter by user_id (defense in depth alongside RLS) - // Wrap journal entry creation in try/catch (non-blocking side effect) + // Business logic... always filter by company_id (defense in depth alongside RLS) return NextResponse.json({ data: result }) } ``` - Dynamic route params: `{ params }: { params: Promise<{ id: string }> }` (Next.js 16) - Response shapes: `{ data }` for success, `{ error }` for failures +- Zod schemas in `lib/api/schemas.ts` — 30+ schemas with shared primitives (uuid, isoDate, accountNumber, nonNegativeAmount) + +--- + +## Key lib/ Directories + +| Directory | Purpose | +|-----------|---------| +| `bookkeeping/` | Engine, entry generators, mapping, templates, BAS data | +| `core/` | Period service, year-end, storno, tax codes, audit, documents | +| `events/` | Event bus singleton, 30+ event types, event log handler | +| `auth/` | API keys, require-auth, MFA, OAuth codes, invite tokens, cron auth | +| `supabase/` | Browser/server/service clients, middleware, fetch-all pagination | +| `api/` | Zod validation (`validateBody`/`validateQuery`), schemas | +| `reports/` | 17 report generators (financial statements, ledgers, tax, exports) | +| `invoices/` | Invoice/supplier matching, payment match log, reminders, VAT rules, PDF template | +| `transactions/` | Multi-source ingestion (`ingest.ts`), AI category suggestions | +| `import/` | SIE parser/import, account mapper | +| `documents/` | Document matcher, receipt matcher, batch matching | +| `extensions/` | Registry, loader, context factory, types, generated files | +| `email/` | Service interface (noop default), Resend provider, templates (invite, invoice, reminder, consent) | +| `company/` | Company context resolution, CRUD actions | +| `reconciliation/` | Bank statement reconciliation | +| `tax/` | Tax calculator, deadline config/generator, expense warnings, Swedish holidays | +| `vat/` | VIES client, EU countries, MOMS box mapping | +| `deadlines/` | Deadline status engine | +| `currency/` | Riksbanken exchange rates | +| `skatteverket/` | Tax authority data formatting | +| `bankgiro/` | Luhn checksum validation | +| `calendar/` | ICS generator, calendar utilities | +| `errors/` | Swedish error message mapping (Zod → Postgres → HTTP → fallback) | +| `hooks/` | React hooks (e.g., `use-unsaved-changes`) | +| `settings/` | Settings utilities | +| `logger.ts` | Structured logger with module prefixes, env-aware filtering | +| `utils.ts` | `cn()`, `formatCurrency()`, `formatDate()`, `formatOrgNumber()` | + +--- + +## App Routes + +### Pages + +| Route | Purpose | +|-------|---------| +| `/login`, `/register`, `/reset-password` | Auth pages | +| `/mfa/enroll`, `/mfa/verify` | MFA flow | +| `/onboarding` | Multi-step company setup wizard | +| `/companies/new` | Create new company | +| `/invite/[token]` | Accept team/company invite | +| `/` | Dashboard home | +| `/transactions` | Bank transaction list & categorization | +| `/invoices`, `/invoices/new`, `/invoices/[id]`, `/invoices/[id]/credit` | Customer invoicing | +| `/supplier-invoices`, `/supplier-invoices/new`, `/supplier-invoices/[id]` | Supplier invoices | +| `/customers`, `/customers/[id]` | Customer management | +| `/suppliers`, `/suppliers/[id]` | Supplier management | +| `/expenses`, `/expenses/new`, `/expenses/[id]` | Expense tracking | +| `/receipts`, `/receipts/scan` | Receipt management | +| `/bookkeeping`, `/bookkeeping/[id]`, `/bookkeeping/year-end` | Journal entries, chart of accounts, year-end | +| `/reports` | Financial reports | +| `/import` | SIE and bank file import | +| `/kpi` | KPI metrics + monthly trend chart | +| `/deadlines` | Tax & business deadlines | +| `/pending` | Pending operations queue | +| `/extensions`, `/extensions/[sector]/[extension]` | Extension marketplace | +| `/e/[sector]/[slug]` | Extension workspace | +| `/settings/*` | Company, invoicing, bookkeeping, tax, team, banking, templates, account, API settings | +| `/dpa`, `/privacy` | Legal pages | +| `/invoice-action/[token]` | Public invoice payment link | +| `/sandbox` | Test environment | + +### API Endpoints (key groups) + +- `/api/bookkeeping/*` — Accounts, fiscal periods (close/lock/year-end/opening-balances/currency-revaluation), journal entries (CRUD/reverse/correct/chain), mapping rules, voucher gaps +- `/api/invoices/*` — CRUD, send, mark-sent/paid, convert, PDF, reminders cron +- `/api/supplier-invoices/*` — CRUD, approve, mark-paid, credit +- `/api/transactions/*` — Categorize, uncategorize, describe, book, match-invoice, match-supplier-invoice, batch operations, AI suggestions +- `/api/customers/*`, `/api/suppliers/*` — CRUD +- `/api/documents/*` — CRUD, versions, link, verify, match-sweep, verify cron +- `/api/reports/*` — 16 report endpoints (general-ledger, trial-balance, balance-sheet, income-statement, journal-register, ar-ledger, supplier-ledger, vat-declaration, sie-export, ink2, ne-bilaga, kpi, audit-trail, continuity-check, monthly-breakdown, full-archive) +- `/api/import/*` — Bank file (parse/execute), SIE (parse/execute/mappings/create-accounts) +- `/api/reconciliation/bank/*` — Link, unlink, run, status, unmatched-entries +- `/api/settings/*` — Company settings, API keys, logo upload, counterparty templates +- `/api/company/members/*` — List, CRUD, invite +- `/api/team/*` — Accept, invite, members +- `/api/deadlines/*`, `/api/tax-deadlines/*` — Deadline CRUD and crons +- `/api/pending-operations/*` — Queue, commit, reject +- `/api/events/*` — Event log and cleanup cron +- `/api/calendar/feed/[token]` — iCal subscription feed +- `/api/mcp-oauth/*` — Register, authorize, token +- `/api/health` — Health check +- `/api/vat/validate` — VIES VAT validation +- `/api/sandbox/*` — Seed, cleanup cron +- `/api/extensions/ext/[...path]` — Dynamic extension API routes --- @@ -168,7 +338,7 @@ export async function POST(request: Request) { **Framework**: Vitest 4, `globals: true`, `environment: 'node'`. Tests colocated in `__tests__/` directories. Scope: business logic in `lib/` and API routes in `app/api/`. No component or E2E tests. -**Test helpers** (`tests/helpers.ts`): `createMockSupabase()`, `createQueuedMockSupabase()`, `createMockRequest()`, `parseJsonResponse()`, `createMockRouteParams()`, and fixture factories (`makeTransaction()`, `makeJournalEntry()`, `makeInvoice()`, `makeCustomer()`, `makeSupplier()`, `makeSupplierInvoice()`, `makeFiscalPeriod()`, `makeReceipt()`, `makeDocumentAttachment()`, `makeCompanySettings()`, `makeInvoiceInboxItem()`, etc.). +**Test helpers** (`tests/helpers.ts`): `createMockSupabase()` (chainable proxy), `createQueuedMockSupabase()` (sequential calls), `createMockRequest()`, `parseJsonResponse()`, `createMockRouteParams()`, and fixture factories: `makeTransaction()`, `makeJournalEntry()`, `makeJournalEntryLine()`, `makeInvoice()`, `makeInvoicePayment()`, `makeCustomer()`, `makeSupplier()`, `makeSupplierInvoice()`, `makeFiscalPeriod()`, `makeReceipt()`, `makeDocumentAttachment()`, `makeCompanySettings()`, `makeCompany()`, `makeCompanyMember()`, `makeInvoiceInboxItem()`, `makeTaxCode()`, `makeCategorizationTemplate()`, `makeSIEVoucher()`, `makeBankConnection()`. **Patterns**: Always mock `@/lib/supabase/server`. Use `vi.clearAllMocks()` and `eventBus.clear()` in `beforeEach`. API route tests: mock `@/lib/init` and lib functions, test auth (401), validation (400), not found (404), errors (500), happy path. @@ -176,14 +346,65 @@ export async function POST(request: Request) { ## Database & Migrations -**Location**: `supabase/migrations/` — 80 files. Early migrations use sequential numbering (`20240101000001`–`20240101000038`), later ones use real timestamps. +**Location**: `supabase/migrations/` — 93 files. Early migrations use sequential numbering (`20240101000001`–`20240101000038`), later ones use real timestamps. + +### Key Tables (~47) + +**Multi-tenant**: `companies`, `company_members`, `company_invitations`, `teams`, `team_members`, `team_invitations`, `user_preferences`, `profiles` + +**Bookkeeping**: `chart_of_accounts`, `fiscal_periods`, `journal_entries`, `journal_entry_lines`, `account_balances`, `voucher_sequences`, `voucher_gap_explanations` + +**Invoicing**: `customers`, `invoices`, `invoice_items`, `invoice_payments`, `invoice_inbox_items` + +**Suppliers**: `suppliers`, `supplier_invoices`, `supplier_invoice_items` + +**Banking**: `bank_connections`, `transactions`, `bank_file_imports`, `payment_match_log` + +**Documents**: `document_attachments` (WORM), `receipts`, `receipt_line_items` + +**Settings & Config**: `company_settings`, `mapping_rules`, `categorization_templates`, `extension_data` + +**Dimensions**: `cost_centers`, `projects` + +**Tax & Deadlines**: `tax_rates`, `deadlines`, `calendar_feeds`, `skatteverket_tokens` + +**API & Auth**: `api_keys` (with scopes), `oauth_used_codes` + +**Audit & Ops**: `audit_log` (immutable), `event_log` (30-day TTL), `pending_operations`, `ai_usage_tracking` + +**Other**: `salary_payments`, `sandbox_users` + +### Key RPC Functions + +- `create_company_with_owner()` — Atomic company + owner creation +- `commit_journal_entry()` — Atomic draft→posted with voucher number +- `next_voucher_number()` — Concurrent-safe voucher generation +- `detect_voucher_gaps()` — BFNAR 2013:2 gap detection +- `generate_invoice_number()`, `get_next_arrival_number()`, `generate_delivery_note_number()` — Sequence generators +- `seed_chart_of_accounts()` — BAS chart seeding per entity type +- `validate_and_increment_api_key()` — Atomic rate limiting +- `user_company_ids()` — RLS helper returning user's company IDs +- `get_unlinked_1930_lines()` — Bank reconciliation helper +- `cleanup_sandbox_user()`, `cleanup_expired_sandbox_users()` — Sandbox lifecycle + +### Key Triggers + +- `check_journal_entry_balance()` — Debit must equal credit +- `enforce_journal_entry_immutability()` — Posted entries cannot be modified +- `enforce_period_lock()` — No entries in closed/locked periods +- `enforce_company_lock_date()` — Company-wide bookkeeping lock date +- `block_document_deletion()` — WORM compliance +- `enforce_retention_journal_entries()` — 7-year retention +- `audit_log_immutable()` — Audit log cannot be modified +- `write_audit_log()` — Auto-audit on DML operations +- `sync_team_member_to_companies()` — Auto-sync team→company membership ### Migration Rules -1. **Always enable RLS** and create `SELECT/INSERT/UPDATE` policies using `auth.uid() = user_id` +1. **Always enable RLS** and create policies using `user_company_ids()` for company-scoped data 2. **Always add `updated_at` trigger** using `update_updated_at_column()` 3. **UUID primary keys**: `DEFAULT uuid_generate_v4()` -4. **User ownership**: `user_id UUID REFERENCES auth.users ON DELETE CASCADE NOT NULL` +4. **Company ownership**: `company_id UUID REFERENCES companies NOT NULL` + `user_id UUID REFERENCES auth.users ON DELETE CASCADE NOT NULL` 5. **Never modify existing migrations** — create new ones 6. **Never modify enforcement triggers** (migration 017) — legally required 7. **Apply via Supabase MCP tool**: `mcp__plugin_supabase_supabase__apply_migration` @@ -192,19 +413,48 @@ export async function POST(request: Request) { ## Skills, Git & CI -**Skills**: Always use `/frontend-design` for new UI. Use `langchain` for AI features. Use `vercel:deploy` for deployment. +**Skills**: Always use `/frontend-design` for new UI. Use `vercel:deploy` for deployment. Use `/supabase-migration` for new migrations. Use `/erp-api-route` for new API routes. Use `/create-extension` for new extensions. Use `/swedish-bookkeeping` for accounting domain questions. **Git**: Conventional commits (`feat:`, `fix:`, `refactor:`, `test:`, `docs:`). Atomic commits, branch from `main`. **CI** (`.github/workflows/core-build.yml`): Resets extensions to empty, runs build + test, verifies no core code imports from `@/extensions/` directly. +**Docker** (`.github/workflows/docker-publish.yml`): Pushes to GHCR (`erp-mafia/erp-base`) on main push. 4-stage Dockerfile (base → deps → builder → runner) with Node 22 Alpine. Runtime env placeholder replacement via `docker-entrypoint.sh`. Docker Compose with app + supercronic cron service. + --- ## Deployment -Hosted on **Vercel**. Cron jobs defined in `vercel.json` (banking sync, deadlines, reminders, tax deadlines, document verification, sandbox cleanup, event cleanup). +### Vercel (Hosted) -**Core env vars**: `NEXT_PUBLIC_SUPABASE_URL`, `NEXT_PUBLIC_SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `NEXT_PUBLIC_APP_URL`, `CRON_SECRET`. **Auth env vars**: `NEXT_PUBLIC_REQUIRE_MFA` (set `true` on hosted), `NEXT_PUBLIC_SELF_HOSTED` (set `true` for Docker). Extension env vars only needed when that extension is enabled. +Cron jobs defined in `vercel.json`: + +| Schedule | Endpoint | Purpose | +|----------|----------|---------| +| `0 6 * * *` | `/api/deadlines/status/cron` | Update deadline statuses | +| `0 8 * * *` | `/api/invoices/reminders/cron` | Send invoice reminders | +| `0 0 2 1 *` | `/api/tax-deadlines/cron` | Generate tax deadlines | +| `0 5 * * *` | `/api/extensions/enable-banking/sync/cron` | Bank transaction sync | +| `0 3 * * 0` | `/api/documents/verify/cron` | Document integrity verification | +| `0 4 * * *` | `/api/sandbox/cleanup/cron` | Sandbox user cleanup | +| `0 2 * * *` | `/api/events/cleanup/cron` | Event log cleanup (30-day TTL) | + +### Docker (Self-Hosted) + +- `Dockerfile`: 4-stage Node 22 Alpine build with standalone output +- `docker-compose.yml`: App service + supercronic cron scheduler +- `docker-entrypoint.sh`: Validates required env vars, replaces build-time placeholders in `.next/static/` JS +- Extension presets: `docker/extensions.self-hosted.json`, `docker/extensions.hosted.json` + +### Environment Variables + +**Required**: `NEXT_PUBLIC_SUPABASE_URL`, `NEXT_PUBLIC_SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `NEXT_PUBLIC_APP_URL`, `CRON_SECRET` + +**Auth**: `NEXT_PUBLIC_REQUIRE_MFA` (set `true` on hosted), `NEXT_PUBLIC_SELF_HOSTED` (set `true` for Docker) + +**Extension-specific** (only when extension is enabled): `ENABLE_BANKING_APP_ID`/`ENABLE_BANKING_APP_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `RESEND_API_KEY`, `VAPID_PUBLIC_KEY`/`VAPID_PRIVATE_KEY` + +**Optional**: `SENTRY_DSN`, `SENTRY_AUTH_TOKEN` ## Other Never create a NUL/nul file: \gnubok\NUL diff --git a/app/(dashboard)/settings/bookkeeping/page.tsx b/app/(dashboard)/settings/bookkeeping/page.tsx index 26d9625f..165ac5eb 100644 --- a/app/(dashboard)/settings/bookkeeping/page.tsx +++ b/app/(dashboard)/settings/bookkeeping/page.tsx @@ -25,8 +25,12 @@ export default function BookkeepingSettingsPage() { auto_lock_period_days: autoLockValue === 'none' ? null : parseInt(autoLockValue), accounting_method: accountingMethod, } - updateSettings(updates as Partial) - return updates + return { + updates, + onSuccess: (data: Record) => { + updateSettings(data as Partial) + }, + } } return ( diff --git a/app/(dashboard)/settings/company/page.tsx b/app/(dashboard)/settings/company/page.tsx index dafff5e1..d7176d26 100644 --- a/app/(dashboard)/settings/company/page.tsx +++ b/app/(dashboard)/settings/company/page.tsx @@ -24,8 +24,12 @@ export default function CompanySettingsPage() { email: (formData.get('email') as string) || '', website: (formData.get('website') as string) || '', } - updateSettings(updates as Partial) - return updates + return { + updates, + onSuccess: (data: Record) => { + updateSettings(data as Partial) + }, + } } return ( diff --git a/app/(dashboard)/settings/invoicing/page.tsx b/app/(dashboard)/settings/invoicing/page.tsx index 31dc98b6..51c80d74 100644 --- a/app/(dashboard)/settings/invoicing/page.tsx +++ b/app/(dashboard)/settings/invoicing/page.tsx @@ -36,8 +36,12 @@ export default function InvoicingSettingsPage() { invoice_default_days: parseInt(formData.get('invoice_default_days') as string) || 30, invoice_default_notes: (formData.get('invoice_default_notes') as string) || null, } - updateSettings(updates as Partial) - return updates + return { + updates, + onSuccess: (data: Record) => { + updateSettings(data as Partial) + }, + } } return ( diff --git a/app/(dashboard)/settings/tax/page.tsx b/app/(dashboard)/settings/tax/page.tsx index 4fc4923c..2d7c3f32 100644 --- a/app/(dashboard)/settings/tax/page.tsx +++ b/app/(dashboard)/settings/tax/page.tsx @@ -12,15 +12,27 @@ export default function TaxSettingsPage() { if (isLoading || !settings) return function handleSave(formData: FormData) { + const vatRegistered = formData.get('vat_registered') === 'true' + const updates: Record = { + f_skatt: formData.get('f_skatt') === 'true', + vat_registered: vatRegistered, + vat_number: vatRegistered ? ((formData.get('vat_number') as string) || null) : null, + moms_period: vatRegistered ? ((formData.get('moms_period') as string) || null) : null, + fiscal_year_start_month: parseInt(formData.get('fiscal_year_start_month') as string) || 1, + pays_salaries: formData.get('pays_salaries') === 'true', preliminary_tax_monthly: parseFloat(formData.get('preliminary_tax_monthly') as string) || null, } - updateSettings(updates as Partial) - return updates + return { + updates, + onSuccess: (data: Record) => { + updateSettings(data as Partial) + }, + } } return ( - + ) diff --git a/app/api/settings/logo/route.ts b/app/api/settings/logo/route.ts index fc61d4d7..9e2a0a49 100644 --- a/app/api/settings/logo/route.ts +++ b/app/api/settings/logo/route.ts @@ -1,4 +1,4 @@ -import { createClient } from '@/lib/supabase/server' +import { createClient, createServiceClient } from '@/lib/supabase/server' import { NextResponse } from 'next/server' import { requireCompanyId } from '@/lib/company/context' @@ -32,8 +32,9 @@ export async function POST(request: Request) { const ext = file.name.split('.').pop() || 'png' const storagePath = `logos/${companyId}/logo.${ext}` - // Upload (upsert to replace existing) - const { error: uploadError } = await supabase.storage + // Upload with service client to bypass storage RLS (auth already verified above) + const serviceClient = createServiceClient() + const { error: uploadError } = await serviceClient.storage .from('documents') .upload(storagePath, buffer, { contentType: file.type, @@ -45,7 +46,7 @@ export async function POST(request: Request) { } // Get public URL - const { data: urlData } = supabase.storage + const { data: urlData } = serviceClient.storage .from('documents') .getPublicUrl(storagePath) @@ -82,7 +83,8 @@ export async function DELETE() { const url = new URL(settings.logo_url) const pathMatch = url.pathname.match(/\/object\/public\/documents\/(.+)/) if (pathMatch) { - await supabase.storage.from('documents').remove([pathMatch[1]]) + const serviceClient = createServiceClient() + await serviceClient.storage.from('documents').remove([pathMatch[1]]) } } diff --git a/components/settings/SettingsFormWrapper.tsx b/components/settings/SettingsFormWrapper.tsx index 54804691..5aaf1781 100644 --- a/components/settings/SettingsFormWrapper.tsx +++ b/components/settings/SettingsFormWrapper.tsx @@ -5,9 +5,13 @@ import { Button } from '@/components/ui/button' import { Loader2, Check } from 'lucide-react' import { useToast } from '@/components/ui/use-toast' +type SaveResult = + | Record + | { updates: Record; onSuccess?: (data: Record) => void } + interface SettingsFormWrapperProps { children: React.ReactNode - onSave?: (formData: FormData) => Record + onSave?: (formData: FormData) => SaveResult className?: string } @@ -28,7 +32,12 @@ export function SettingsFormWrapper({ children, onSave, className }: SettingsFor if (!onSave) return const formData = new FormData(e.currentTarget) - const updates = onSave(formData) + const saveResult = onSave(formData) + + // Support both plain object and { updates, onSuccess } return types + const isStructured = saveResult && 'updates' in saveResult && typeof saveResult.updates === 'object' + const updates = isStructured ? saveResult.updates : saveResult + const onSuccess = isStructured ? (saveResult as { onSuccess?: (data: Record) => void }).onSuccess : undefined if (!updates || Object.keys(updates).length === 0) return @@ -48,6 +57,7 @@ export function SettingsFormWrapper({ children, onSave, className }: SettingsFor throw new Error(result.error || 'Kunde inte spara inställningar') } + onSuccess?.(result.data ?? updates) setSaved(true) timerRef.current = setTimeout(() => setSaved(false), 2000) } catch (error) { diff --git a/components/settings/TaxSettingsForm.tsx b/components/settings/TaxSettingsForm.tsx index 2b77a5d2..108285b1 100644 --- a/components/settings/TaxSettingsForm.tsx +++ b/components/settings/TaxSettingsForm.tsx @@ -1,7 +1,11 @@ 'use client' +import { useState } from 'react' import { Input } from '@/components/ui/input' import { Label } from '@/components/ui/label' +import { Checkbox } from '@/components/ui/checkbox' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { Badge } from '@/components/ui/badge' import type { CompanySettings } from '@/types' interface TaxSettingsFormProps { @@ -9,26 +13,183 @@ interface TaxSettingsFormProps { } export function TaxSettingsForm({ settings }: TaxSettingsFormProps) { - return ( -
-

- Preliminärskatt -

+ const [vatRegistered, setVatRegistered] = useState(settings.vat_registered ?? false) + const [fSkatt, setFSkatt] = useState(settings.f_skatt ?? true) + const [paysSalaries, setPaysSalaries] = useState(settings.pays_salaries ?? false) -
- - -

- Belopp i SEK som betalas varje månad. -

-
-
+ const isEnskildFirma = settings.entity_type === 'enskild_firma' + + return ( +
+ {/* Entity type — read-only */} +
+

+ Företagsform +

+
+ + {settings.entity_type === 'aktiebolag' ? 'Aktiebolag' : 'Enskild firma'} + +

+ Företagsform kan inte ändras. Kontakta support vid behov. +

+
+
+ + {/* F-skatt */} +
+

+ Skatt & moms +

+ +
+
+ setFSkatt(v === true)} + /> + +
+ +

+ Godkänd för F-skatt (självständig näringsverksamhet). +

+
+
+ +
+ setVatRegistered(v === true)} + /> + +
+ +

+ Obligatoriskt om omsättningen överstiger 120 000 kr per år. +

+
+
+ + {vatRegistered && ( +
+
+ + +

+ Format: SE + organisationsnummer + 01 +

+
+ +
+ + +

+ Enligt beslut från Skatteverket. +

+
+
+ )} +
+
+ + {/* Fiscal year & salaries */} +
+

+ Räkenskapsår & löner +

+ +
+ + {isEnskildFirma ? ( + <> + + +

+ Enskild firma måste använda kalenderår (BFL 3 kap.). +

+ + ) : ( + <> + +

+ Ändring påverkar framtida räkenskapsår. +

+ + )} +
+ +
+ setPaysSalaries(v === true)} + /> + +
+ +

+ Påverkar vilka skattedeadlines som visas (arbetsgivardeklaration m.m.). +

+
+
+
+ + {/* Preliminary tax */} +
+

+ Preliminärskatt +

+ +
+ + +

+ Belopp i SEK som betalas varje månad. +

+
+
+
) } diff --git a/lib/api/schemas.ts b/lib/api/schemas.ts index 75c9c7f7..768345a0 100644 --- a/lib/api/schemas.ts +++ b/lib/api/schemas.ts @@ -383,7 +383,7 @@ export const UpdateSettingsSchema = z.object({ invoice_default_days: z.number().int().positive().optional(), invoice_default_notes: z.string().nullable().optional(), phone: z.string().optional(), - email: z.string().email().optional(), + email: z.string().email().optional().or(z.literal('')), website: z.string().optional().or(z.literal('')), pays_salaries: z.boolean().optional(), sector_slug: z.string().nullable().optional(),