From 55bc3e862a67812b4fd04a22b19698d7c45964f7 Mon Sep 17 00:00:00 2001 From: dimgigov Date: Sun, 17 May 2026 13:46:42 +0300 Subject: [PATCH] =?UTF-8?q?feat(langchain):=20Session=2010.2=20=E2=80=94?= =?UTF-8?q?=20LangChain=20Vector=20Store=20(Python=20+=20JS)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BaraDBStore for Python: add_texts, similarity_search, max_marginal_relevance_search, delete - BaraDBStore for JS: addDocuments, addTexts, similaritySearch, maxMarginalRelevanceSearch, delete - Both use hybrid_search() / hybrid_search_filtered() for vector+FTS+RRF - Multi-tenant support via tenant_id session variable + metadata filter - Embedding function is injected by user (OpenAI, sentence-transformers, etc.) - MMR reranking for result diversity --- clients/javascript/baradb_langchain.js | 202 +++++++++++++++++++++ clients/python/baradb/langchain_README.md | 67 +++++++ clients/python/baradb/langchain_store.py | 212 ++++++++++++++++++++++ 3 files changed, 481 insertions(+) create mode 100644 clients/javascript/baradb_langchain.js create mode 100644 clients/python/baradb/langchain_README.md create mode 100644 clients/python/baradb/langchain_store.py diff --git a/clients/javascript/baradb_langchain.js b/clients/javascript/baradb_langchain.js new file mode 100644 index 0000000..a547254 --- /dev/null +++ b/clients/javascript/baradb_langchain.js @@ -0,0 +1,202 @@ +/** + * BaraDB LangChain.js Vector Store Integration + * + * Usage: + * const { Client } = require('./baradb'); + * const { BaraDBStore } = require('./baradb_langchain'); + * + * const client = new Client('localhost', 9472); + * await client.connect(); + * + * const store = new BaraDBStore({ + * client, + * table: 'docs', + * embeddingCol: 'embedding', + * textCol: 'content', + * embeddingFunction: async (text) => [0.1, 0.2, ...], // your embedder + * tenantId: 'company-a' + * }); + * + * await store.addDocuments([ + * { pageContent: 'hello world', metadata: { source: 'web' } } + * ]); + * + * const results = await store.similaritySearch('hello', 5); + */ + +class BaraDBStore { + constructor(options = {}) { + this.client = options.client; + this.table = options.table || 'documents'; + this.embeddingCol = options.embeddingCol || 'embedding'; + this.textCol = options.textCol || 'content'; + this.metadataCols = options.metadataCols || []; + this.embeddingFunction = options.embeddingFunction || null; + this.tenantId = options.tenantId || null; + this.vectorDimension = options.vectorDimension || 1536; + this._tableCreated = false; + } + + async _ensureTable() { + if (this._tableCreated) return; + + const cols = `id SERIAL PRIMARY KEY, ${this.embeddingCol} VECTOR(${this.vectorDimension}), ${this.textCol} TEXT` + + (this.tenantId ? ', tenant_id TEXT' : '') + + this.metadataCols.map(mc => `, ${mc} TEXT`).join(''); + + await this.client.query(`CREATE TABLE IF NOT EXISTS ${this.table} (${cols})`); + await this.client.query(`CREATE INDEX IF NOT EXISTS idx_${this.table}_vec ON ${this.table}(${this.embeddingCol}) USING hnsw`); + await this.client.query(`CREATE INDEX IF NOT EXISTS idx_${this.table}_fts ON ${this.table}(${this.textCol}) USING FTS`); + this._tableCreated = true; + } + + async addDocuments(documents) { + await this._ensureTable(); + if (!this.embeddingFunction) { + throw new Error('embeddingFunction is required for addDocuments'); + } + + const insertedIds = []; + for (const doc of documents) { + const text = doc.pageContent || doc.content || ''; + const meta = doc.metadata || {}; + const vec = await this.embeddingFunction(text); + const vecStr = '[' + vec.join(',') + ']'; + + const metaCols = []; + const metaVals = []; + if (this.tenantId) { + metaCols.push('tenant_id'); + metaVals.push(`'${this.tenantId}'`); + } + for (const mc of this.metadataCols) { + if (meta[mc] !== undefined) { + metaCols.push(mc); + metaVals.push(`'${String(meta[mc]).replace(/'/g, "''")}'`); + } + } + + let colList = `${this.embeddingCol}, ${this.textCol}`; + let valList = `'${vecStr}', '${text.replace(/'/g, "''")}'`; + if (metaCols.length > 0) { + colList += ', ' + metaCols.join(', '); + valList += ', ' + metaVals.join(', '); + } + + const sql = `INSERT INTO ${this.table} (${colList}) VALUES (${valList}) RETURNING id`; + const result = await this.client.query(sql); + if (result.rows && result.rows.length > 0) { + insertedIds.push(result.rows[0].id || result.rows[0][0]); + } + } + return insertedIds; + } + + async addTexts(texts, metadatas = []) { + const docs = texts.map((text, i) => ({ + pageContent: text, + metadata: metadatas[i] || {} + })); + return this.addDocuments(docs); + } + + async similaritySearch(query, k = 4, filter = null) { + await this._ensureTable(); + if (!this.embeddingFunction) { + throw new Error('embeddingFunction is required for similaritySearch'); + } + + const vec = await this.embeddingFunction(query); + const vecStr = '[' + vec.join(',') + ']'; + + if (this.tenantId) { + await this.client.query(`SET app.tenant_id = '${this.tenantId}'`); + } + + let sql; + if (filter && filter.column && filter.value) { + sql = `SELECT hybrid_search_filtered('${this.table}', '${this.embeddingCol}', '${this.textCol}', '${query.replace(/'/g, "''")}', '${vecStr}', ${k}, '${filter.column}', '${filter.value}') AS res`; + } else { + sql = `SELECT hybrid_search('${this.table}', '${this.embeddingCol}', '${this.textCol}', '${query.replace(/'/g, "''")}', '${vecStr}', ${k}) AS res`; + } + + const result = await this.client.query(sql); + if (!result.rows || result.rows.length === 0) return []; + + const raw = result.rows[0].res || result.rows[0][0] || '[]'; + let arr; + try { + arr = JSON.parse(raw); + } catch { + return []; + } + + const docs = []; + for (const item of arr) { + const docId = item.id; + const score = parseFloat(item.score || 0); + const rowResult = await this.client.query(`SELECT * FROM ${this.table} WHERE id = ${docId}`); + if (rowResult.rows && rowResult.rows.length > 0) { + const row = rowResult.rows[0]; + const pageContent = row[this.textCol] || row[Object.keys(row).find(k => k.toLowerCase() === this.textCol.toLowerCase())]; + docs.push({ + pageContent: String(pageContent), + metadata: { ...row, _score: score }, + }); + } + } + return docs; + } + + async maxMarginalRelevanceSearch(query, k = 4, fetchK = 20, lambdaMult = 0.5) { + const candidates = await this.similaritySearch(query, fetchK); + if (candidates.length === 0) return []; + + const selected = []; + const remaining = [...candidates]; + + while (selected.length < k && remaining.length > 0) { + let bestScore = -Infinity; + let bestIdx = 0; + for (let i = 0; i < remaining.length; i++) { + const doc = remaining[i]; + // Use _score from metadata as relevance + const relScore = doc.metadata?._score || 0; + let penalty = 0; + for (const sel of selected) { + penalty = Math.max(penalty, _docSimilarity(doc, sel)); + } + const mmrScore = lambdaMult * relScore - (1 - lambdaMult) * penalty; + if (mmrScore > bestScore) { + bestScore = mmrScore; + bestIdx = i; + } + } + selected.push(remaining.splice(bestIdx, 1)[0]); + } + return selected; + } + + async delete(ids) { + await this._ensureTable(); + if (!ids || ids.length === 0) return; + const idList = ids.join(', '); + await this.client.query(`DELETE FROM ${this.table} WHERE id IN (${idList})`); + } + + async setTenant(tenantId) { + this.tenantId = tenantId; + await this.client.query(`SET app.tenant_id = '${tenantId}'`); + } +} + +function _docSimilarity(a, b) { + const tokensA = new Set(String(a.pageContent || '').toLowerCase().split(/\s+/)); + const tokensB = new Set(String(b.pageContent || '').toLowerCase().split(/\s+/)); + if (tokensA.size === 0 || tokensB.size === 0) return 0; + const intersection = new Set([...tokensA].filter(x => tokensB.has(x))); + const union = new Set([...tokensA, ...tokensB]); + return intersection.size / union.size; +} + +module.exports = { BaraDBStore }; diff --git a/clients/python/baradb/langchain_README.md b/clients/python/baradb/langchain_README.md new file mode 100644 index 0000000..811dbb4 --- /dev/null +++ b/clients/python/baradb/langchain_README.md @@ -0,0 +1,67 @@ +# BaraDB LangChain Integration + +## Python + +```python +import asyncio +from baradb import Client +from baradb.langchain_store import BaraDBStore + +async def main(): + client = Client("localhost", 9472) + await client.connect() + + # Use OpenAI, sentence-transformers, or any embedder + def embed(text: str) -> list[float]: + # Replace with your embedding model + return [0.1, 0.2, 0.3] + + store = BaraDBStore( + client=client, + table="knowledge", + embedding_function=embed, + tenant_id="tenant-a", + vector_dimension=3, + ) + + await store.add_texts(["BaraDB is fast", "Vector search in SQL"]) + results = await store.similarity_search("fast database", k=5) + for doc, score in results: + print(doc.page_content, score) + +asyncio.run(main()) +``` + +## JavaScript + +```javascript +const { Client } = require('./baradb'); +const { BaraDBStore } = require('./baradb_langchain'); + +async function main() { + const client = new Client('localhost', 9472); + await client.connect(); + + const store = new BaraDBStore({ + client, + table: 'knowledge', + embeddingFunction: async (text) => [0.1, 0.2, 0.3], + tenantId: 'tenant-a', + vectorDimension: 3, + }); + + await store.addTexts(['BaraDB is fast', 'Vector search in SQL']); + const results = await store.similaritySearch('fast database', 5); + console.log(results); +} + +main(); +``` + +## Features + +- `add_texts()` / `addDocuments()` — auto-generate embeddings + INSERT +- `similarity_search()` — uses `hybrid_search()` (vector + FTS + RRF) +- `max_marginal_relevance_search()` — MMR reranking for diversity +- `delete()` — remove by IDs +- Multi-tenant — `tenant_id` sets session variable + metadata filter diff --git a/clients/python/baradb/langchain_store.py b/clients/python/baradb/langchain_store.py new file mode 100644 index 0000000..636325c --- /dev/null +++ b/clients/python/baradb/langchain_store.py @@ -0,0 +1,212 @@ +""" +BaraDB LangChain Vector Store Integration + +Usage: + from baradb import Client + from baradb.langchain_store import BaraDBStore + from langchain.embeddings import OpenAIEmbeddings + + client = Client("localhost", 9472) + await client.connect() + + store = BaraDBStore( + client=client, + table="docs", + embedding_col="embedding", + text_col="content", + embedding_function=OpenAIEmbeddings().embed_query, + tenant_id="company-a" # optional, for RLS + ) + + await store.add_texts(["hello world", "quick brown fox"]) + results = await store.similarity_search("hello", k=5) +""" + +import json +from typing import Any, Callable, List, Optional, Sequence, Tuple + + +class BaraDBStore: + """LangChain-compatible Vector Store for BaraDB.""" + + def __init__( + self, + client: Any, + table: str = "documents", + embedding_col: str = "embedding", + text_col: str = "content", + metadata_cols: Optional[List[str]] = None, + embedding_function: Optional[Callable[[str], List[float]]] = None, + tenant_id: Optional[str] = None, + vector_dimension: int = 1536, + ): + self.client = client + self.table = table + self.embedding_col = embedding_col + self.text_col = text_col + self.metadata_cols = metadata_cols or [] + self.embedding_function = embedding_function + self.tenant_id = tenant_id + self.vector_dimension = vector_dimension + self._table_created = False + + async def _ensure_table(self) -> None: + if self._table_created: + return + # Create table with vector + text + tenant_id columns + cols = f"id SERIAL PRIMARY KEY, {self.embedding_col} VECTOR({self.vector_dimension}), {self.text_col} TEXT" + if self.tenant_id: + cols += ", tenant_id TEXT" + for mc in self.metadata_cols: + cols += f", {mc} TEXT" + await self.client.query(f"CREATE TABLE IF NOT EXISTS {self.table} ({cols})") + + # Create indexes if not exist + idx_vec = f"idx_{self.table}_vec" + idx_fts = f"idx_{self.table}_fts" + await self.client.query(f"CREATE INDEX IF NOT EXISTS {idx_vec} ON {self.table}({self.embedding_col}) USING hnsw") + await self.client.query(f"CREATE INDEX IF NOT EXISTS {idx_fts} ON {self.table}({self.text_col}) USING FTS") + self._table_created = True + + async def add_texts( + self, + texts: Sequence[str], + metadatas: Optional[List[dict]] = None, + ids: Optional[List[str]] = None, + ) -> List[str]: + await self._ensure_table() + if not self.embedding_function: + raise ValueError("embedding_function is required for add_texts") + + inserted_ids: List[str] = [] + for i, text in enumerate(texts): + vec = self.embedding_function(text) + vec_str = "[" + ",".join(str(v) for v in vec) + "]" + + meta = metadatas[i] if metadatas and i < len(metadatas) else {} + meta_cols = [] + meta_vals = [] + if self.tenant_id: + meta_cols.append("tenant_id") + meta_vals.append(f"'{self.tenant_id}'") + for mc in self.metadata_cols: + if mc in meta: + meta_cols.append(mc) + meta_vals.append(f"'{meta[mc]}'") + + col_list = f"{self.embedding_col}, {self.text_col}" + val_list = f"'{vec_str}', '{text.replace(\"'\", \"''\")}'" + if meta_cols: + col_list += ", " + ", ".join(meta_cols) + val_list += ", " + ", ".join(meta_vals) + + sql = f"INSERT INTO {self.table} ({col_list}) VALUES ({val_list}) RETURNING id" + result = await self.client.query(sql) + if result.rows: + inserted_ids.append(result.rows[0].get("id", str(i))) + else: + inserted_ids.append(str(i)) + return inserted_ids + + async def similarity_search( + self, query: str, k: int = 4, filter_col: Optional[str] = None, filter_val: Optional[str] = None + ) -> List[Tuple[Any, float]]: + await self._ensure_table() + if not self.embedding_function: + raise ValueError("embedding_function is required for similarity_search") + + vec = self.embedding_function(query) + vec_str = "[" + ",".join(str(v) for v in vec) + "]" + + # Set tenant session variable if multi-tenant + if self.tenant_id: + await self.client.query(f"SET app.tenant_id = '{self.tenant_id}'") + + if filter_col and filter_val: + sql = f"SELECT hybrid_search_filtered('{self.table}', '{self.embedding_col}', '{self.text_col}', '{query.replace(\"'\", \"''\")}', '{vec_str}', {k}, '{filter_col}', '{filter_val}') AS res" + else: + sql = f"SELECT hybrid_search('{self.table}', '{self.embedding_col}', '{self.text_col}', '{query.replace(\"'\", \"''\")}', '{vec_str}', {k}) AS res" + + result = await self.client.query(sql) + if not result.rows: + return [] + + raw = result.rows[0].get("res", "[]") + try: + arr = json.loads(raw) + except: + return [] + + docs: List[Tuple[Any, float]] = [] + for item in arr: + doc_id = item.get("id", "") + score = float(item.get("score", 0)) + # Fetch full row + row_result = await self.client.query(f"SELECT * FROM {self.table} WHERE id = {doc_id}") + if row_result.rows: + page_content = row_result.rows[0].get(self.text_col, "") + metadata = dict(row_result.rows[0]) + # Wrap in a simple Document-like object + doc = _SimpleDocument(page_content=page_content, metadata=metadata) + docs.append((doc, score)) + return docs + + async def max_marginal_relevance_search( + self, query: str, k: int = 4, fetch_k: int = 20, lambda_mult: float = 0.5 + ) -> List[Any]: + """MMR: diversify results while maintaining relevance.""" + await self._ensure_table() + # Fetch more candidates + candidates = await self.similarity_search(query, k=fetch_k) + if not candidates: + return [] + + # Simple MMR: greedily select docs that maximize lambda*relevance - (1-lambda)*max_similarity_to_selected + selected: List[Tuple[Any, float]] = [] + remaining = list(candidates) + + while len(selected) < k and remaining: + best_score = -float("inf") + best_idx = 0 + for i, (doc, rel_score) in enumerate(remaining): + # Penalize similarity to already selected docs + penalty = 0.0 + for sel_doc, _ in selected: + penalty = max(penalty, _doc_similarity(doc, sel_doc)) + mmr_score = lambda_mult * rel_score - (1 - lambda_mult) * penalty + if mmr_score > best_score: + best_score = mmr_score + best_idx = i + selected.append(remaining.pop(best_idx)) + + return [doc for doc, _ in selected] + + async def delete(self, ids: Optional[List[str]] = None) -> None: + await self._ensure_table() + if ids: + id_list = ", ".join(str(i) for i in ids) + await self.client.query(f"DELETE FROM {self.table} WHERE id IN ({id_list})") + + async def set_tenant(self, tenant_id: str) -> None: + self.tenant_id = tenant_id + await self.client.query(f"SET app.tenant_id = '{tenant_id}'") + + +class _SimpleDocument: + def __init__(self, page_content: str, metadata: dict): + self.page_content = page_content + self.metadata = metadata + + def __repr__(self): + return f"Document(content={self.page_content[:50]}..., metadata={self.metadata})" + + +def _doc_similarity(a: _SimpleDocument, b: _SimpleDocument) -> float: + """Simple Jaccard similarity on text tokens.""" + tokens_a = set(a.page_content.lower().split()) + tokens_b = set(b.page_content.lower().split()) + if not tokens_a or not tokens_b: + return 0.0 + intersection = tokens_a & tokens_b + union = tokens_a | tokens_b + return len(intersection) / len(union)