diff --git a/.gitignore b/.gitignore index 521bd7c96c..07494bd151 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +x.py +yarn.lock .DS_Store node_modules /build diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 9f699d9e7c..0d451a02dd 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1994,6 +1994,41 @@ PINECONE_DIMENSION = int(os.getenv("PINECONE_DIMENSION", 1536)) # or 3072, 1024 PINECONE_METRIC = os.getenv("PINECONE_METRIC", "cosine") PINECONE_CLOUD = os.getenv("PINECONE_CLOUD", "aws") # or "gcp" or "azure" +# ORACLE23AI (Oracle23ai Vector Search) + +ORACLE_DB_USE_WALLET = os.environ.get("ORACLE_DB_USE_WALLET", "false").lower() == "true" +ORACLE_DB_USER = os.environ.get("ORACLE_DB_USER", None) # +ORACLE_DB_PASSWORD = os.environ.get("ORACLE_DB_PASSWORD", None) # +ORACLE_DB_DSN = os.environ.get("ORACLE_DB_DSN", None) # +ORACLE_WALLET_DIR = os.environ.get("ORACLE_WALLET_DIR", None) +ORACLE_WALLET_PASSWORD = os.environ.get("ORACLE_WALLET_PASSWORD", None) +ORACLE_VECTOR_LENGTH = os.environ.get("ORACLE_VECTOR_LENGTH", 768) + +ORACLE_DB_POOL_MIN = int(os.environ.get("ORACLE_DB_POOL_MIN", 2)) +ORACLE_DB_POOL_MAX = int(os.environ.get("ORACLE_DB_POOL_MAX", 10)) +ORACLE_DB_POOL_INCREMENT = int(os.environ.get("ORACLE_DB_POOL_INCREMENT", 1)) + +log.info(f"VECTOR_DB: {VECTOR_DB}") +log.info(f"ORACLE_DB_USE_WALLET: {ORACLE_DB_USE_WALLET}/type: {type(ORACLE_DB_USE_WALLET)}") +log.info(f"ORACLE_DB_USER: {ORACLE_DB_USER}/type: {type(ORACLE_DB_USER)}") +log.info(f"ORACLE_DB_PASSWORD: {ORACLE_DB_PASSWORD}/type: {type(ORACLE_DB_PASSWORD)}") +log.info(f"ORACLE_DB_DSN: {ORACLE_DB_DSN}/type: {type(ORACLE_DB_DSN)}") +log.info(f"ORACLE_WALLET_DIR: {ORACLE_WALLET_DIR}/type: {type(ORACLE_WALLET_DIR)}") +log.info(f"ORACLE_WALLET_PASSWORD: {ORACLE_WALLET_PASSWORD}/type: {type(ORACLE_WALLET_PASSWORD)}") +log.info(f"ORACLE_VECTOR_LENGTH: {ORACLE_VECTOR_LENGTH}") +log.info(f"ORACLE_DB_POOL_MIN: {ORACLE_DB_POOL_MIN}") +log.info(f"ORACLE_DB_POOL_MAX: {ORACLE_DB_POOL_MAX}") +log.info(f"ORACLE_DB_POOL_INCREMENT: {ORACLE_DB_POOL_INCREMENT}") + +if VECTOR_DB == "oracle23ai" and not ORACLE_DB_USER or not ORACLE_DB_PASSWORD or not ORACLE_DB_DSN: + raise ValueError( + "Oracle23ai requires setting ORACLE_DB_USER, ORACLE_DB_PASSWORD, and ORACLE_DB_DSN." + ) +if VECTOR_DB == "oracle23ai" and ORACLE_DB_USE_WALLET and (not ORACLE_WALLET_DIR or not ORACLE_WALLET_PASSWORD): + raise ValueError( + "Oracle23ai requires setting ORACLE_WALLET_DIR and ORACLE_WALLET_PASSWORD when using wallet authentication." + ) + # S3 Vector S3_VECTOR_BUCKET_NAME = os.environ.get("S3_VECTOR_BUCKET_NAME", None) S3_VECTOR_REGION = os.environ.get("S3_VECTOR_REGION", None) diff --git a/backend/open_webui/retrieval/vector/dbs/oracle23ai.py b/backend/open_webui/retrieval/vector/dbs/oracle23ai.py new file mode 100644 index 0000000000..84e40ca6dc --- /dev/null +++ b/backend/open_webui/retrieval/vector/dbs/oracle23ai.py @@ -0,0 +1,888 @@ +""" +Oracle 23ai Vector Database Client - Fixed Version + +# .env +VECTOR_DB = "oracle23ai" + +## DBCS or oracle 23ai free +ORACLE_DB_USE_WALLET = false +ORACLE_DB_USER = "DEMOUSER" +ORACLE_DB_PASSWORD = "Welcome123456" +ORACLE_DB_DSN = "localhost:1521/FREEPDB1" + +## ADW or ATP +# ORACLE_DB_USE_WALLET = true +# ORACLE_DB_USER = "DEMOUSER" +# ORACLE_DB_PASSWORD = "Welcome123456" +# ORACLE_DB_DSN = "medium" +# ORACLE_DB_DSN = "(description= (retry_count=3)(retry_delay=3)(address=(protocol=tcps)(port=1522)(host=xx.oraclecloud.com))(connect_data=(service_name=yy.adb.oraclecloud.com))(security=(ssl_server_dn_match=no)))" +# ORACLE_WALLET_DIR = "/home/opc/adb_wallet" +# ORACLE_WALLET_PASSWORD = "Welcome1" + +ORACLE_VECTOR_LENGTH = 768 + +ORACLE_DB_POOL_MIN = 2 +ORACLE_DB_POOL_MAX = 10 +ORACLE_DB_POOL_INCREMENT = 1 +""" + +from typing import Optional, List, Dict, Any, Union +from decimal import Decimal +import logging +import os +import threading +import time +import json +import array +import oracledb + +from open_webui.retrieval.vector.main import ( + VectorDBBase, + VectorItem, + SearchResult, + GetResult, +) + +from open_webui.config import ( + ORACLE_DB_USE_WALLET, + ORACLE_DB_USER, + ORACLE_DB_PASSWORD, + ORACLE_DB_DSN, + ORACLE_WALLET_DIR, + ORACLE_WALLET_PASSWORD, + ORACLE_VECTOR_LENGTH, + ORACLE_DB_POOL_MIN, + ORACLE_DB_POOL_MAX, + ORACLE_DB_POOL_INCREMENT, +) +from open_webui.env import SRC_LOG_LEVELS + +log = logging.getLogger(__name__) +log.setLevel(SRC_LOG_LEVELS["RAG"]) + + +class Oracle23aiClient(VectorDBBase): + """ + Oracle Vector Database Client for vector similarity search using Oracle Database 23ai. + + This client provides an interface to store, retrieve, and search vector embeddings + in an Oracle database. It uses connection pooling for efficient database access + and supports vector similarity search operations. + + Attributes: + pool: Connection pool for Oracle database connections + """ + + def __init__(self) -> None: + """ + Initialize the Oracle23aiClient with a connection pool. + + Creates a connection pool with configurable min/max connections, initializes + the database schema if needed, and sets up necessary tables and indexes. + + Raises: + ValueError: If required configuration parameters are missing + Exception: If database initialization fails + """ + self.pool = None + + try: + # Create the appropriate connection pool based on DB type + if ORACLE_DB_USE_WALLET: + self._create_adb_pool() + else: # DBCS + self._create_dbcs_pool() + + dsn = ORACLE_DB_DSN + log.info(f"Creating Connection Pool [{ORACLE_DB_USER}:**@{dsn}]") + + with self.get_connection() as connection: + log.info(f"Connection version: {connection.version}") + self._initialize_database(connection) + + log.info("Oracle Vector Search initialization complete.") + except Exception as e: + log.exception(f"Error during Oracle Vector Search initialization: {e}") + raise + + def _create_adb_pool(self) -> None: + """ + Create connection pool for Oracle Autonomous Database. + + Uses wallet-based authentication. + """ + self.pool = oracledb.create_pool( + user=ORACLE_DB_USER, + password=ORACLE_DB_PASSWORD, + dsn=ORACLE_DB_DSN, + min=ORACLE_DB_POOL_MIN, + max=ORACLE_DB_POOL_MAX, + increment=ORACLE_DB_POOL_INCREMENT, + config_dir=ORACLE_WALLET_DIR, + wallet_location=ORACLE_WALLET_DIR, + wallet_password=ORACLE_WALLET_PASSWORD + ) + log.info("Created ADB connection pool with wallet authentication.") + + def _create_dbcs_pool(self) -> None: + """ + Create connection pool for Oracle Database Cloud Service. + + Uses basic authentication without wallet. + """ + self.pool = oracledb.create_pool( + user=ORACLE_DB_USER, + password=ORACLE_DB_PASSWORD, + dsn=ORACLE_DB_DSN, + min=ORACLE_DB_POOL_MIN, + max=ORACLE_DB_POOL_MAX, + increment=ORACLE_DB_POOL_INCREMENT + ) + log.info("Created DB connection pool with basic authentication.") + + def get_connection(self): + """ + Acquire a connection from the connection pool with retry logic. + + Returns: + connection: A database connection with output type handler configured + """ + max_retries = 3 + for attempt in range(max_retries): + try: + connection = self.pool.acquire() + connection.outputtypehandler = self._output_type_handler + return connection + except oracledb.DatabaseError as e: + error_obj, = e.args + log.exception(f"Connection attempt {attempt + 1} failed: {error_obj.message}") + + if attempt < max_retries - 1: + wait_time = 2 ** attempt + log.info(f"Retrying in {wait_time} seconds...") + time.sleep(wait_time) + else: + raise + + def start_health_monitor(self, interval_seconds: int = 60): + """ + Start a background thread to periodically check the health of the connection pool. + + Args: + interval_seconds (int): Number of seconds between health checks + """ + def _monitor(): + while True: + try: + log.info("[HealthCheck] Running periodic DB health check...") + self.ensure_connection() + log.info("[HealthCheck] Connection is healthy.") + except Exception as e: + log.exception(f"[HealthCheck] Connection health check failed: {e}") + time.sleep(interval_seconds) + + thread = threading.Thread(target=_monitor, daemon=True) + thread.start() + log.info(f"Started DB health monitor every {interval_seconds} seconds.") + + def _reconnect_pool(self): + """ + Attempt to reinitialize the connection pool if it's been closed or broken. + """ + try: + log.info("Attempting to reinitialize the Oracle connection pool...") + + # Close existing pool if it exists + if self.pool: + try: + self.pool.close() + except Exception as close_error: + log.warning(f"Error closing existing pool: {close_error}") + + # Re-create the appropriate connection pool based on DB type + if ORACLE_DB_USE_WALLET: + self._create_adb_pool() + else: # DBCS + self._create_dbcs_pool() + + log.info("Connection pool reinitialized.") + except Exception as e: + log.exception(f"Failed to reinitialize the connection pool: {e}") + raise + + def ensure_connection(self): + """ + Ensure the database connection is alive, reconnecting pool if needed. + """ + try: + with self.get_connection() as connection: + with connection.cursor() as cursor: + cursor.execute("SELECT 1 FROM dual") + except Exception as e: + log.exception(f"Connection check failed: {e}, attempting to reconnect pool...") + self._reconnect_pool() + + def _output_type_handler(self, cursor, metadata): + """ + Handle Oracle vector type conversion. + + Args: + cursor: Oracle database cursor + metadata: Metadata for the column + + Returns: + A variable with appropriate conversion for vector types + """ + if metadata.type_code is oracledb.DB_TYPE_VECTOR: + return cursor.var(metadata.type_code, arraysize=cursor.arraysize, + outconverter=list) + + def _initialize_database(self, connection) -> None: + """ + Initialize database schema, tables and indexes. + + Creates the document_chunk table and necessary indexes if they don't exist. + + Args: + connection: Oracle database connection + + Raises: + Exception: If schema initialization fails + """ + with connection.cursor() as cursor: + try: + log.info("Creating Table document_chunk") + cursor.execute(""" + BEGIN + EXECUTE IMMEDIATE ' + CREATE TABLE IF NOT EXISTS document_chunk ( + id VARCHAR2(255) PRIMARY KEY, + collection_name VARCHAR2(255) NOT NULL, + text CLOB, + vmetadata JSON, + vector vector(*, float32) + ) + '; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -955 THEN + RAISE; + END IF; + END; + """) + + log.info("Creating Index document_chunk_collection_name_idx") + cursor.execute(""" + BEGIN + EXECUTE IMMEDIATE ' + CREATE INDEX IF NOT EXISTS document_chunk_collection_name_idx + ON document_chunk (collection_name) + '; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -955 THEN + RAISE; + END IF; + END; + """) + + log.info("Creating VECTOR INDEX document_chunk_vector_ivf_idx") + cursor.execute(""" + BEGIN + EXECUTE IMMEDIATE ' + CREATE VECTOR INDEX IF NOT EXISTS document_chunk_vector_ivf_idx + ON document_chunk(vector) + ORGANIZATION NEIGHBOR PARTITIONS + DISTANCE COSINE + WITH TARGET ACCURACY 95 + PARAMETERS (TYPE IVF, NEIGHBOR PARTITIONS 100) + '; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -955 THEN + RAISE; + END IF; + END; + """) + + connection.commit() + log.info("Database initialization completed successfully.") + + except Exception as e: + connection.rollback() + log.exception(f"Error during database initialization: {e}") + raise + + def check_vector_length(self) -> None: + """ + Check vector length compatibility (placeholder). + + This method would check if the configured vector length matches the database schema. + Currently implemented as a placeholder. + """ + pass + + def _vector_to_blob(self, vector: List[float]) -> bytes: + """ + Convert a vector to Oracle BLOB format. + + Args: + vector (List[float]): The vector to convert + + Returns: + bytes: The vector in Oracle BLOB format + """ + return array.array("f", vector) + + def adjust_vector_length(self, vector: List[float]) -> List[float]: + """ + Adjust vector to the expected length if needed. + + Args: + vector (List[float]): The vector to adjust + + Returns: + List[float]: The adjusted vector + """ + return vector + + def _decimal_handler(self, obj): + """ + Handle Decimal objects for JSON serialization. + + Args: + obj: Object to serialize + + Returns: + float: Converted decimal value + + Raises: + TypeError: If object is not JSON serializable + """ + if isinstance(obj, Decimal): + return float(obj) + raise TypeError(f"{obj} is not JSON serializable") + + def _metadata_to_json(self, metadata: Dict) -> str: + """ + Convert metadata dictionary to JSON string. + + Args: + metadata (Dict): Metadata dictionary + + Returns: + str: JSON representation of metadata + """ + return json.dumps(metadata, default=self._decimal_handler) if metadata else "{}" + + def _json_to_metadata(self, json_str: str) -> Dict: + """ + Convert JSON string to metadata dictionary. + + Args: + json_str (str): JSON string + + Returns: + Dict: Metadata dictionary + """ + return json.loads(json_str) if json_str else {} + + def insert(self, collection_name: str, items: List[VectorItem]) -> None: + """ + Insert vector items into the database. + + Args: + collection_name (str): Name of the collection + items (List[VectorItem]): List of vector items to insert + + Raises: + Exception: If insertion fails + + Example: + >>> client = Oracle23aiClient() + >>> items = [ + ... {"id": "1", "text": "Sample text", "vector": [0.1, 0.2, ...], "metadata": {"source": "doc1"}}, + ... {"id": "2", "text": "Another text", "vector": [0.3, 0.4, ...], "metadata": {"source": "doc2"}} + ... ] + >>> client.insert("my_collection", items) + """ + log.info(f"Inserting {len(items)} items into collection '{collection_name}'.") + + with self.get_connection() as connection: + try: + with connection.cursor() as cursor: + for item in items: + vector_blob = self._vector_to_blob(item["vector"]) + metadata_json = self._metadata_to_json(item["metadata"]) + + cursor.execute(""" + INSERT INTO document_chunk + (id, collection_name, text, vmetadata, vector) + VALUES (:id, :collection_name, :text, :metadata, :vector) + """, { + 'id': item["id"], + 'collection_name': collection_name, + 'text': item["text"], + 'metadata': metadata_json, + 'vector': vector_blob + }) + + connection.commit() + log.info(f"Successfully inserted {len(items)} items into collection '{collection_name}'.") + + except Exception as e: + connection.rollback() + log.exception(f"Error during insert: {e}") + raise + + def upsert(self, collection_name: str, items: List[VectorItem]) -> None: + """ + Update or insert vector items into the database. + + If an item with the same ID exists, it will be updated; + otherwise, it will be inserted. + + Args: + collection_name (str): Name of the collection + items (List[VectorItem]): List of vector items to upsert + + Raises: + Exception: If upsert operation fails + + Example: + >>> client = Oracle23aiClient() + >>> items = [ + ... {"id": "1", "text": "Updated text", "vector": [0.1, 0.2, ...], "metadata": {"source": "doc1"}}, + ... {"id": "3", "text": "New item", "vector": [0.5, 0.6, ...], "metadata": {"source": "doc3"}} + ... ] + >>> client.upsert("my_collection", items) + """ + log.info(f"Upserting {len(items)} items into collection '{collection_name}'.") + + with self.get_connection() as connection: + try: + with connection.cursor() as cursor: + for item in items: + vector_blob = self._vector_to_blob(item["vector"]) + metadata_json = self._metadata_to_json(item["metadata"]) + + cursor.execute(""" + MERGE INTO document_chunk d + USING (SELECT :merge_id as id FROM dual) s + ON (d.id = s.id) + WHEN MATCHED THEN + UPDATE SET + collection_name = :upd_collection_name, + text = :upd_text, + vmetadata = :upd_metadata, + vector = :upd_vector + WHEN NOT MATCHED THEN + INSERT (id, collection_name, text, vmetadata, vector) + VALUES (:ins_id, :ins_collection_name, :ins_text, :ins_metadata, :ins_vector) + """, { + 'merge_id': item["id"], + 'upd_collection_name': collection_name, + 'upd_text': item["text"], + 'upd_metadata': metadata_json, + 'upd_vector': vector_blob, + 'ins_id': item["id"], + 'ins_collection_name': collection_name, + 'ins_text': item["text"], + 'ins_metadata': metadata_json, + 'ins_vector': vector_blob + }) + + connection.commit() + log.info(f"Successfully upserted {len(items)} items into collection '{collection_name}'.") + + except Exception as e: + connection.rollback() + log.exception(f"Error during upsert: {e}") + raise + + def search( + self, + collection_name: str, + vectors: List[List[Union[float, int]]], + limit: int + ) -> Optional[SearchResult]: + """ + Search for similar vectors in the database. + + Performs vector similarity search using cosine distance. + + Args: + collection_name (str): Name of the collection to search + vectors (List[List[Union[float, int]]]): Query vectors to find similar items for + limit (int): Maximum number of results to return per query + + Returns: + Optional[SearchResult]: Search results containing ids, distances, documents, and metadata + + Example: + >>> client = Oracle23aiClient() + >>> query_vector = [0.1, 0.2, 0.3, ...] # Must match VECTOR_LENGTH + >>> results = client.search("my_collection", [query_vector], limit=5) + >>> if results: + ... log.info(f"Found {len(results.ids[0])} matches") + ... for i, (id, dist) in enumerate(zip(results.ids[0], results.distances[0])): + ... log.info(f"Match {i+1}: id={id}, distance={dist}") + """ + log.info(f"Searching items from collection '{collection_name}' with limit {limit}.") + + try: + if not vectors: + log.warning("No vectors provided for search.") + return None + + num_queries = len(vectors) + + ids = [[] for _ in range(num_queries)] + distances = [[] for _ in range(num_queries)] + documents = [[] for _ in range(num_queries)] + metadatas = [[] for _ in range(num_queries)] + + with self.get_connection() as connection: + with connection.cursor() as cursor: + for qid, vector in enumerate(vectors): + vector_blob = self._vector_to_blob(vector) + + cursor.execute(""" + SELECT dc.id, dc.text, + JSON_SERIALIZE(dc.vmetadata RETURNING VARCHAR2(4096)) as vmetadata, + VECTOR_DISTANCE(dc.vector, :query_vector, COSINE) as distance + FROM document_chunk dc + WHERE dc.collection_name = :collection_name + ORDER BY VECTOR_DISTANCE(dc.vector, :query_vector, COSINE) + FETCH APPROX FIRST :limit ROWS ONLY + """, { + 'query_vector': vector_blob, + 'collection_name': collection_name, + 'limit': limit + }) + + results = cursor.fetchall() + + for row in results: + ids[qid].append(row[0]) + documents[qid].append(row[1].read() if isinstance(row[1], oracledb.LOB) else str(row[1])) + # 🔧 FIXED: Parse JSON metadata properly + metadata_str = row[2].read() if isinstance(row[2], oracledb.LOB) else row[2] + metadatas[qid].append(self._json_to_metadata(metadata_str)) + distances[qid].append(float(row[3])) + + log.info(f"Search completed. Found {sum(len(ids[i]) for i in range(num_queries))} total results.") + + return SearchResult( + ids=ids, + distances=distances, + documents=documents, + metadatas=metadatas + ) + + except Exception as e: + log.exception(f"Error during search: {e}") + return None + + def query( + self, + collection_name: str, + filter: Dict, + limit: Optional[int] = None + ) -> Optional[GetResult]: + """ + Query items based on metadata filters. + + Retrieves items that match specified metadata criteria. + + Args: + collection_name (str): Name of the collection to query + filter (Dict[str, Any]): Metadata filters to apply + limit (Optional[int]): Maximum number of results to return + + Returns: + Optional[GetResult]: Query results containing ids, documents, and metadata + + Example: + >>> client = Oracle23aiClient() + >>> filter = {"source": "doc1", "category": "finance"} + >>> results = client.query("my_collection", filter, limit=20) + >>> if results: + ... print(f"Found {len(results.ids[0])} matching documents") + """ + log.info(f"Querying items from collection '{collection_name}' with filters.") + + try: + limit = limit or 100 + + query = """ + SELECT id, text, JSON_SERIALIZE(vmetadata RETURNING VARCHAR2(4096)) as vmetadata + FROM document_chunk + WHERE collection_name = :collection_name + """ + + params = {'collection_name': collection_name} + + for i, (key, value) in enumerate(filter.items()): + param_name = f"value_{i}" + query += f" AND JSON_VALUE(vmetadata, '$.{key}' RETURNING VARCHAR2(4096)) = :{param_name}" + params[param_name] = str(value) + + query += " FETCH FIRST :limit ROWS ONLY" + params['limit'] = limit + + with self.get_connection() as connection: + with connection.cursor() as cursor: + cursor.execute(query, params) + results = cursor.fetchall() + + if not results: + log.info("No results found for query.") + return None + + ids = [[row[0] for row in results]] + documents = [[row[1].read() if isinstance(row[1], oracledb.LOB) else str(row[1]) for row in results]] + # 🔧 FIXED: Parse JSON metadata properly + metadatas = [[self._json_to_metadata(row[2].read() if isinstance(row[2], oracledb.LOB) else row[2]) for row in results]] + + log.info(f"Query completed. Found {len(results)} results.") + + return GetResult( + ids=ids, + documents=documents, + metadatas=metadatas + ) + + except Exception as e: + log.exception(f"Error during query: {e}") + return None + + def get( + self, + collection_name: str + ) -> Optional[GetResult]: + """ + Get all items in a collection. + + Retrieves items from a specified collection up to the limit. + + Args: + collection_name (str): Name of the collection to retrieve + limit (Optional[int]): Maximum number of items to retrieve + + Returns: + Optional[GetResult]: Result containing ids, documents, and metadata + + Example: + >>> client = Oracle23aiClient() + >>> results = client.get("my_collection", limit=50) + >>> if results: + ... print(f"Retrieved {len(results.ids[0])} documents from collection") + """ + log.info(f"Getting items from collection '{collection_name}' with limit {limit}.") + + try: + limit = limit or 1000 + + with self.get_connection() as connection: + with connection.cursor() as cursor: + cursor.execute(""" + SELECT /*+ MONITOR */ id, text, JSON_SERIALIZE(vmetadata RETURNING VARCHAR2(4096)) as vmetadata + FROM document_chunk + WHERE collection_name = :collection_name + FETCH FIRST :limit ROWS ONLY + """, { + 'collection_name': collection_name, + 'limit': limit + }) + + results = cursor.fetchall() + + if not results: + log.info("No results found.") + return None + + ids = [[row[0] for row in results]] + documents = [[row[1].read() if isinstance(row[1], oracledb.LOB) else str(row[1]) for row in results]] + # 🔧 FIXED: Parse JSON metadata properly + metadatas = [[self._json_to_metadata(row[2].read() if isinstance(row[2], oracledb.LOB) else row[2]) for row in results]] + + return GetResult( + ids=ids, + documents=documents, + metadatas=metadatas + ) + + except Exception as e: + log.exception(f"Error during get: {e}") + return None + + def delete( + self, + collection_name: str, + ids: Optional[List[str]] = None, + filter: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Delete items from the database. + + Deletes items from a collection based on IDs or metadata filters. + + Args: + collection_name (str): Name of the collection to delete from + ids (Optional[List[str]]): Specific item IDs to delete + filter (Optional[Dict[str, Any]]): Metadata filters for deletion + + Raises: + Exception: If deletion fails + + Example: + >>> client = Oracle23aiClient() + >>> # Delete specific items by ID + >>> client.delete("my_collection", ids=["1", "3", "5"]) + >>> # Or delete by metadata filter + >>> client.delete("my_collection", filter={"source": "deprecated_source"}) + """ + log.info(f"Deleting items from collection '{collection_name}'.") + + try: + query = "DELETE FROM document_chunk WHERE collection_name = :collection_name" + params = {'collection_name': collection_name} + + if ids: + # 🔧 FIXED: Use proper parameterized query to prevent SQL injection + placeholders = ','.join([f':id_{i}' for i in range(len(ids))]) + query += f" AND id IN ({placeholders})" + for i, id_val in enumerate(ids): + params[f'id_{i}'] = id_val + + if filter: + for i, (key, value) in enumerate(filter.items()): + param_name = f"value_{i}" + query += f" AND JSON_VALUE(vmetadata, '$.{key}' RETURNING VARCHAR2(4096)) = :{param_name}" + params[param_name] = str(value) + + with self.get_connection() as connection: + with connection.cursor() as cursor: + cursor.execute(query, params) + deleted = cursor.rowcount + connection.commit() + + log.info(f"Deleted {deleted} items from collection '{collection_name}'.") + + except Exception as e: + log.exception(f"Error during delete: {e}") + raise + + def reset(self) -> None: + """ + Reset the database by deleting all items. + + Deletes all items from the document_chunk table. + + Raises: + Exception: If reset fails + + Example: + >>> client = Oracle23aiClient() + >>> client.reset() # Warning: Removes all data! + """ + log.info("Resetting database - deleting all items.") + + try: + with self.get_connection() as connection: + with connection.cursor() as cursor: + cursor.execute("DELETE FROM document_chunk") + deleted = cursor.rowcount + connection.commit() + + log.info(f"Reset complete. Deleted {deleted} items from 'document_chunk' table.") + + except Exception as e: + log.exception(f"Error during reset: {e}") + raise + + def close(self) -> None: + """ + Close the database connection pool. + + Properly closes the connection pool and releases all resources. + + Example: + >>> client = Oracle23aiClient() + >>> # After finishing all operations + >>> client.close() + """ + try: + if hasattr(self, 'pool') and self.pool: + self.pool.close() + log.info("Oracle Vector Search connection pool closed.") + except Exception as e: + log.exception(f"Error closing connection pool: {e}") + + def has_collection(self, collection_name: str) -> bool: + """ + Check if a collection exists. + + Args: + collection_name (str): Name of the collection to check + + Returns: + bool: True if the collection exists, False otherwise + + Example: + >>> client = Oracle23aiClient() + >>> if client.has_collection("my_collection"): + ... print("Collection exists!") + ... else: + ... print("Collection does not exist.") + """ + try: + with self.get_connection() as connection: + with connection.cursor() as cursor: + cursor.execute(""" + SELECT COUNT(*) + FROM document_chunk + WHERE collection_name = :collection_name + FETCH FIRST 1 ROWS ONLY + """, {'collection_name': collection_name}) + + count = cursor.fetchone()[0] + + return count > 0 + + except Exception as e: + log.exception(f"Error checking collection existence: {e}") + return False + + def delete_collection(self, collection_name: str) -> None: + """ + Delete an entire collection. + + Removes all items belonging to the specified collection. + + Args: + collection_name (str): Name of the collection to delete + + Example: + >>> client = Oracle23aiClient() + >>> client.delete_collection("obsolete_collection") + """ + log.info(f"Deleting collection '{collection_name}'.") + + try: + with self.get_connection() as connection: + with connection.cursor() as cursor: + cursor.execute(""" + DELETE FROM document_chunk + WHERE collection_name = :collection_name + """, {'collection_name': collection_name}) + + deleted = cursor.rowcount + connection.commit() + + log.info(f"Collection '{collection_name}' deleted. Removed {deleted} items.") + + except Exception as e: + log.exception(f"Error deleting collection '{collection_name}': {e}") + raise \ No newline at end of file diff --git a/backend/open_webui/retrieval/vector/factory.py b/backend/open_webui/retrieval/vector/factory.py index e90ae4f322..36cb85c948 100644 --- a/backend/open_webui/retrieval/vector/factory.py +++ b/backend/open_webui/retrieval/vector/factory.py @@ -52,6 +52,10 @@ class Vector: from open_webui.retrieval.vector.dbs.chroma import ChromaClient return ChromaClient() + case VectorType.ORACLE23AI: + from open_webui.retrieval.vector.dbs.oracle23ai import Oracle23aiClient + + return Oracle23aiClient() case _: raise ValueError(f"Unsupported vector type: {vector_type}") diff --git a/backend/open_webui/retrieval/vector/type.py b/backend/open_webui/retrieval/vector/type.py index 481c36902e..7e517c169c 100644 --- a/backend/open_webui/retrieval/vector/type.py +++ b/backend/open_webui/retrieval/vector/type.py @@ -9,4 +9,5 @@ class VectorType(StrEnum): ELASTICSEARCH = "elasticsearch" OPENSEARCH = "opensearch" PGVECTOR = "pgvector" + ORACLE23AI = "oracle23ai" S3VECTOR = "s3vector" diff --git a/backend/requirements.txt b/backend/requirements.txt index 4c9bd85ead..adea571a0c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -58,6 +58,7 @@ opensearch-py==2.8.0 playwright==1.49.1 # Caution: version must match docker-compose.playwright.yaml elasticsearch==9.0.1 pinecone==6.0.2 +oracledb==3.2.0 transformers sentence-transformers==4.1.0 diff --git a/package-lock.json b/package-lock.json index c8b5e78998..b4799fe40d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1326,10 +1326,11 @@ } }, "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1429,10 +1430,11 @@ } }, "node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2858,6 +2860,15 @@ "svelte": "^3.55.0 || ^4.0.0 || ^5.0.0" } }, + "node_modules/@sveltejs/acorn-typescript": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.5.tgz", + "integrity": "sha512-IwQk4yfwLdibDlrXVE04jTZYlLnwsTT2PIOQQGNLWfjavGifnk1JD1LcZjZaBTRcxZu2FfPfNLOE04DSu9lqtQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^8.9.0" + } + }, "node_modules/@sveltejs/adapter-auto": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/@sveltejs/adapter-auto/-/adapter-auto-3.2.2.tgz", @@ -2895,16 +2906,17 @@ } }, "node_modules/@sveltejs/kit": { - "version": "2.20.2", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.20.2.tgz", - "integrity": "sha512-Dv8TOAZC9vyfcAB9TMsvUEJsRbklRTeNfcYBPaeH6KnABJ99i3CvCB2eNx8fiiliIqe+9GIchBg4RodRH5p1BQ==", + "version": "2.22.4", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.22.4.tgz", + "integrity": "sha512-BXK9hTbP8AeQIfoz6+P3uoyVYStVHc5CIKqoTSF7hXm3Q5P9BwFMdEus4jsQuhaYmXGHzukcGlxe2QrsE8BJfQ==", "license": "MIT", "dependencies": { + "@sveltejs/acorn-typescript": "^1.0.5", "@types/cookie": "^0.6.0", + "acorn": "^8.14.1", "cookie": "^0.6.0", "devalue": "^5.1.0", "esm-env": "^1.2.2", - "import-meta-resolve": "^4.1.0", "kleur": "^4.1.5", "magic-string": "^0.30.5", "mrmime": "^2.0.0", @@ -2919,9 +2931,9 @@ "node": ">=18.13" }, "peerDependencies": { - "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0", + "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0 || ^6.0.0-next.0", "svelte": "^4.0.0 || ^5.0.0-next.0", - "vite": "^5.0.3 || ^6.0.0" + "vite": "^5.0.3 || ^6.0.0 || ^7.0.0-beta.0" } }, "node_modules/@sveltejs/svelte-virtual-list": { @@ -4965,9 +4977,10 @@ "license": "ISC" }, "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -7094,10 +7107,11 @@ } }, "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -8210,6 +8224,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", + "dev": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -9389,10 +9404,11 @@ } }, "node_modules/matcher-collection/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -10848,10 +10864,11 @@ } }, "node_modules/quick-temp/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -11076,10 +11093,11 @@ } }, "node_modules/rimraf/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -11264,10 +11282,11 @@ } }, "node_modules/sander/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -12785,9 +12804,9 @@ } }, "node_modules/undici": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.3.0.tgz", - "integrity": "sha512-Qy96NND4Dou5jKoSJ2gm8ax8AJM/Ey9o9mz7KN1bb9GP+G0l20Zw8afxTnY2f4b7hmhn/z8aC2kfArVQlAhFBw==", + "version": "7.11.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.11.0.tgz", + "integrity": "sha512-heTSIac3iLhsmZhUCjyS3JQEkZELateufzZuBaVM5RHXdSBMb1LPMQf5x+FH7qjsZYDP0ttAc3nnVpUB+wYbOg==", "license": "MIT", "engines": { "node": ">=20.18.1" @@ -12969,9 +12988,9 @@ } }, "node_modules/vite": { - "version": "5.4.15", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.15.tgz", - "integrity": "sha512-6ANcZRivqL/4WtwPGTKNaosuNJr5tWiftOC7liM7G9+rMb8+oeJeyzymDu4rTN93seySBmbjSfsS3Vzr19KNtA==", + "version": "5.4.19", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.19.tgz", + "integrity": "sha512-qO3aKv3HoQC8QKiNSTuUM1l9o/XX3+c+VTgLHbJWHZGeTPVAg2XwazI9UWzoxjIJCGCV2zU60uqMzjeLZuULqA==", "license": "MIT", "dependencies": { "esbuild": "^0.21.3", @@ -13736,10 +13755,11 @@ } }, "node_modules/walk-sync/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" diff --git a/pyproject.toml b/pyproject.toml index 5812a6b29a..e56a0de1e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,7 +135,7 @@ dependencies = [ "gcp-storage-emulator>=2024.8.3", "moto[s3]>=5.0.26", - + "oracledb>=3.2.0", "posthog==5.4.0", ] diff --git a/uv.lock b/uv.lock index e21dc579d1..7bde0eeb01 100644 --- a/uv.lock +++ b/uv.lock @@ -969,6 +969,20 @@ version = "0.8" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7d/7d/60ee3f2b16d9bfdfa72e8599470a2c1a5b759cb113c6fe1006be28359327/docx2txt-0.8.tar.gz", hash = "sha256:2c06d98d7cfe2d3947e5760a57d924e3ff07745b379c8737723922e7009236e5", size = 2814, upload-time = "2019-06-23T19:58:36.94Z" } +[[package]] +name = "duckduckgo-search" +version = "8.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "lxml" }, + { name = "primp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/c0/e18c2148d33a9d87f6a0cc00acba30b4e547be0f8cb85ccb313a6e8fbac7/duckduckgo_search-8.0.2.tar.gz", hash = "sha256:3109a99967b29cab8862823bbe320d140d5c792415de851b9d6288de2311b3ec", size = 21807, upload-time = "2025-05-15T08:43:25.311Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/6c/e36d22e76f4aa4e1ea7ea9b443bd49b5ffd2f13d430840f47e35284f797a/duckduckgo_search-8.0.2-py3-none-any.whl", hash = "sha256:b5ff8b6b8f169b8e1b15a788a5749aa900ebcefd6e1ab485787582f8d5b4f1ef", size = 18184, upload-time = "2025-05-15T08:43:23.713Z" }, +] + [[package]] name = "durationpy" version = "0.9" @@ -1044,6 +1058,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "eval-type-backport" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/ea/8b0ac4469d4c347c6a385ff09dc3c048c2d021696664e26c7ee6791631b5/eval_type_backport-0.2.2.tar.gz", hash = "sha256:f0576b4cf01ebb5bd358d02314d31846af5e07678387486e2c798af0e7d849c1", size = 9079, upload-time = "2024-12-21T20:09:46.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/31/55cd413eaccd39125368be33c46de24a1f639f2e12349b0361b4678f3915/eval_type_backport-0.2.2-py3-none-any.whl", hash = "sha256:cb6ad7c393517f476f96d456d0412ea80f0a8cf96f6892834cd9340149111b0a", size = 5830, upload-time = "2024-12-21T20:09:44.175Z" }, +] + [[package]] name = "events" version = "0.5" @@ -1585,6 +1608,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/90/40/972271de05f9315c0d69f9f7ebbcadd83bc85322f538637d11bb8c67803d/grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8", size = 14448, upload-time = "2024-08-06T00:30:15.702Z" }, ] +[[package]] +name = "grpcio-tools" +version = "1.62.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/fa/b69bd8040eafc09b88bb0ec0fea59e8aacd1a801e688af087cead213b0d0/grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833", size = 4538520, upload-time = "2024-08-06T00:37:11.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/52/2dfe0a46b63f5ebcd976570aa5fc62f793d5a8b169e211c6a5aede72b7ae/grpcio_tools-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23", size = 5147623, upload-time = "2024-08-06T00:30:54.894Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2e/29fdc6c034e058482e054b4a3c2432f84ff2e2765c1342d4f0aa8a5c5b9a/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492", size = 2719538, upload-time = "2024-08-06T00:30:57.928Z" }, + { url = "https://files.pythonhosted.org/packages/f9/60/abe5deba32d9ec2c76cdf1a2f34e404c50787074a2fee6169568986273f1/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7", size = 3070964, upload-time = "2024-08-06T00:31:00.267Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ad/e2b066684c75f8d9a48508cde080a3a36618064b9cadac16d019ca511444/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43", size = 2805003, upload-time = "2024-08-06T00:31:02.565Z" }, + { url = "https://files.pythonhosted.org/packages/9c/3f/59bf7af786eae3f9d24ee05ce75318b87f541d0950190ecb5ffb776a1a58/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a", size = 3685154, upload-time = "2024-08-06T00:31:05.339Z" }, + { url = "https://files.pythonhosted.org/packages/f1/79/4dd62478b91e27084c67b35a2316ce8a967bd8b6cb8d6ed6c86c3a0df7cb/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3", size = 3297942, upload-time = "2024-08-06T00:31:08.456Z" }, + { url = "https://files.pythonhosted.org/packages/b8/cb/86449ecc58bea056b52c0b891f26977afc8c4464d88c738f9648da941a75/grpcio_tools-1.62.3-cp311-cp311-win32.whl", hash = "sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5", size = 910231, upload-time = "2024-08-06T00:31:11.464Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/9736215e3945c30ab6843280b0c6e1bff502910156ea2414cd77fbf1738c/grpcio_tools-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f", size = 1052496, upload-time = "2024-08-06T00:31:13.665Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a5/d6887eba415ce318ae5005e8dfac3fa74892400b54b6d37b79e8b4f14f5e/grpcio_tools-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5", size = 5147690, upload-time = "2024-08-06T00:31:16.436Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7c/3cde447a045e83ceb4b570af8afe67ffc86896a2fe7f59594dc8e5d0a645/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133", size = 2720538, upload-time = "2024-08-06T00:31:18.905Z" }, + { url = "https://files.pythonhosted.org/packages/88/07/f83f2750d44ac4f06c07c37395b9c1383ef5c994745f73c6bfaf767f0944/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa", size = 3071571, upload-time = "2024-08-06T00:31:21.684Z" }, + { url = "https://files.pythonhosted.org/packages/37/74/40175897deb61e54aca716bc2e8919155b48f33aafec8043dda9592d8768/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0", size = 2806207, upload-time = "2024-08-06T00:31:24.208Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ee/d8de915105a217cbcb9084d684abdc032030dcd887277f2ef167372287fe/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d", size = 3685815, upload-time = "2024-08-06T00:31:26.917Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d9/4360a6c12be3d7521b0b8c39e5d3801d622fbb81cc2721dbd3eee31e28c8/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc", size = 3298378, upload-time = "2024-08-06T00:31:30.401Z" }, + { url = "https://files.pythonhosted.org/packages/29/3b/7cdf4a9e5a3e0a35a528b48b111355cd14da601413a4f887aa99b6da468f/grpcio_tools-1.62.3-cp312-cp312-win32.whl", hash = "sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b", size = 910416, upload-time = "2024-08-06T00:31:33.118Z" }, + { url = "https://files.pythonhosted.org/packages/6c/66/dd3ec249e44c1cc15e902e783747819ed41ead1336fcba72bf841f72c6e9/grpcio_tools-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7", size = 1052856, upload-time = "2024-08-06T00:31:36.519Z" }, +] + [[package]] name = "h11" version = "0.14.0" @@ -2665,6 +2717,7 @@ dependencies = [ { name = "opencv-python-headless" }, { name = "openpyxl" }, { name = "opensearch-py" }, + { name = "oracledb" }, { name = "pandas" }, { name = "passlib", extra = ["bcrypt"] }, { name = "peewee" }, @@ -2744,6 +2797,7 @@ requires-dist = [ { name = "ddgs", specifier = "==9.0.0" }, { name = "docker", specifier = "~=7.1.0" }, { name = "docx2txt", specifier = "==0.8" }, + { name = "duckduckgo-search", specifier = "==8.0.2" }, { name = "einops", specifier = "==0.8.1" }, { name = "elasticsearch", specifier = "==9.0.1" }, { name = "fake-useragent", specifier = "==2.1.0" }, @@ -2774,6 +2828,7 @@ requires-dist = [ { name = "opencv-python-headless", specifier = "==4.11.0.86" }, { name = "openpyxl", specifier = "==3.1.5" }, { name = "opensearch-py", specifier = "==2.8.0" }, + { name = "oracledb", specifier = ">=3.2.0" }, { name = "pandas", specifier = "==2.2.3" }, { name = "passlib", extras = ["bcrypt"], specifier = "==1.7.4" }, { name = "peewee", specifier = "==3.18.1" }, @@ -3032,6 +3087,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/8a/9e1b54f50d1fddebbeac9a9b0632f8db6ece7add904fb593ee2e268ee4de/opentelemetry_util_http-0.50b0-py3-none-any.whl", hash = "sha256:21f8aedac861ffa3b850f8c0a6c373026189eb8630ac6e14a2bf8c55695cc090", size = 6942, upload-time = "2024-12-11T17:05:13.342Z" }, ] +[[package]] +name = "oracledb" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/2d/8336527f248dbeb7183b7b4bfafe3499119c467236f8916ce0ec3b8ef2b6/oracledb-3.2.0.tar.gz", hash = "sha256:9bf9f1c93e53142b33d1c5ebf5ababeebd2062a01d5ead68bbb640439ecf2223", size = 872574, upload-time = "2025-06-26T21:57:13.001Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/02/8279be85defe274a5dbc565c27f927d7d772117c070008853d75d4d780a8/oracledb-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:42e936190c5c76b115e2803e9c51a416a895c31ca551bc154b8c52121a84ee78", size = 3985060, upload-time = "2025-06-26T21:57:30.753Z" }, + { url = "https://files.pythonhosted.org/packages/38/a2/ce33fca6a15b3357d00c92bffcfa614b2d9f008d8e997f57e0302c760fb6/oracledb-3.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a87a331df19a84cad3d50a99cd505f8d0ec500e5e22438a025bd66e037b20715", size = 2507408, upload-time = "2025-06-26T21:57:33.34Z" }, + { url = "https://files.pythonhosted.org/packages/17/1b/dcaeb174ca777f28e05ac3d862cb468ce4b0a9c36900aa5f12ecbf29029f/oracledb-3.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b62f0c018e437b91254f019715a5fd5daafcf3182afee86ccd9d0f3dac75c0a7", size = 2687910, upload-time = "2025-06-26T21:57:36.173Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f6/a575191c027b89486947597a7b6ae9c2abbc05eae2b79de037a6039ae455/oracledb-3.2.0-cp311-cp311-win32.whl", hash = "sha256:01477e65f129f8927ef64cd4b48d68646fc0115e1e95484dac5b82dd9da7b98b", size = 1548061, upload-time = "2025-06-26T21:57:38.273Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ec/a9b5488858d3b45213ce03f442c5cdf5aef356269071b84caecadb326e29/oracledb-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:a1dc529f06a95ca3560d233609c266c17e828a08a70b8a434e2c3fe7000825eb", size = 1891446, upload-time = "2025-06-26T21:57:40.737Z" }, + { url = "https://files.pythonhosted.org/packages/56/d1/04632c2fab7c9ab91c68630eb221e17019e74d5b023badc0c191e83119cf/oracledb-3.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1c2658983520b460776e74c75bb50e51a78e8ab743b64adc844a26a3a8a0bc7c", size = 3959613, upload-time = "2025-06-26T21:57:43.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/de/ebb82b6193583d0c7f13f908756d44bd2c03207b501457773b0793018231/oracledb-3.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c7a599ffe3238824951948992ab6b4532a0c1d4b33900d412f738a7da476d47", size = 2336800, upload-time = "2025-06-26T21:57:45.997Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f5/f81f72fac3cfb52fc18965d4d07d76e26103c2cc60641a796b9618904b54/oracledb-3.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9388ad0e09dc4030cd62779acc0ae4e9dfd338da7d30c72768fb0589461485ff", size = 2509427, upload-time = "2025-06-26T21:57:48.686Z" }, + { url = "https://files.pythonhosted.org/packages/02/ad/ded50edb5010b27173f203421f69f90e5f00acaf5686ccd4f7963e1c695b/oracledb-3.2.0-cp312-cp312-win32.whl", hash = "sha256:94ac95e52e6f4a9394408aba6cc5f90581219ecb872d450b2a80df9aa3cc4216", size = 1511775, upload-time = "2025-06-26T21:57:50.474Z" }, + { url = "https://files.pythonhosted.org/packages/5f/75/684d2e18d57d72abf85366e8e1f61aaa2e6b15c71bf95f5bb6e9d7e0c9a5/oracledb-3.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:a367604d091ee82c9e3f6d97e34b8ec919646209bda78337f6156b76252dd024", size = 1850296, upload-time = "2025-06-26T21:57:52.649Z" }, +] + [[package]] name = "orjson" version = "3.10.14"