|
@@ -318,53 +318,55 @@ def create_qdrant_indexes():
|
|
|
|
|
|
page += 1
|
|
|
for dataset in datasets:
|
|
|
- try:
|
|
|
- click.echo('Create dataset qdrant index: {}'.format(dataset.id))
|
|
|
- try:
|
|
|
- embedding_model = ModelFactory.get_embedding_model(
|
|
|
- tenant_id=dataset.tenant_id,
|
|
|
- model_provider_name=dataset.embedding_model_provider,
|
|
|
- model_name=dataset.embedding_model
|
|
|
- )
|
|
|
- except Exception:
|
|
|
- provider = Provider(
|
|
|
- id='provider_id',
|
|
|
- tenant_id=dataset.tenant_id,
|
|
|
- provider_name='openai',
|
|
|
- provider_type=ProviderType.CUSTOM.value,
|
|
|
- encrypted_config=json.dumps({'openai_api_key': 'TEST'}),
|
|
|
- is_valid=True,
|
|
|
- )
|
|
|
- model_provider = OpenAIProvider(provider=provider)
|
|
|
- embedding_model = OpenAIEmbedding(name="text-embedding-ada-002", model_provider=model_provider)
|
|
|
- embeddings = CacheEmbedding(embedding_model)
|
|
|
-
|
|
|
- from core.index.vector_index.qdrant_vector_index import QdrantVectorIndex, QdrantConfig
|
|
|
-
|
|
|
- index = QdrantVectorIndex(
|
|
|
- dataset=dataset,
|
|
|
- config=QdrantConfig(
|
|
|
- endpoint=current_app.config.get('QDRANT_URL'),
|
|
|
- api_key=current_app.config.get('QDRANT_API_KEY'),
|
|
|
- root_path=current_app.root_path
|
|
|
- ),
|
|
|
- embeddings=embeddings
|
|
|
- )
|
|
|
- if index:
|
|
|
- index_struct = {
|
|
|
- "type": 'qdrant',
|
|
|
- "vector_store": {"class_prefix": dataset.index_struct_dict['vector_store']['class_prefix']}
|
|
|
- }
|
|
|
- dataset.index_struct = json.dumps(index_struct)
|
|
|
- db.session.commit()
|
|
|
- index.create_qdrant_dataset(dataset)
|
|
|
- create_count += 1
|
|
|
- else:
|
|
|
- click.echo('passed.')
|
|
|
- except Exception as e:
|
|
|
- click.echo(
|
|
|
- click.style('Create dataset index error: {} {}'.format(e.__class__.__name__, str(e)), fg='red'))
|
|
|
- continue
|
|
|
+ if dataset.index_struct_dict:
|
|
|
+ if dataset.index_struct_dict['type'] != 'qdrant':
|
|
|
+ try:
|
|
|
+ click.echo('Create dataset qdrant index: {}'.format(dataset.id))
|
|
|
+ try:
|
|
|
+ embedding_model = ModelFactory.get_embedding_model(
|
|
|
+ tenant_id=dataset.tenant_id,
|
|
|
+ model_provider_name=dataset.embedding_model_provider,
|
|
|
+ model_name=dataset.embedding_model
|
|
|
+ )
|
|
|
+ except Exception:
|
|
|
+ provider = Provider(
|
|
|
+ id='provider_id',
|
|
|
+ tenant_id=dataset.tenant_id,
|
|
|
+ provider_name='openai',
|
|
|
+ provider_type=ProviderType.CUSTOM.value,
|
|
|
+ encrypted_config=json.dumps({'openai_api_key': 'TEST'}),
|
|
|
+ is_valid=True,
|
|
|
+ )
|
|
|
+ model_provider = OpenAIProvider(provider=provider)
|
|
|
+ embedding_model = OpenAIEmbedding(name="text-embedding-ada-002", model_provider=model_provider)
|
|
|
+ embeddings = CacheEmbedding(embedding_model)
|
|
|
+
|
|
|
+ from core.index.vector_index.qdrant_vector_index import QdrantVectorIndex, QdrantConfig
|
|
|
+
|
|
|
+ index = QdrantVectorIndex(
|
|
|
+ dataset=dataset,
|
|
|
+ config=QdrantConfig(
|
|
|
+ endpoint=current_app.config.get('QDRANT_URL'),
|
|
|
+ api_key=current_app.config.get('QDRANT_API_KEY'),
|
|
|
+ root_path=current_app.root_path
|
|
|
+ ),
|
|
|
+ embeddings=embeddings
|
|
|
+ )
|
|
|
+ if index:
|
|
|
+ index.create_qdrant_dataset(dataset)
|
|
|
+ index_struct = {
|
|
|
+ "type": 'qdrant',
|
|
|
+ "vector_store": {"class_prefix": dataset.index_struct_dict['vector_store']['class_prefix']}
|
|
|
+ }
|
|
|
+ dataset.index_struct = json.dumps(index_struct)
|
|
|
+ db.session.commit()
|
|
|
+ create_count += 1
|
|
|
+ else:
|
|
|
+ click.echo('passed.')
|
|
|
+ except Exception as e:
|
|
|
+ click.echo(
|
|
|
+ click.style('Create dataset index error: {} {}'.format(e.__class__.__name__, str(e)), fg='red'))
|
|
|
+ continue
|
|
|
|
|
|
click.echo(click.style('Congratulations! Create {} dataset indexes.'.format(create_count), fg='green'))
|
|
|
|