Skip to content

Commit b861d67

Browse files
diberrydiberry
committed
dev container, outputs, db & container name, docs already inserted, suppress Mongo warning
1 parent c1de893 commit b861d67

11 files changed

Lines changed: 165 additions & 90 deletions

File tree

‎.devcontainer/python/devcontainer.json‎

Lines changed: 9 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,24 @@
1-
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
2-
// README at: https://github.com/devcontainers/templates/tree/main/src/universal
31
{
4-
"name": "Default Python",
5-
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
6-
"image": "mcr.microsoft.com/devcontainers/python:latest",
2+
"name": "Python",
3+
"image": "python:3.12-bookworm",
74
"features": {
85
"ghcr.io/devcontainers/features/azure-cli:1": {},
9-
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
106
"ghcr.io/azure/azure-dev/azd:0": {}
117
},
8+
"remoteUser": "vscode",
129
"customizations": {
1310
"vscode": {
1411
"extensions": [
12+
"ms-python.python",
1513
"ms-azuretools.vscode-cosmosdb",
1614
"buildwithlayer.mongodb-integration-expert-qS6DB",
1715
"mongodb.mongodb-vscode",
1816
"ms-azuretools.vscode-documentdb"
1917
]
20-
}
18+
},
19+
"extensions": [
20+
"ms-python.python",
21+
"ms-azuretools.vscode-bicep"
22+
]
2123
}
22-
23-
// Features to add to the dev container. More info: https://containers.dev/features.
24-
// "features": {},
25-
26-
// Use 'forwardPorts' to make a list of ports inside the container available locally.
27-
// "forwardPorts": [],
28-
29-
// Use 'postCreateCommand' to run commands after the container is created.
30-
// "postCreateCommand": "uname -a",
31-
32-
// Configure tool-specific properties.
33-
// "customizations": {},
34-
35-
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
36-
// "remoteUser": "root"
3724
}

‎ai/vector-search-python/.env.example‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ DEBUG=false
66
AZURE_OPENAI_EMBEDDING_MODEL=text-embedding-3-small
77

88
# Azure OpenAI API version for embeddings
9-
AZURE_OPENAI_EMBEDDING_API_VERSION=2024-02-01
9+
AZURE_OPENAI_EMBEDDING_API_VERSION=2023-05-15
1010

1111
# Azure OpenAI service endpoint URL
1212
AZURE_OPENAI_EMBEDDING_ENDPOINT=https://your-openai-resource.openai.azure.com/
@@ -24,8 +24,6 @@ DATA_FILE_WITHOUT_VECTORS=../data/Hotels.json
2424
# Path to JSON file with generated vector embeddings
2525
DATA_FILE_WITH_VECTORS=../data/Hotels_Vector.json
2626

27-
# Path to JSON file with vector similarity data (optional)
28-
DATA_FILE_WITH_SIMILARITY=../data/Hotels_with_similarity.json
2927
# Path to query file with vectors (optional)
3028
QUERY_FILE_WITH_VECTORS=data/query_vectors.json
3129

@@ -49,4 +47,4 @@ LOAD_SIZE_BATCH=100
4947
MONGO_CONNECTION_STRING=mongodb+srv://username:password@cluster.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000
5048

5149
# Name of the MongoDB/Cosmos DB cluster
52-
MONGO_CLUSTER_NAME=vectorSearch
50+
MONGO_CLUSTER_NAME=your-cluster-name-here

‎ai/vector-search-python/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,11 +105,11 @@ cp .env.example .env
105105
AZURE_OPENAI_EMBEDDING_MODEL=text-embedding-3-small
106106
AZURE_OPENAI_EMBEDDING_ENDPOINT=https://your-openai-resource.openai.azure.com/
107107
AZURE_OPENAI_EMBEDDING_KEY=your-azure-openai-api-key
108-
AZURE_OPENAI_EMBEDDING_API_VERSION=2024-02-01
108+
AZURE_OPENAI_EMBEDDING_API_VERSION=2023-05-15
109109
110110
# MongoDB/DocumentDB Configuration
111111
MONGO_CONNECTION_STRING=mongodb+srv://username:password@your-cluster.mongocluster.cosmos.azure.com/?tls=true&authMechanism=SCRAM-SHA-256&retrywrites=false&maxIdleTimeMS=120000
112-
MONGO_CLUSTER_NAME=vectorSearch
112+
MONGO_CLUSTER_NAME=your-cluster-name
113113
114114
# Data Configuration (defaults should work)
115115
DATA_FILE_WITHOUT_VECTORS=data/Hotels_Vector.json

‎ai/vector-search-python/output/diskann.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
Initializing MongoDB and Azure OpenAI clients...
2+
3+
Initializing clients...
4+
5+
Loading data from ../data/Hotels_Vector.json...
6+
Loaded 50 documents
7+
8+
Inserting data into collection 'vectorSearchCollection'...
9+
Cleared existing data from collection
10+
Starting batch insertion of 50 documents...
11+
Batch 1 completed: 50 documents inserted
12+
Creating DiskANN vector index on field 'DescriptionVector'...
13+
No existing vector indexes found to drop
14+
DiskANN vector index created successfully
15+
Waiting for index to be ready...
16+
Performing DiskANN vector search for: 'quintessential lodging near running trails, eateries, retail'
17+
18+
Search Results (showing top 5):
19+
================================================================================
20+
HotelName: Royal Cottage Resort, Score: 0.4991
21+
HotelName: Country Comfort Inn, Score: 0.4785
22+
HotelName: Nordick's Valley Motel, Score: 0.4635
23+
HotelName: Economy Universe Motel, Score: 0.4461
24+
HotelName: Roach Motel, Score: 0.4388

‎ai/vector-search-python/output/hnsw.txt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
Starting HNSW vector search demonstration...
2+
3+
Initializing clients...
4+
5+
Loading data from ../data/Hotels_Vector.json...
6+
Loaded 50 documents
7+
8+
Preparing collection 'vectorSearchCollection'...
9+
Cleared existing data from collection
10+
Starting batch insertion of 50 documents...
11+
Batch 1 completed: 50 documents inserted
12+
13+
Creating HNSW vector index...
14+
Creating HNSW vector index on field 'DescriptionVector'...
15+
Dropping existing vector index: diskann_index_DescriptionVector
16+
Dropped 1 existing vector index(es)
17+
HNSW vector index created successfully
18+
Waiting for index to be ready...
19+
Performing HNSW vector search for: 'quintessential lodging near running trails, eateries, retail'
20+
21+
Search Results (showing top 5):
22+
================================================================================
23+
HotelName: Royal Cottage Resort, Score: 0.4991
24+
HotelName: Country Comfort Inn, Score: 0.4785
25+
HotelName: Nordick's Valley Motel, Score: 0.4635
26+
HotelName: Economy Universe Motel, Score: 0.4461
27+
HotelName: Roach Motel, Score: 0.4388

‎ai/vector-search-python/output/ivf.txt‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
Starting IVF vector search demonstration...
2+
3+
Initializing clients...
4+
5+
Loading data from ../data/Hotels_Vector.json...
6+
Loaded 50 documents
7+
8+
Preparing collection 'vectorSearchCollection'...
9+
Cleared existing data from collection
10+
Starting batch insertion of 50 documents...
11+
Batch 1 completed: 50 documents inserted
12+
13+
Creating IVF vector index...
14+
Creating IVF vector index on field 'DescriptionVector'...
15+
Dropping existing vector index: hnsw_index_DescriptionVector
16+
Dropped 1 existing vector index(es)
17+
IVF vector index created successfully
18+
Waiting for index clustering to complete...
19+
Performing IVF vector search for: 'quintessential lodging near running trails, eateries, retail'
20+
21+
Search Results (showing top 5):
22+
================================================================================
23+
HotelName: Royal Cottage Resort, Score: 0.4991
24+
HotelName: Country Comfort Inn, Score: 0.4785
25+
HotelName: Nordick's Valley Motel, Score: 0.4635
26+
HotelName: Economy Universe Motel, Score: 0.4461
27+
HotelName: Roach Motel, Score: 0.4388

‎ai/vector-search-python/src/diskann.py‎

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -123,9 +123,9 @@ def main():
123123

124124
# Load configuration from environment variables
125125
config = {
126-
'cluster_name': os.getenv('MONGO_CLUSTER_NAME', 'vectorSearch'),
127-
'database_name': 'vectorSearchDB',
128-
'collection_name': 'vectorSearchCollection',
126+
'cluster_name': os.getenv('MONGO_CLUSTER_NAME'),
127+
'database_name': 'Hotels',
128+
'collection_name': 'hotels_diskann',
129129
'data_file': os.getenv('DATA_FILE_WITH_VECTORS', 'data/Hotels_Vector.json'),
130130
'vector_field': os.getenv('EMBEDDED_FIELD', 'DescriptionVector'),
131131
'model_name': os.getenv('AZURE_OPENAI_EMBEDDING_MODEL', 'text-embedding-3-small'),
@@ -156,31 +156,28 @@ def main():
156156
# Insert data into collection
157157
print(f"\nInserting data into collection '{config['collection_name']}'...")
158158

159-
# Clear existing data to ensure clean state
160-
collection.delete_many({})
161-
print("Cleared existing data from collection")
162-
163159
# Insert the hotel data
164160
stats = insert_data(
165161
collection,
166162
documents_with_embeddings,
167163
batch_size=config['batch_size']
168164
)
169165

170-
if stats['inserted'] == 0:
166+
if stats['inserted'] == 0 and not stats.get('skipped'):
171167
raise ValueError("No documents were inserted successfully")
172168

173-
# Create DiskANN vector index
174-
create_diskann_vector_index(
175-
collection,
176-
config['vector_field'],
177-
config['dimensions']
178-
)
179-
180-
# Wait briefly for index to be ready
181-
import time
182-
print("Waiting for index to be ready...")
183-
time.sleep(2)
169+
# Create DiskANN vector index (skip if data was already present)
170+
if not stats.get('skipped'):
171+
create_diskann_vector_index(
172+
collection,
173+
config['vector_field'],
174+
config['dimensions']
175+
)
176+
177+
# Wait briefly for index to be ready
178+
import time
179+
print("Waiting for index to be ready...")
180+
time.sleep(2)
184181

185182
# Perform sample vector search
186183
query = "quintessential lodging near running trails, eateries, retail"

‎ai/vector-search-python/src/hnsw.py‎

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,9 @@ def main():
117117

118118
# Load configuration from environment variables
119119
config = {
120-
'cluster_name': os.getenv('MONGO_CLUSTER_NAME', 'vectorSearch'),
121-
'database_name': 'vectorSearchDB',
122-
'collection_name': 'vectorSearchCollection',
120+
'cluster_name': os.getenv('MONGO_CLUSTER_NAME'),
121+
'database_name': 'Hotels',
122+
'collection_name': 'hotels_hnsw',
123123
'data_file': os.getenv('DATA_FILE_WITH_VECTORS', 'data/Hotels_Vector.json'),
124124
'vector_field': os.getenv('EMBEDDED_FIELD', 'DescriptionVector'),
125125
'model_name': os.getenv('AZURE_OPENAI_EMBEDDING_MODEL', 'text-embedding-3-small'),
@@ -150,32 +150,29 @@ def main():
150150
# Insert data into MongoDB collection
151151
print(f"\nPreparing collection '{config['collection_name']}'...")
152152

153-
# Clear any existing data to start fresh
154-
collection.delete_many({})
155-
print("Cleared existing data from collection")
156-
157153
# Insert hotel data with embeddings
158154
stats = insert_data(
159155
collection,
160156
documents_with_embeddings,
161157
batch_size=config['batch_size']
162158
)
163159

164-
if stats['inserted'] == 0:
160+
if stats['inserted'] == 0 and not stats.get('skipped'):
165161
raise ValueError("No documents were inserted successfully")
166162

167-
# Create HNSW vector index for efficient similarity search
168-
print("\nCreating HNSW vector index...")
169-
create_hnsw_vector_index(
170-
collection,
171-
config['vector_field'],
172-
config['dimensions']
173-
)
174-
175-
# Allow time for index to become ready
176-
import time
177-
print("Waiting for index to be ready...")
178-
time.sleep(2)
163+
# Create HNSW vector index (skip if data was already present)
164+
if not stats.get('skipped'):
165+
print("\nCreating HNSW vector index...")
166+
create_hnsw_vector_index(
167+
collection,
168+
config['vector_field'],
169+
config['dimensions']
170+
)
171+
172+
# Allow time for index to become ready
173+
import time
174+
print("Waiting for index to be ready...")
175+
time.sleep(2)
179176

180177
# Demonstrate HNSW search with various queries
181178
query = "quintessential lodging near running trails, eateries, retail"

‎ai/vector-search-python/src/ivf.py‎

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,9 @@ def main():
114114

115115
# Load configuration from environment variables
116116
config = {
117-
'cluster_name': os.getenv('MONGO_CLUSTER_NAME', 'vectorSearch'),
118-
'database_name': 'vectorSearchDB',
119-
'collection_name': 'vectorSearchCollection',
117+
'cluster_name': os.getenv('MONGO_CLUSTER_NAME'),
118+
'database_name': 'Hotels',
119+
'collection_name': 'hotels_ivf',
120120
'data_file': os.getenv('DATA_FILE_WITH_VECTORS', 'data/Hotels_Vector.json'),
121121
'vector_field': os.getenv('EMBEDDED_FIELD', 'DescriptionVector'),
122122
'model_name': os.getenv('AZURE_OPENAI_EMBEDDING_MODEL', 'text-embedding-3-small'),
@@ -147,32 +147,29 @@ def main():
147147
# Prepare collection with fresh data
148148
print(f"\nPreparing collection '{config['collection_name']}'...")
149149

150-
# Remove any existing data for clean state
151-
collection.delete_many({})
152-
print("Cleared existing data from collection")
153-
154150
# Insert hotel data with embeddings
155151
stats = insert_data(
156152
collection,
157153
documents_with_embeddings,
158154
batch_size=config['batch_size']
159155
)
160156

161-
if stats['inserted'] == 0:
157+
if stats['inserted'] == 0 and not stats.get('skipped'):
162158
raise ValueError("No documents were inserted successfully")
163159

164-
# Create IVF vector index for clustering-based search
165-
print("\nCreating IVF vector index...")
166-
create_ivf_vector_index(
167-
collection,
168-
config['vector_field'],
169-
config['dimensions']
170-
)
171-
172-
# Wait for index to be built and ready
173-
import time
174-
print("Waiting for index clustering to complete...")
175-
time.sleep(3) # IVF may need more time for clustering
160+
# Create IVF vector index (skip if data was already present)
161+
if not stats.get('skipped'):
162+
print("\nCreating IVF vector index...")
163+
create_ivf_vector_index(
164+
collection,
165+
config['vector_field'],
166+
config['dimensions']
167+
)
168+
169+
# Wait for index to be built and ready
170+
import time
171+
print("Waiting for index clustering to complete...")
172+
time.sleep(3) # IVF may need more time for clustering
176173

177174
# Demonstrate IVF search
178175
query = "quintessential lodging near running trails, eateries, retail"

‎ai/vector-search-python/src/show_indexes.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ def main():
186186
# Load configuration from environment variables
187187
config = {
188188
'cluster_name': os.getenv('MONGO_CLUSTER_NAME', 'vectorSearch'),
189-
'default_database': 'vectorSearchDB',
189+
'default_database': 'Hotels',
190190
'default_collection': 'vectorSearchCollection'
191191
}
192192

0 commit comments

Comments
 (0)