Merge pull request #1898 from vespa-engine/kaibor737/add-scaling-tutorial

fixed python script and improved node naming

Author: kaibor737

scaling-tutorial/ext/transform_ms_marco.py

@@ -1,24 +1,36 @@
 import json
+import itertools
 
-with (
-    open("ext/corpus.jsonl", "r") as infile, 
-    open("ext/corpus_transformed_full.jsonl", "w") as outfile_full,
-    open("ext/corpus_transformed_500000.jsonl", "w") as outfile_500000,
-    open("ext/corpus_transformed_50000.jsonl", "w") as outfile_50000,
-    open("ext/corpus_transformed_1000.jsonl", "w") as outfile_1000,
-    ):
-    for line in infile:
-        doc = json.loads(line)
-        doc_id = doc["docid"]
-        transformed = {
-            "put": f"id:msmarco:passage::{doc_id}",
-            "fields": {
-                "text": doc["text"],
-                "title": doc["title"],
-                "id": doc_id,
-            },
-        }
-        outfile_full.write(json.dumps(transformed) + "\n")
-        outfile_500000.write(json.dumps(transformed) + "\n")
-        outfile_50000.write(json.dumps(transformed) + "\n")
-        outfile_1000.write(json.dumps(transformed) + "\n")
+SUBSETS = {
+    "full": None,
+    "500000": 500000,
+    "50000": 50000,
+    "5000": 5000,
+    "1000": 1000,
+}
+
+def transform(doc):
+    doc_id = doc["docid"]
+    return {
+        "put": f"id:msmarco:passage::{doc_id}",
+        "fields": {
+            "text": doc["text"],
+            "id": doc_id,
+        },
+    }
+
+with open("ext/corpus.jsonl", "r") as infile:
+    outfiles = {
+        name: open(f"ext/corpus_transformed_{name}.jsonl", "w")
+        for name in SUBSETS
+    }
+    try:
+        for i, line in enumerate(infile):
+            transformed = transform(json.loads(line))
+            serialized = json.dumps(transformed) + "\n"
+            for name, limit in SUBSETS.items():
+                if limit is None or i < limit:
+                    outfiles[name].write(serialized)
+    finally:
+        for f in outfiles.values():
+            f.close()

scaling-tutorial/services.xml

@@ -2,7 +2,7 @@
 <!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
 <services version="1.0" xmlns:deploy="vespa" xmlns:preprocess="properties" minimum-required-vespa-version="8.311.28">
 
-  <container id="default" version="1.0">
+  <container id="container-default" version="1.0">
 
     <nodes deploy:environment="dev" count="1">
       <resources vcpu="1.0" memory="8Gb" architecture="arm64" storage-type="local" disk="59Gb"/>
@@ -34,12 +34,12 @@
 
   </container>
 
-  <content id="msmarco" version="1.0">
+  <content id="content-msmarco" version="1.0">
     <min-redundancy>1</min-redundancy>
     <documents>
       <document mode="index" type="passage"/>
     </documents>
-    <nodes count="1">
+    <nodes deploy:environment="dev" count="1">
       <resources vcpu="1.0" memory="8Gb" architecture="arm64" storage-type="local" disk="59Gb"/>
     </nodes> 
     <engine>