Plugin instance invocation + Block and Tag (#30)

Combined Dave/Ted work for bringing Python client up to current.

* Changes to accommodate plugin instances - all plugin invocations happen via a Plugin Instance
* Many changes to client inputs and outputs based on block and tag refactor
* Temporary separation of File 'verbs' into extension package; this likely needs generalized / revisited later Generalize the data/extensions pattern; probably rename the packages #29
* Adjustments to test expectations based on block / tag refactor

Author: dkolas

.gitignore

@@ -20,7 +20,6 @@ __pycache__/*
 .pydevproject
 .settings
 .idea
-tags
 
 # Package files
 *.egg

src/steamship/base/client.py

@@ -3,6 +3,7 @@
 from typing import Any, TypeVar, Union
 
 import requests  # type: ignore
+
 from steamship.base.configuration import Configuration
 from steamship.base.error import SteamshipError
 from steamship.base.mime_types import MimeTypes
@@ -206,7 +207,7 @@ def call(
             appCall: bool = False,
             appOwner: str = None,
             appId: str = None,
-            appInstanceId: str = None
+            appInstanceId: str = None,
     ) -> Union[Any, Response[T]]:
         """Post to the Steamship API.
 

src/steamship/base/mime_types.py

@@ -16,3 +16,12 @@ class MimeTypes:
     PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
     RTF = "application/rtf"
     BINARY = "application/octet-stream"
+
+
+TEXT_MIME_TYPES = [
+    MimeTypes.TXT,
+    MimeTypes.MKD,
+    MimeTypes.HTML,
+    MimeTypes.DOCX,
+    MimeTypes.PPTX
+]

src/steamship/base/response.py

@@ -24,7 +24,7 @@ def update(self, response: "Response[T]"):
     def wait(self, max_timeout_s: float = 60, retry_delay_s: float = 1):
         """Polls and blocks until the task has succeeded or failed (or timeout reached)."""
         start = time.time()
-        if self.task is None:
+        if self.task is None or self.task.state == TaskStatus.failed:
             return
 
         self.check()

src/steamship/client/client.py

@@ -1,21 +1,25 @@
 import logging
 from typing import List
 
+from steamship import Block
 from steamship.base import Client, Response
+from steamship.client.operations.embedder import EmbedRequest
+from steamship.client.operations.tagger import TagRequest
 from steamship.client.tasks import Tasks
-from steamship.data import Block, Classifier, Corpus, File
-from steamship.data.embedding import EmbedAndSearchRequest, EmbedAndSearchResponse, EmbedRequest, EmbedResponse
-from steamship.data.embedding_index import EmbeddingIndex
-from steamship.data.parser import TokenMatcher, PhraseMatcher, DependencyMatcher, ParseRequest, ParseResponse
+from steamship.data import File
+from steamship.data.embeddings import EmbedAndSearchRequest, EmbedAndSearchResponse, EmbeddingIndex
 from steamship.data.space import Space
-from steamship.data.tagging import TagRequest, TagResponse
 
 __copyright__ = "Steamship"
 __license__ = "MIT"
 
+from steamship.extension.file import TagResponse
+from steamship.plugin.outputs.embedded_items_plugin_output import EmbeddedItemsPluginOutput
+
 _logger = logging.getLogger(__name__)
 
 
+
 class Steamship(Client):
     """Steamship Python Client."""
 
@@ -56,40 +60,11 @@ def __init__(
         """
         self.tasks = Tasks(self)
 
-    def create_corpus(
-            self,
-            name: str,
-            handle: str = None,
-            description: str = None,
-            externalId: str = None,
-            externalType: str = None,
-            metadata: any = None,
-            isPublic: bool = False,
-            upsert: bool = False,
-            spaceId: bool = False,
-            spaceHandle: bool = False,
-            space: Space = None
-    ) -> Corpus:
-        return Corpus.create(
-            client=self,
-            name=name,
-            handle=handle,
-            description=description,
-            isPublic=isPublic,
-            upsert=upsert,
-            externalId=externalId,
-            externalType=externalType,
-            metadata=metadata,
-            spaceId=spaceId,
-            spaceHandle=spaceHandle,
-            space=space
-        )
-
     def create_index(
             self,
             handle: str = None,
             name: str = None,
-            plugin: str = None,
+            pluginInstance: str = None,
             upsert: bool = True,
             externalId: str = None,
             externalType: str = None,
@@ -102,7 +77,7 @@ def create_index(
             client=self,
             handle=handle,
             name=name,
-            plugin=plugin,
+            pluginInstance=pluginInstance,
             upsert=upsert,
             externalId=externalId,
             externalType=externalType,
@@ -112,29 +87,6 @@ def create_index(
             space=space
         )
 
-    def create_classifier(
-            self,
-            name: str,
-            plugin: str,
-            upsert: bool = True,
-            save: bool = None,
-            labels: List[str] = None,
-            spaceId: str = None,
-            spaceHandle: str = None,
-            space: Space = None
-    ) -> Classifier:
-        return Classifier.create(
-            client=self,
-            plugin=plugin,
-            name=name,
-            upsert=upsert,
-            save=save,
-            labels=labels,
-            spaceId=spaceId,
-            spaceHandle=spaceHandle,
-            space=space
-        )
-
     def upload(
             self,
             filename: str = None,
@@ -146,7 +98,7 @@ def upload(
             spaceHandle: str = None,
             space: Space = None
     ) -> File:
-        return File.upload(
+        return File.create(
             self,
             filename=filename,
             name=name,
@@ -179,19 +131,19 @@ def scrape(
     def embed(
             self,
             docs: List[str],
-            plugin: str,
+            pluginInstance: str,
             spaceId: str = None,
             spaceHandle: str = None,
             space: Space = None
-    ) -> Response[EmbedResponse]:
+    ) -> Response[EmbeddedItemsPluginOutput]:
         req = EmbedRequest(
             docs=docs,
-            plugin=plugin
+            pluginInstance=pluginInstance
         )
         return self.post(
             'embedding/create',
             req,
-            expect=EmbedResponse,
+            expect=EmbeddedItemsPluginOutput,
             spaceId=spaceId,
             spaceHandle=spaceHandle,
             space=space
@@ -201,7 +153,7 @@ def embed_and_search(
             self,
             query: str,
             docs: List[str],
-            plugin: str,
+            pluginInstance: str,
             k: int = 1,
             spaceId: str = None,
             spaceHandle: str = None,
@@ -210,7 +162,7 @@ def embed_and_search(
         req = EmbedAndSearchRequest(
             query=query,
             docs=docs,
-            plugin=plugin,
+            pluginInstance=pluginInstance,
             k=k
         )
         return self.post(
@@ -222,61 +174,28 @@ def embed_and_search(
             space=space
         )
 
-    def parse(
-            self,
-            docs: List[str],
-            plugin: str = None,
-            tokenMatchers: List[TokenMatcher] = None,
-            phraseMatchers: List[PhraseMatcher] = None,
-            dependencyMatchers: List[DependencyMatcher] = None,
-            includeTokens: bool = True,
-            includeParseData: bool = True,
-            includeEntities: bool = False,
-            metadata: any = None,
-            spaceId: str = None,
-            spaceHandle: str = None,
-            space: Space = None
-    ) -> Response[ParseResponse]:
-        req = ParseRequest(
-            type="inline",
-            docs=docs,
-            plugin=plugin,
-            tokenMatchers=tokenMatchers,
-            phraseMatchers=phraseMatchers,
-            dependencyMatchers=dependencyMatchers,
-            includeTokens=includeTokens,
-            includeParseData=includeParseData,
-            includeEntities=includeEntities,
-            metadata=metadata
-        )
-        return self.post(
-            'plugin/parse',
-            req,
-            expect=ParseResponse,
-            spaceId=spaceId,
-            spaceHandle=spaceHandle,
-            space=space
-        )
-
     def tag(
             self,
-            blocks: List[Block],
-            plugin: str = None,
-            metadata: any = None,
+            doc: str,
+            pluginInstance: str = None,
             spaceId: str = None,
             spaceHandle: str = None,
             space: Space = None
-    ) -> Response[ParseResponse]:
+    ) -> Response[TagResponse]:
         req = TagRequest(
-            blocks=blocks,
-            plugin=plugin,
-            metadata=metadata
+            type="inline",
+            file=File.CreateRequest(
+                blocks=[Block.CreateRequest(
+                    text=doc
+                )]
+            ),
+            pluginInstance=pluginInstance,
         )
         return self.post(
-            'tagger/tag',
+            'plugin/instance/tag',
             req,
             expect=TagResponse,
             spaceId=spaceId,
             spaceHandle=spaceHandle,
-            space=Space
+            space=space
         )

src/steamship/client/operations/__init__.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass
+
+from steamship.base import Client
+
+
+@dataclass
+class ConvertRequest:
+    type: str = None
+    pluginInstance: str = None
+    id: str = None
+    handle: str = None
+    name: str = None
+
+    @staticmethod
+    def from_dict(d: any, client: Client = None) -> "ConvertRequest":
+        return ConvertRequest(
+            type=d.get('type', None),
+            pluginInstance=d.get('pluginInstance', None),
+            id=d.get('id', None),
+            handle=d.get('handle', None),
+            name=d.get('name', None)
+        )

src/steamship/client/operations/converter.py

@@ -0,0 +1,71 @@
+from abc import ABC
+from dataclasses import dataclass
+from typing import List
+
+from steamship import File
+from steamship.base import Client
+
+@dataclass
+class CorpusImportRequest:
+    # The Corpus Identifiers
+    client: Client = None
+    id: str = None
+    handle: str = None
+    type: str = 'corpus'
+
+    # Data for the plugin
+    value: str = None
+    data: str = None
+    url: str = None
+    pluginInstance: str = None
+    fileImporterPluginInstance: str = None
+
+    @staticmethod
+    def from_dict(d: any, client: Client = None) -> "CorpusImportRequest":
+        return CorpusImportRequest(
+            client=client,
+            id=d.get('id', None),
+            handle=d.get('handle', None),
+            type='corpus',
+            value=d.get('value', None),
+            data=d.get('data', None),
+            url=d.get('url', None),
+            pluginInstance=d.get('pluginInstance', None),
+            fileImporterPluginInstance=d.get('fileImporterPluginInstance', None)
+        )
+
+    def to_dict(self) -> dict:
+        return dict(
+            id=self.id,
+            handle=self.handle,
+            type=self.type,
+            value=self.value,
+            data=self.data,
+            url=self.url,
+            pluginInstance=self.pluginInstance,
+            fileImporterPluginInstance=self.fileImporterPluginInstance
+        )
+
+
+@dataclass
+class CorpusImportResponse:
+    client: Client = None
+    fileImportRequests: List[File.CreateRequest] = None
+
+    def __init__(
+            self,
+            fileImportRequests: List[File.CreateRequest] = None
+    ):
+        self.fileImportRequests = fileImportRequests
+
+    @staticmethod
+    def from_dict(d: any, client: Client = None) -> "CorpusImportResponse":
+        return CorpusImportResponse(
+            client=client,
+            fileImportRequests=[File.CreateRequest.from_dict(req) for req in d.get('fileImportRequests', [])]
+        )
+
+    def to_dict(self) -> dict:
+        return dict(
+            fileImportRequests=self.fileImportRequests
+        )

src/steamship/client/operations/corpus_importer.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+from typing import List, Dict
+from steamship.base import Client
+
+
+@dataclass
+class EmbedRequest:
+    docs: List[str]
+    pluginInstance: str
+    metadata: Dict = None
+
+    @staticmethod
+    def from_dict(d: any, client: Client = None) -> "EmbedRequest":
+        return EmbedRequest(
+            docs=d.get('docs', None),
+            pluginInstance=d.get('pluginInstance', None),
+            metadata=d.get('metadata', {})
+        )