feat: Add directory argument to allow restricting file browser on specific folder (HEXA-1356)

Author: bramj

openhexa/graphql/graphql_client/__init__.py

@@ -167,6 +167,7 @@
     PermissionMode,
     PinDatasetError,
     PipelineError,
+    PipelineFunctionalType,
     PipelineNotificationLevel,
     PipelineRecipientError,
     PipelineRunOrderBy,
@@ -645,6 +646,7 @@
     "PinDatasetInput",
     "Pipeline",
     "PipelineError",
+    "PipelineFunctionalType",
     "PipelineNotificationLevel",
     "PipelinePipelineByCode",
     "PipelinePipelineByCodeCurrentVersion",

openhexa/graphql/graphql_client/create_pipeline_from_template_version.py

@@ -10,9 +10,9 @@
 
 
 class CreatePipelineFromTemplateVersion(BaseModel):
-    create_pipeline_from_template_version: (
-        "CreatePipelineFromTemplateVersionCreatePipelineFromTemplateVersion"
-    ) = Field(alias="createPipelineFromTemplateVersion")
+    create_pipeline_from_template_version: "CreatePipelineFromTemplateVersionCreatePipelineFromTemplateVersion" = Field(
+        alias="createPipelineFromTemplateVersion"
+    )
 
 
 class CreatePipelineFromTemplateVersionCreatePipelineFromTemplateVersion(BaseModel):

openhexa/graphql/graphql_client/create_pipeline_template_version.py

@@ -10,9 +10,9 @@
 
 
 class CreatePipelineTemplateVersion(BaseModel):
-    create_pipeline_template_version: (
-        "CreatePipelineTemplateVersionCreatePipelineTemplateVersion"
-    ) = Field(alias="createPipelineTemplateVersion")
+    create_pipeline_template_version: "CreatePipelineTemplateVersionCreatePipelineTemplateVersion" = Field(
+        alias="createPipelineTemplateVersion"
+    )
 
 
 class CreatePipelineTemplateVersionCreatePipelineTemplateVersion(BaseModel):

openhexa/graphql/graphql_client/enums.py

@@ -546,13 +546,21 @@ class PipelineError(str, Enum):
     PERMISSION_DENIED = "PERMISSION_DENIED"
     PIPELINE_ALREADY_COMPLETED = "PIPELINE_ALREADY_COMPLETED"
     PIPELINE_ALREADY_STOPPED = "PIPELINE_ALREADY_STOPPED"
+    PIPELINE_CODE_PARSING_ERROR = "PIPELINE_CODE_PARSING_ERROR"
     PIPELINE_DOES_NOT_SUPPORT_PARAMETERS = "PIPELINE_DOES_NOT_SUPPORT_PARAMETERS"
     PIPELINE_NOT_FOUND = "PIPELINE_NOT_FOUND"
     PIPELINE_VERSION_NOT_FOUND = "PIPELINE_VERSION_NOT_FOUND"
     TABLE_NOT_FOUND = "TABLE_NOT_FOUND"
     WORKSPACE_NOT_FOUND = "WORKSPACE_NOT_FOUND"
 
 
+class PipelineFunctionalType(str, Enum):
+    computation = "computation"
+    extraction = "extraction"
+    loading = "loading"
+    transformation = "transformation"
+
+
 class PipelineNotificationLevel(str, Enum):
     ALL = "ALL"
     ERROR = "ERROR"

openhexa/graphql/graphql_client/input_types.py

@@ -14,6 +14,7 @@
     OrganizationMembershipRole,
     ParameterWidget,
     PermissionMode,
+    PipelineFunctionalType,
     PipelineNotificationLevel,
     WorkspaceMembershipRole,
 )
@@ -141,6 +142,9 @@ class CreatePipelineFromTemplateVersionInput(BaseModel):
 
 class CreatePipelineInput(BaseModel):
     code: Optional[str] = None
+    functional_type: Optional[PipelineFunctionalType] = Field(
+        alias="functionalType", default=None
+    )
     name: str
     notebook_path: Optional[str] = Field(alias="notebookPath", default=None)
     workspace_slug: str = Field(alias="workspaceSlug")
@@ -379,6 +383,7 @@ class ParameterInput(BaseModel):
     code: str
     connection: Optional[str] = None
     default: Optional[Any] = None
+    directory: Optional[str] = None
     help: Optional[str] = None
     multiple: Optional[bool] = None
     name: Optional[str] = None
@@ -598,9 +603,13 @@ class UpdatePipelineInput(BaseModel):
     )
     config: Optional[Any] = None
     description: Optional[str] = None
+    functional_type: Optional[PipelineFunctionalType] = Field(
+        alias="functionalType", default=None
+    )
     id: Any
     name: Optional[str] = None
     schedule: Optional[str] = None
+    tags: Optional[List[str]] = None
     webhook_enabled: Optional[bool] = Field(alias="webhookEnabled", default=None)
 
 
@@ -675,9 +684,13 @@ class UploadPipelineInput(BaseModel):
     config: Optional[Any] = None
     description: Optional[str] = None
     external_link: Optional[Any] = Field(alias="externalLink", default=None)
+    functional_type: Optional[PipelineFunctionalType] = Field(
+        alias="functionalType", default=None
+    )
     name: Optional[str] = None
-    parameters: List["ParameterInput"]
+    parameters: Optional[List["ParameterInput"]] = None
     pipeline_code: Optional[str] = Field(alias="pipelineCode", default=None)
+    tags: Optional[List[str]] = None
     timeout: Optional[int] = None
     workspace_slug: str = Field(alias="workspaceSlug")
     zipfile: str

openhexa/graphql/graphql_client/upgrade_pipeline_version_from_template.py

@@ -10,9 +10,9 @@
 
 
 class UpgradePipelineVersionFromTemplate(BaseModel):
-    upgrade_pipeline_version_from_template: (
-        "UpgradePipelineVersionFromTemplateUpgradePipelineVersionFromTemplate"
-    ) = Field(alias="upgradePipelineVersionFromTemplate")
+    upgrade_pipeline_version_from_template: "UpgradePipelineVersionFromTemplateUpgradePipelineVersionFromTemplate" = Field(
+        alias="upgradePipelineVersionFromTemplate"
+    )
 
 
 class UpgradePipelineVersionFromTemplateUpgradePipelineVersionFromTemplate(BaseModel):

openhexa/graphql/schema.generated.graphql

@@ -794,6 +794,7 @@ type CreatePipelineFromTemplateVersionResult {
 """Represents the input for creating a pipeline."""
 input CreatePipelineInput {
   code: String @deprecated(reason: "The code will be autogenerated")
+  functionalType: PipelineFunctionalType
   name: String!
   notebookPath: String
   workspaceSlug: String!
@@ -2634,7 +2635,7 @@ type Organization {
   invitations(includeAccepted: Boolean, page: Int, perPage: Int): OrganizationInvitationPage!
 
   """The members of the organization."""
-  members(page: Int, perPage: Int, term: String): OrganizationMembershipPage!
+  members(page: Int, perPage: Int, role: OrganizationMembershipRole, term: String): OrganizationMembershipPage!
 
   """The name of the organization."""
   name: String!
@@ -2748,6 +2749,7 @@ input ParameterInput {
   code: String!
   connection: String
   default: Generic
+  directory: String
   help: String
   multiple: Boolean
   name: String
@@ -2829,6 +2831,7 @@ type Pipeline {
   createdAt: DateTime!
   currentVersion: PipelineVersion
   description: String
+  functionalType: PipelineFunctionalType
   hasNewTemplateVersions: Boolean!
   id: UUID!
   name: String
@@ -2839,6 +2842,7 @@ type Pipeline {
   runs(orderBy: PipelineRunOrderBy, page: Int, perPage: Int): PipelineRunPage!
   schedule: String
   sourceTemplate: PipelineTemplate
+  tags: [Tag!]!
   template: PipelineTemplate
   type: PipelineType!
   updatedAt: DateTime
@@ -2857,13 +2861,32 @@ enum PipelineError {
   PERMISSION_DENIED
   PIPELINE_ALREADY_COMPLETED
   PIPELINE_ALREADY_STOPPED
+  PIPELINE_CODE_PARSING_ERROR
   PIPELINE_DOES_NOT_SUPPORT_PARAMETERS
   PIPELINE_NOT_FOUND
   PIPELINE_VERSION_NOT_FOUND
   TABLE_NOT_FOUND
   WORKSPACE_NOT_FOUND
 }
 
+"""
+Represents the functional purpose of a pipeline in data workflows.
+
+Categorizes WHAT a pipeline does in terms of business purpose:
+- extraction: Data ingestion from external sources
+- transformation: Data processing operations
+- loading: Data output to destinations
+- computation: Analytics and computational workflows
+
+Used for pipeline categorization, filtering, and workflow organization.
+"""
+enum PipelineFunctionalType {
+  computation
+  extraction
+  loading
+  transformation
+}
+
 """Represents the notification level for a pipeline recipient."""
 enum PipelineNotificationLevel {
   ALL
@@ -3067,7 +3090,13 @@ type PipelineTokenResult {
   token: String
 }
 
-"""Represents the type of a pipeline."""
+"""
+Represents the technical implementation format of a pipeline.
+
+This determines HOW a pipeline is packaged and executed:
+- zipFile: Code archive with Python modules (.zip format)
+- notebook: Jupyter notebook-based pipeline (.ipynb format)
+"""
 enum PipelineType {
   notebook
   zipFile
@@ -3310,12 +3339,12 @@ type Query {
   pipelineVersion(id: UUID!): PipelineVersion
 
   """Retrieves a page of pipelines ordered by relevant name."""
-  pipelines(name: String, page: Int, perPage: Int, search: String, workspaceSlug: String): PipelinesPage!
+  pipelines(functionalType: PipelineFunctionalType, name: String, page: Int, perPage: Int, search: String, tags: [String!], workspaceSlug: String): PipelinesPage!
   searchDatabaseTables(organizationId: UUID, page: Int = 1, perPage: Int = 15, query: String!, workspaceSlugs: [String]): DatabaseTableResultPage!
   searchDatasets(organizationId: UUID, page: Int = 1, perPage: Int = 15, query: String!, workspaceSlugs: [String]): DatasetResultPage!
   searchFiles(organizationId: UUID, page: Int = 1, perPage: Int = 15, query: String!, workspaceSlugs: [String]): FileResultPage!
   searchPipelineTemplates(organizationId: UUID, page: Int = 1, perPage: Int = 15, query: String!, workspaceSlugs: [String]): PipelineTemplateResultPage!
-  searchPipelines(organizationId: UUID, page: Int = 1, perPage: Int = 15, query: String!, workspaceSlugs: [String]): PipelineResultPage!
+  searchPipelines(functionalType: PipelineFunctionalType, organizationId: UUID, page: Int = 1, perPage: Int = 15, query: String!, workspaceSlugs: [String]): PipelineResultPage!
   team(id: UUID!): Team
   teams(page: Int, perPage: Int, term: String): TeamPage!
 
@@ -4028,9 +4057,11 @@ input UpdatePipelineInput {
   autoUpdateFromTemplate: Boolean
   config: JSON
   description: String
+  functionalType: PipelineFunctionalType
   id: UUID!
   name: String
   schedule: String
+  tags: [String!]
   webhookEnabled: Boolean
 }
 
@@ -4311,16 +4342,19 @@ input UploadPipelineInput {
   config: JSON
   description: String
   externalLink: URL
+  functionalType: PipelineFunctionalType
   name: String
-  parameters: [ParameterInput!]!
+  parameters: [ParameterInput!]
   pipelineCode: String
+  tags: [String!]
   timeout: Int
   workspaceSlug: String!
   zipfile: String!
 }
 
 """Represents the result of uploading a pipeline."""
 type UploadPipelineResult {
+  details: String
   errors: [PipelineError!]!
   pipelineVersion: PipelineVersion
   success: Boolean!

openhexa/sdk/pipelines/parameter.py

@@ -358,6 +358,16 @@ def expected_type(self) -> type:
         """Returns the python type expected for values."""
         return File
 
+    @property
+    def accepts_multiple(self) -> bool:
+        """Only allow single file selection."""
+        return False
+
+    @property
+    def accepts_choices(self) -> bool:
+        """Don't allow choices for file."""
+        return False
+
     def validate_default(self, value: typing.Any | None):
         """Validate the default value configured for this type."""
         if value is None:
@@ -444,6 +454,7 @@ def __init__(
         connection: str | None = None,
         required: bool = True,
         multiple: bool = False,
+        directory: str | None = None,
     ):
         validate_pipeline_parameter_code(code)
         self.code = code
@@ -480,6 +491,7 @@ def __init__(
 
         self.widget = widget
         self.connection = connection
+        self.directory = directory
 
         self._validate_default(default, multiple)
         self.default = default
@@ -504,6 +516,7 @@ def to_dict(self) -> dict[str, typing.Any]:
             "connection": self.connection,
             "required": self.required,
             "multiple": self.multiple,
+            "directory": self.directory,
         }
 
     def _validate_single(self, value: typing.Any):
@@ -622,6 +635,7 @@ def parameter(
     default: typing.Any | None = None,
     required: bool = True,
     multiple: bool = False,
+    directory: str | None = None,
 ):
     """Decorate a pipeline function by attaching a parameter to it..
 
@@ -651,6 +665,8 @@ def parameter(
     multiple : bool, default=True
         Whether this parameter should be provided multiple values (if True, the value must be provided as a list of
         values of the chosen type)
+    directory : str, optional
+        An optional parameter to force file selection to specific directory (only used for parater type File). If the directory does not exist, it will be ignored.
 
     Returns
     -------
@@ -673,6 +689,7 @@ def decorator(fun):
                 widget=widget,
                 connection=connection,
                 multiple=multiple,
+                directory=directory,
             ),
         )
 

openhexa/sdk/pipelines/runtime.py

@@ -294,6 +294,7 @@ def get_pipeline(pipeline_path: Path) -> Pipeline:
                     Argument("connection", [ast.Constant]),
                     Argument("required", [ast.Constant], default_value=True),
                     Argument("multiple", [ast.Constant], default_value=False),
+                    Argument("directory", [ast.Constant]),
                 ),
             )