diff --git a/.gitignore b/.gitignore index 229853e..10350df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +__pycache__/ build/ .env .envlocal diff --git a/script/migrate_lib/__init__.py b/script/migrate_lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/script/migrate_lib/connections.py b/script/migrate_lib/connections.py new file mode 100644 index 0000000..c70f067 --- /dev/null +++ b/script/migrate_lib/connections.py @@ -0,0 +1,141 @@ +"""Connection migration: copy every workspace connection to the target. + +Secret field values come through the source API only because we authenticate +as a Django superuser: the ``value`` resolver redacts secret fields unless the +caller has ``workspaces.update_connection``, which a superuser short-circuits +to True. If a secret still comes back empty, we create the connection anyway +and warn — the user must set that secret manually on the target. + +Connection slugs are preserved: unlike createWorkspace / createPipeline (which +re-derive the slug/code server-side), createConnection honors a caller-supplied +slug and only suffixes it on collision. The target workspace is fresh, so the +source slug carries over intact, keeping pipeline parameters that reference a +connection by slug valid. +""" + +import sys +from dataclasses import dataclass, field +from typing import Any + +from openhexa.graphql.graphql_client.client import Client +from openhexa.graphql.graphql_client.input_types import ( + ConnectionFieldInput, + ConnectionType, + CreateConnectionInput, +) + +from .transport import GraphQLError, gql + + +# `workspace.connections` is a (non-paginated) field on Workspace returning the +# full list; the SDK's workspace() doesn't pull fields, so we query it raw. +LIST_CONNECTIONS_QUERY = """ +query ListConnections($slug: String!) { + workspace(slug: $slug) { + connections { + id name slug description type + fields { code value secret } + } + } +} +""" + + +@dataclass +class ConnectionsResult: + """What migrate_all() did, for the orchestrator to print.""" + + created: list[tuple[str, int]] = field(default_factory=list) + """(slug, field_count) for each connection created on target.""" + + skipped: list[str] = field(default_factory=list) + """Slugs that already existed on target.""" + + failed: list[str] = field(default_factory=list) + """Slugs whose creation failed; user must handle manually.""" + + warnings: list[str] = field(default_factory=list) + """Human-readable warnings (e.g. secret fields created with no value).""" + + +def _list_connections(client: Client, slug: str) -> list[dict[str, Any]] | None: + """Return the workspace's connections, or None if the workspace is absent.""" + data = gql(client, LIST_CONNECTIONS_QUERY, {"slug": slug}, "ListConnections") + ws = data["workspace"] + if ws is None: + return None + return list(ws["connections"]) + + +def _build_fields( + conn: dict[str, Any], result: ConnectionsResult +) -> list[ConnectionFieldInput]: + """Map source fields to ConnectionFieldInput, warning on empty secrets.""" + fields_in: list[ConnectionFieldInput] = [] + for f in conn.get("fields") or []: + value = f.get("value") + if f.get("secret") and not value: + result.warnings.append( + f"connection '{conn['slug']}' field '{f['code']}' is a secret " + "with no readable value on source — created empty; set it " + "manually on the target." + ) + fields_in.append( + ConnectionFieldInput( + code=f["code"], secret=bool(f.get("secret")), value=value + ) + ) + return fields_in + + +def migrate_all( + source: Client, + target: Client, + source_slug: str, + target_slug: str, +) -> ConnectionsResult: + """Copy every connection from `source_slug` into `target_slug`.""" + result = ConnectionsResult() + + print("=> Listing source connections ...") + conns = _list_connections(source, source_slug) + if conns is None: + raise GraphQLError( + f"source workspace '{source_slug}' not found while listing connections" + ) + print(f" found {len(conns)} connection(s)") + + existing = {c["slug"] for c in (_list_connections(target, target_slug) or [])} + + for conn in conns: + slug = conn["slug"] + if slug in existing: + print(f" - connection '{slug}' already exists on target — skipping") + result.skipped.append(slug) + continue + try: + print(f" - migrating connection '{slug}' ...") + fields_in = _build_fields(conn, result) + res = target.create_connection( + input=CreateConnectionInput( + workspace_slug=target_slug, + name=conn["name"], + slug=slug, + type=ConnectionType(conn["type"]), + description=conn.get("description") or "", + fields=fields_in, + ) + ) + if not res.success or res.connection is None: + raise GraphQLError( + f"createConnection failed for '{slug}': " + + ",".join(e.value for e in (res.errors or [])) + ) + result.created.append((slug, len(fields_in))) + except GraphQLError as exc: + # Collect and continue (like files.py) so one bad connection + # doesn't abort the rest of the migration. + print(f"\tFAILED to migrate connection '{slug}': {exc}", file=sys.stderr) + result.failed.append(slug) + + return result diff --git a/script/migrate_lib/files.py b/script/migrate_lib/files.py new file mode 100644 index 0000000..b43b293 --- /dev/null +++ b/script/migrate_lib/files.py @@ -0,0 +1,189 @@ +"""Workspace file transfer: list / download / upload, plus full bucket migration.""" + +import sys +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import Any + +import httpx +from openhexa.graphql.graphql_client.client import Client + +from .transport import GraphQLError, _dbg, gql + + +OBJECTS_PAGE_SIZE = 100 + + +PREPARE_DOWNLOAD_MUTATION = """ +mutation PrepareDownload($input: PrepareObjectDownloadInput!) { + prepareObjectDownload(input: $input) { + success errors downloadUrl + } +} +""" + +PREPARE_UPLOAD_MUTATION = """ +mutation PrepareUpload($input: PrepareObjectUploadInput!) { + prepareObjectUpload(input: $input) { + success errors uploadUrl headers + } +} +""" + +LIST_OBJECTS_QUERY = """ +query ListObjects($slug: String!, $prefix: String, $page: Int!, $perPage: Int!) { + workspace(slug: $slug) { + bucket { + objects(prefix: $prefix, page: $page, perPage: $perPage, ignoreHiddenFiles: false) { + hasNextPage + items { + key name path size type + } + } + } + } +} +""" + + +@dataclass +class FilesResult: + """What migrate_all() did, for the orchestrator to print.""" + + copied: list[tuple[str, int]] = field(default_factory=list) + """(object_key, byte_size) for each file copied to target.""" + + failed: list[str] = field(default_factory=list) + """Object keys whose download or upload failed; user must handle manually.""" + + +def download(source: Client, ws_slug: str, file_path: str) -> bytes: + """Download a file from the source workspace via a presigned URL.""" + data = gql( + source, + PREPARE_DOWNLOAD_MUTATION, + { + "input": { + "workspaceSlug": ws_slug, + "objectKey": file_path, + "forceAttachment": False, + } + }, + "PrepareDownload", + ) + result = data["prepareObjectDownload"] + if not result["success"] or not result.get("downloadUrl"): + raise GraphQLError( + f"prepareObjectDownload failed for '{file_path}': " + + ",".join(result.get("errors") or []) + ) + url = result["downloadUrl"] + _dbg(f"download {file_path} <- {url}") + with httpx.Client(timeout=300) as c: + resp = c.get(url) + if not resp.is_success: + raise GraphQLError( + f"download of '{file_path}' returned HTTP {resp.status_code}: " + f"{resp.text[:500]}" + ) + return resp.content + + +def upload( + target: Client, + ws_slug: str, + file_path: str, + content: bytes, + content_type: str = "application/octet-stream", +) -> None: + """Upload bytes to the target workspace at the given object key.""" + data = gql( + target, + PREPARE_UPLOAD_MUTATION, + { + "input": { + "workspaceSlug": ws_slug, + "objectKey": file_path, + "contentType": content_type, + } + }, + "PrepareUpload", + ) + result = data["prepareObjectUpload"] + if not result["success"] or not result.get("uploadUrl"): + raise GraphQLError( + f"prepareObjectUpload failed for '{file_path}': " + + ",".join(result.get("errors") or []) + ) + url = result["uploadUrl"] + headers = dict(result.get("headers") or {}) + headers.setdefault("Content-Type", content_type) + _dbg(f"upload {file_path} ({len(content)} bytes) -> {url}") + with httpx.Client(timeout=300) as c: + resp = c.put(url, content=content, headers=headers) + if not resp.is_success: + raise GraphQLError( + f"upload of '{file_path}' returned HTTP {resp.status_code}: " + f"{resp.text[:500]}" + ) + + +def walk(client: Client, ws_slug: str, prefix: str = "") -> Iterator[dict[str, Any]]: + """Recursively yield FILE BucketObject dicts under `prefix`. + + The bucket.objects field is delimited (returns DIRECTORY entries rather + than recursing), so we walk each directory ourselves. + """ + # Buffer all directory entries at this level so we don't interleave + # recursive listings with this level's pagination. + subdirs: list[str] = [] + page = 1 + while True: + data = gql( + client, + LIST_OBJECTS_QUERY, + { + "slug": ws_slug, + "prefix": prefix or None, + "page": page, + "perPage": OBJECTS_PAGE_SIZE, + }, + "ListObjects", + ) + ws = data["workspace"] + if ws is None: + return + page_data = ws["bucket"]["objects"] + for obj in page_data["items"]: + if obj["type"] == "FILE": + yield obj + elif obj["type"] == "DIRECTORY": + subdirs.append(obj["key"]) + if not page_data["hasNextPage"]: + break + page += 1 + for sub in subdirs: + yield from walk(client, ws_slug, sub) + + +def migrate_all( + source: Client, target: Client, source_slug: str, target_slug: str +) -> FilesResult: + """Copy every file from the source workspace bucket to the target bucket.""" + result = FilesResult() + print("=> Listing source files ...") + for obj in walk(source, source_slug): + path = obj["key"] + size = obj.get("size") or 0 + try: + print(f" - copying '{path}' ({size} bytes) ...") + content = download(source, source_slug, path) + upload(target, target_slug, path, content) + result.copied.append((path, len(content))) + except GraphQLError: + # Surface the failure in the live log without dumping the (often + # HTML) server response. The full path goes into result.failed for + # the final summary so the user can re-attempt manually. + print(f"\tFAILED to copy '{path}'", file=sys.stderr) + result.failed.append(path) + return result diff --git a/script/migrate_lib/pipelines.py b/script/migrate_lib/pipelines.py new file mode 100644 index 0000000..f5fb6a4 --- /dev/null +++ b/script/migrate_lib/pipelines.py @@ -0,0 +1,336 @@ +"""Pipeline migration: list source pipelines, copy each (with versions) to target.""" + +import base64 +from dataclasses import dataclass, field +from typing import Any + +from openhexa.graphql.graphql_client.client import Client +from openhexa.graphql.graphql_client.input_types import CreatePipelineInput + +from .transport import GraphQLError, gql + + +PIPELINES_PAGE_SIZE = 50 +VERSIONS_PAGE_SIZE = 50 + + +PIPELINE_DETAIL_QUERY = """ +query PipelineDetail($id: UUID!, $vPage: Int!, $vPerPage: Int!) { + pipeline(id: $id) { + id code name description type functionalType notebookPath + schedule config webhookEnabled + tags { name } + scheduledPipelineVersion { id versionNumber } + versions(page: $vPage, perPage: $vPerPage) { + pageNumber + totalPages + items { + id versionNumber name description externalLink config timeout + zipfile + parameters { + code name type multiple required default help + widget connection choices + } + } + } + } +} +""" + +UPLOAD_PIPELINE_MUTATION = """ +mutation UploadPipeline($input: UploadPipelineInput!) { + uploadPipeline(input: $input) { + success errors details + pipelineVersion { id versionNumber versionName } + } +} +""" + +UPDATE_PIPELINE_MUTATION = """ +mutation UpdatePipeline($input: UpdatePipelineInput!) { + updatePipeline(input: $input) { + success errors + pipeline { id code } + } +} +""" + + +@dataclass +class PipelinesResult: + """What migrate_all() did, for the orchestrator to print.""" + + created: list[tuple[str, list[str]]] = field(default_factory=list) + """(pipeline_code, [version_name, ...]) for each pipeline created on target.""" + + skipped: list[str] = field(default_factory=list) + """Pipeline codes that already existed on target or could not be migrated.""" + + warnings: list[str] = field(default_factory=list) + """Human-readable warnings to print in the summary.""" + + +# --------------------------------------------------------------------------- +# Source fetch +# --------------------------------------------------------------------------- + + +def _list_source_ids(source: Client, slug: str) -> list[tuple[str, str]]: + """Return [(pipeline_id, pipeline_code), ...] across all pages.""" + pairs: list[tuple[str, str]] = [] + page = 1 + while True: + result = source.pipelines( + workspace_slug=slug, page=page, per_page=PIPELINES_PAGE_SIZE + ) + pairs.extend((str(item.id), item.code) for item in result.items) + if page >= result.total_pages or result.total_pages == 0: + break + page += 1 + return pairs + + +def _fetch_source_detail(source: Client, pipeline_id: str) -> dict[str, Any]: + """Fetch full pipeline data + all versions for one pipeline.""" + first = gql( + source, + PIPELINE_DETAIL_QUERY, + {"id": pipeline_id, "vPage": 1, "vPerPage": VERSIONS_PAGE_SIZE}, + "PipelineDetail", + ) + detail = first["pipeline"] + if detail is None: + raise GraphQLError(f"source pipeline id={pipeline_id} disappeared") + versions = list(detail["versions"]["items"]) + total_pages = detail["versions"]["totalPages"] + for vpage in range(2, total_pages + 1): + more = gql( + source, + PIPELINE_DETAIL_QUERY, + {"id": pipeline_id, "vPage": vpage, "vPerPage": VERSIONS_PAGE_SIZE}, + "PipelineDetail", + ) + versions.extend(more["pipeline"]["versions"]["items"]) + detail["versions"] = sorted(versions, key=lambda v: v["versionNumber"]) + return detail + + +# --------------------------------------------------------------------------- +# Target writes +# --------------------------------------------------------------------------- + + +def _create_on_target( + target: Client, target_slug: str, src_pipeline: dict[str, Any] +) -> tuple[str, str]: + """Create an empty pipeline on the target and return (id, code). + + The server (see ``Pipeline.objects.create_if_has_perm`` in + openhexa-app/.../pipelines/models.py) auto-generates the pipeline + code from ``slugify(name)`` with a collision suffix and rejects any + code the caller tries to pass. So we never pass a code; we always + read the actual one back from the response. + """ + is_notebook = src_pipeline.get("type") == "notebook" + tags = [t["name"] for t in (src_pipeline.get("tags") or [])] + create_input = CreatePipelineInput( + name=src_pipeline["name"] or src_pipeline["code"], + description=src_pipeline.get("description") or None, + workspace_slug=target_slug, + notebook_path=src_pipeline.get("notebookPath") if is_notebook else None, + functional_type=src_pipeline.get("functionalType"), + tags=tags or None, + ) + result = target.create_pipeline(input=create_input) + if not result.success or result.pipeline is None: + raise GraphQLError( + f"createPipeline failed for '{src_pipeline['code']}': " + + ",".join(e.value for e in (result.errors or [])) + ) + created_code = result.pipeline.code + + # createPipeline returns only the code; fetch by code to get the id + # needed for subsequent updatePipeline. + created = target.pipeline(workspace_slug=target_slug, pipeline_code=created_code) + if created is None: + raise GraphQLError(f"could not look up created pipeline '{created_code}'") + return str(created.id), created_code + + +def _clean_parameters(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Strip nulls that the input type doesn't accept verbatim.""" + cleaned = [] + for p in parameters: + item = {k: v for k, v in p.items() if v is not None} + cfile = item.get("choicesFromFile") + if cfile is not None: + item["choicesFromFile"] = {k: v for k, v in cfile.items() if v is not None} + cleaned.append(item) + return cleaned + + +def _upload_version( + target: Client, + workspace_slug: str, + pipeline_code: str, + version: dict[str, Any], +) -> dict[str, Any]: + """Upload one version to an existing target pipeline.""" + try: + base64.b64decode(version["zipfile"], validate=True) + except Exception as exc: + raise GraphQLError( + f"version {version['versionNumber']} has invalid base64 zipfile: {exc}" + ) + + input_: dict[str, Any] = { + "workspaceSlug": workspace_slug, + "pipelineCode": pipeline_code, + "name": version.get("name"), + "description": version.get("description"), + "externalLink": version.get("externalLink"), + "parameters": _clean_parameters(version.get("parameters") or []), + "zipfile": version["zipfile"], + "config": version.get("config") or {}, + "timeout": version.get("timeout"), + } + input_ = {k: v for k, v in input_.items() if v is not None} + data = gql(target, UPLOAD_PIPELINE_MUTATION, {"input": input_}, "UploadPipeline") + result = data["uploadPipeline"] + if not result["success"]: + raise GraphQLError( + f"uploadPipeline failed for {pipeline_code} v{version['versionNumber']}: " + + ",".join(result.get("errors") or []) + + (f" ({result['details']})" if result.get("details") else "") + ) + return result["pipelineVersion"] + + +def _update_settings( + target: Client, + target_pipeline_id: str, + src_pipeline: dict[str, Any], + scheduled_version_id: str | None, +) -> None: + """Apply pipeline-level fields that createPipeline/uploadPipeline cannot set.""" + input_: dict[str, Any] = { + "id": target_pipeline_id, + "schedule": src_pipeline.get("schedule"), + "webhookEnabled": src_pipeline.get("webhookEnabled"), + "config": src_pipeline.get("config") or None, + "scheduledPipelineVersionId": scheduled_version_id, + "autoUpdateFromTemplate": False, + } + # Only call updatePipeline if at least one migrated field has a value. + meaningful = { + k: v + for k, v in input_.items() + if k != "id" + and k != "autoUpdateFromTemplate" + and v not in (None, False, {}, "") + } + if not meaningful: + return + input_ = {k: v for k, v in input_.items() if v is not None} + data = gql(target, UPDATE_PIPELINE_MUTATION, {"input": input_}, "UpdatePipeline") + result = data["updatePipeline"] + if not result["success"]: + raise GraphQLError( + f"updatePipeline failed for {src_pipeline['code']}: " + + ",".join(result.get("errors") or []) + ) + + +# --------------------------------------------------------------------------- +# Orchestration (one resource) +# --------------------------------------------------------------------------- + + +def migrate_all( + source: Client, + target: Client, + source_slug: str, + target_slug: str, +) -> PipelinesResult: + """Migrate every pipeline from `source_slug` into `target_slug`.""" + result = PipelinesResult() + + print("=> Listing source pipelines ...") + pairs = _list_source_ids(source, source_slug) + print(f" found {len(pairs)} pipeline(s)") + + for pipeline_id, src_code in pairs: + existing = target.pipeline(workspace_slug=target_slug, pipeline_code=src_code) + if existing is not None: + print(f" - pipeline '{src_code}' already exists on target — skipping") + result.skipped.append(src_code) + continue + + print(f" - migrating pipeline '{src_code}' ...") + detail = _fetch_source_detail(source, pipeline_id) + is_notebook = detail.get("type") == "notebook" + + if is_notebook and not detail.get("notebookPath"): + result.warnings.append( + f"notebook pipeline '{src_code}' has no notebookPath; skipped." + ) + result.skipped.append(src_code) + continue + + target_pid, target_code = _create_on_target(target, target_slug, detail) + if target_code != src_code: + print( + f" created pipeline as '{target_code}' (source was " + f"'{src_code}'; server re-derives code from name) — id={target_pid}" + ) + else: + print(f" created pipeline '{target_code}' (id={target_pid})") + + uploaded_names, scheduled_version_id = _upload_versions( + target, target_slug, target_code, detail, is_notebook, result + ) + + _update_settings(target, target_pid, detail, scheduled_version_id) + result.created.append((target_code, uploaded_names)) + + return result + + +def _upload_versions( + target: Client, + target_slug: str, + target_code: str, + detail: dict[str, Any], + is_notebook: bool, + result: PipelinesResult, +) -> tuple[list[str], str | None]: + """Upload all zipfile versions; return (uploaded_version_names, scheduled_version_id).""" + uploaded_names: list[str] = [] + scheduled_version_id: str | None = None + scheduled_src = detail.get("scheduledPipelineVersion") or {} + scheduled_src_number = scheduled_src.get("versionNumber") + + if is_notebook: + if detail.get("versions"): + result.warnings.append( + f"pipeline '{target_code}' has {len(detail['versions'])} version(s) " + "on the source, but uploadPipeline is not supported for " + "notebook pipelines — versions were not migrated." + ) + return uploaded_names, scheduled_version_id + + versions = detail.get("versions") or [] + if not versions: + result.warnings.append( + f"pipeline '{target_code}' has no versions on source; created with no version." + ) + for v in versions: + up = _upload_version(target, target_slug, target_code, v) + uploaded_names.append(up["versionName"]) + print(f" uploaded version v{v['versionNumber']} -> {up['versionName']}") + if ( + scheduled_src_number is not None + and up.get("versionNumber") == scheduled_src_number + ): + scheduled_version_id = up["id"] + return uploaded_names, scheduled_version_id diff --git a/script/migrate_lib/templates.py b/script/migrate_lib/templates.py new file mode 100644 index 0000000..ab2cae2 --- /dev/null +++ b/script/migrate_lib/templates.py @@ -0,0 +1,605 @@ +"""Pipeline template migration: server-wide templates copied to target. + +Templates are globally visible, so this runs once per target server, +independently of any workspace migration. Each source template is recreated +on the target by: + +1. Locating a host pipeline that can back the template on target: + - Prefer the *already-migrated* source pipeline (in its original + workspace on target) if it exists — keeps the underlying pipeline + visible inside its real workspace. + - Otherwise create a host pipeline inside a dedicated + "Template pipelines" workspace. +2. Ensuring each template version's underlying pipeline version exists on + the host pipeline (upload the source zipfile if missing). +3. Calling ``createPipelineTemplateVersion`` per missing template version, + in source ``versionNumber`` order. The first call also creates the + ``PipelineTemplate`` on target (via ``Pipeline.get_or_create_template``). +4. Applying metadata (description / functionalType / tags) via + ``updatePipelineTemplate``. + +Idempotent: matches templates by name (globally unique) and existing +versions by ``versionNumber``. +""" + +import sys +import time +from dataclasses import dataclass, field +from typing import Any + +from openhexa.graphql.graphql_client.client import Client +from openhexa.graphql.graphql_client.input_types import ( + CreatePipelineInput, + CreateWorkspaceInput, +) + +from .pipelines import _upload_version +from .transport import GraphQLError, gql + + +TEMPLATES_PAGE_SIZE = 50 +TEMPLATE_VERSIONS_PAGE_SIZE = 50 +PIPELINE_VERSIONS_PAGE_SIZE = 100 + +HOST_WORKSPACE_NAME = "Template pipelines" + +# Back-off between createPipelineTemplateVersion calls. Each one triggers +# auto-update of every dependent pipeline server-side, which can be heavy. +PER_VERSION_DELAY_SECONDS = 0.5 + + +LIST_SOURCE_TEMPLATES_QUERY = """ +query ListSourceTemplates($page: Int!, $perPage: Int!) { + pipelineTemplates(page: $page, perPage: $perPage) { + pageNumber totalPages totalItems + items { + id name code description functionalType + validatedAt + tags { name } + sourcePipeline { id code name workspace { slug } } + } + } +} +""" + +LIST_TARGET_TEMPLATES_QUERY = """ +query ListTargetTemplates($page: Int!, $perPage: Int!) { + pipelineTemplates(page: $page, perPage: $perPage) { + pageNumber totalPages + items { + id name code + sourcePipeline { id code workspace { slug } } + } + } +} +""" + +TEMPLATE_VERSIONS_QUERY = """ +query TemplateVersions($code: String!, $page: Int!, $perPage: Int!) { + templateByCode(code: $code) { + id name + versions(page: $page, perPage: $perPage) { + pageNumber totalPages + items { + id versionNumber changelog + sourcePipelineVersion { + id versionNumber name description externalLink config timeout + zipfile + parameters { + code name type multiple required default help + widget connection choices + } + } + } + } + } +} +""" + +TARGET_TEMPLATE_VERSIONS_QUERY = """ +query TargetTemplateVersions($code: String!, $page: Int!, $perPage: Int!) { + templateByCode(code: $code) { + id + versions(page: $page, perPage: $perPage) { + pageNumber totalPages + items { id versionNumber } + } + } +} +""" + +HOST_PIPELINE_VERSIONS_QUERY = """ +query HostPipelineVersions($id: UUID!, $page: Int!, $perPage: Int!) { + pipeline(id: $id) { + id + versions(page: $page, perPage: $perPage) { + pageNumber totalPages + items { id versionNumber } + } + } +} +""" + +CREATE_TEMPLATE_VERSION_MUTATION = """ +mutation CreateTemplateVersion($input: CreatePipelineTemplateVersionInput!) { + createPipelineTemplateVersion(input: $input) { + success errors + pipelineTemplate { id name code } + } +} +""" + +UPDATE_TEMPLATE_MUTATION = """ +mutation UpdateTemplate($input: UpdateTemplateInput!) { + updatePipelineTemplate(input: $input) { + success errors + template { id } + } +} +""" + + +@dataclass +class TemplatesResult: + """What migrate_all() did, for the orchestrator to print.""" + + created: list[str] = field(default_factory=list) + """Names of templates newly created on target (didn't exist before).""" + + versions_added: list[tuple[str, list[int]]] = field(default_factory=list) + """(template_name, [versionNumber, ...]) for versions added this run.""" + + skipped_unchanged: list[str] = field(default_factory=list) + """Names of templates that were already fully present on target.""" + + warnings: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Orchestration +# --------------------------------------------------------------------------- + + +def migrate_all(source: Client, target: Client) -> TemplatesResult: + """Migrate every template from `source` into `target`.""" + result = TemplatesResult() + + print(f"=> Ensuring '{HOST_WORKSPACE_NAME}' workspace exists on target ...") + host_ws_slug = _ensure_host_workspace(target) + print(f" host workspace slug: '{host_ws_slug}'") + + # Listed once so re-runs find host pipelines whose code the server + # re-derived from name (e.g. underscores -> dashes), not the source's + # raw code. + host_pipelines_by_name = _list_pipelines_by_name(target, host_ws_slug) + + print("=> Listing source templates ...") + src_templates = _list_all_source_templates(source) + print(f" found {len(src_templates)} template(s)") + + print("=> Listing existing target templates ...") + target_by_name = _list_all_target_templates(target) + print(f" target already has {len(target_by_name)} template(s)") + + for src in src_templates: + try: + _migrate_one( + source, + target, + src, + target_by_name, + host_ws_slug, + host_pipelines_by_name, + result, + ) + except GraphQLError as exc: + msg = f"template '{src['name']}' (code='{src['code']}'): {exc}" + result.warnings.append(msg) + print(f" FAILED: {msg}", file=sys.stderr) + + return result + + +def _migrate_one( + source: Client, + target: Client, + src_template: dict[str, Any], + target_by_name: dict[str, dict[str, Any]], + host_ws_slug: str, + host_pipelines_by_name: dict[str, str], + result: TemplatesResult, +) -> None: + name = src_template["name"] + code = src_template["code"] + print(f" - template '{name}' (code='{code}') ...") + + src_pipe = src_template.get("sourcePipeline") + if src_pipe is None: + result.warnings.append( + f"template '{name}' has no source pipeline on source — skipped." + ) + return + + existing_target = target_by_name.get(name) + + # 1. Decide host (workspace_slug, pipeline_code) and existing version numbers. + if existing_target is not None: + host_pipe_code = existing_target["sourcePipeline"]["code"] + host_pipe_ws = existing_target["sourcePipeline"]["workspace"]["slug"] + existing_version_numbers = _fetch_target_template_version_numbers(target, code) + print( + f" already exists on target at " + f"'{host_pipe_ws}/{host_pipe_code}'; " + f"{len(existing_version_numbers)} version(s) present" + ) + else: + host_pipe_code, host_pipe_ws = _resolve_or_create_host_pipeline( + target, src_pipe, host_ws_slug, host_pipelines_by_name + ) + existing_version_numbers = set() + + # 2. Resolve host pipeline id and the versions it already has (by number). + host_pipe = target.pipeline( + workspace_slug=host_pipe_ws, pipeline_code=host_pipe_code + ) + if host_pipe is None: + raise GraphQLError( + f"host pipeline '{host_pipe_ws}/{host_pipe_code}' could not be looked up" + ) + host_pipe_id = str(host_pipe.id) + host_versions_by_number = _fetch_pipeline_versions_by_number(target, host_pipe_id) + + # 3. Add missing template versions. + src_versions = _fetch_source_template_versions(source, code) + if not src_versions: + result.warnings.append( + f"template '{name}' has no versions on source — skipped." + ) + return + + added: list[int] = [] + target_template_id: str | None = ( + existing_target["id"] if existing_target is not None else None + ) + + for ver in src_versions: + vnum = ver["versionNumber"] + if vnum in existing_version_numbers: + continue + + src_pipe_ver = ver["sourcePipelineVersion"] + target_pv_id = _ensure_host_pipeline_version( + target, + host_pipe_ws, + host_pipe_code, + src_pipe_ver, + host_versions_by_number, + ) + + input_: dict[str, Any] = { + "workspaceSlug": host_pipe_ws, + "pipelineId": host_pipe_id, + "pipelineVersionId": target_pv_id, + "name": name, + "code": code, + "description": src_template.get("description") or "", + "changelog": ver.get("changelog") or None, + # versionName / documentation are not queryable on older source + # servers, so let the target extract documentation from the + # zipfile's README on its own (server default behavior). + } + input_ = {k: v for k, v in input_.items() if v is not None} + + data = gql( + target, + CREATE_TEMPLATE_VERSION_MUTATION, + {"input": input_}, + "CreateTemplateVersion", + ) + cptv = data["createPipelineTemplateVersion"] + if not cptv["success"]: + raise GraphQLError( + f"createPipelineTemplateVersion v{vnum} failed: " + + ",".join(cptv.get("errors") or []) + ) + if target_template_id is None and cptv.get("pipelineTemplate"): + target_template_id = cptv["pipelineTemplate"]["id"] + + print(f" added template version v{vnum}") + added.append(vnum) + time.sleep(PER_VERSION_DELAY_SECONDS) + + if added: + if existing_target is None: + result.created.append(name) + result.versions_added.append((name, added)) + else: + result.skipped_unchanged.append(name) + + # 4. Apply metadata (idempotent re-application is fine). + if target_template_id is not None: + _apply_metadata(target, src_template, target_template_id) + + # 5. Warn on validated-on-source. + if src_template.get("validatedAt"): + result.warnings.append( + f"template '{name}' is an official Bluesquare template on source; " + "on target it will appear as a community template " + "(validatedAt is not settable via the API)." + ) + + +# --------------------------------------------------------------------------- +# Host workspace & host pipeline +# --------------------------------------------------------------------------- + + +def _ensure_host_workspace(target: Client) -> str: + """Return the slug of an existing target workspace named HOST_WORKSPACE_NAME, creating it if needed.""" + page = 1 + while True: + page_result = target.workspaces(page=page, per_page=100) + for ws in page_result.items: + if ws.name == HOST_WORKSPACE_NAME: + return ws.slug + if page >= page_result.total_pages or page_result.total_pages == 0: + break + page += 1 + + created = target.create_workspace( + input=CreateWorkspaceInput( + name=HOST_WORKSPACE_NAME, + description=( + "Host workspace for source pipelines backing migrated templates. " + "Auto-created by migrate_templates.py." + ), + countries=[], + load_sample_data=False, + configuration={}, + ) + ) + if not created.success or created.workspace is None: + raise GraphQLError( + f"could not create '{HOST_WORKSPACE_NAME}' workspace: " + + ",".join(created.errors or []) + ) + return created.workspace.slug + + +def _resolve_or_create_host_pipeline( + target: Client, + src_pipeline: dict[str, Any], + host_ws_slug: str, + host_pipelines_by_name: dict[str, str], +) -> tuple[str, str]: + """Return (pipeline_code, workspace_slug) of the host pipeline to use. + + Reuses the source pipeline in its original workspace on target if it + exists there (because the workspace migration already brought it + over). Otherwise reuses or creates a host pipeline in + HOST_WORKSPACE_NAME, matched by source pipeline name (since the + server re-derives code from name on create, e.g. dropping + underscores). + """ + src_code = src_pipeline["code"] + src_name = src_pipeline["name"] or src_code + src_ws = src_pipeline.get("workspace") or {} + src_ws_slug = src_ws.get("slug") + + if src_ws_slug: + existing = target.pipeline(workspace_slug=src_ws_slug, pipeline_code=src_code) + if existing is not None: + print( + f" reusing already-migrated source pipeline " + f"'{src_ws_slug}/{existing.code}'" + ) + return existing.code, src_ws_slug + + # Fallback: ensure pipeline exists inside the dedicated host workspace. + if src_name in host_pipelines_by_name: + host_code = host_pipelines_by_name[src_name] + print(f" reusing existing host pipeline '{host_ws_slug}/{host_code}'") + return host_code, host_ws_slug + + create_result = target.create_pipeline( + input=CreatePipelineInput( + name=src_name, + workspace_slug=host_ws_slug, + ) + ) + if not create_result.success or create_result.pipeline is None: + raise GraphQLError( + f"createPipeline for template host '{src_code}' failed: " + + ",".join(e.value for e in (create_result.errors or [])) + ) + new_code = create_result.pipeline.code + host_pipelines_by_name[src_name] = new_code + print(f" created host pipeline '{host_ws_slug}/{new_code}'") + return new_code, host_ws_slug + + +def _ensure_host_pipeline_version( + target: Client, + ws_slug: str, + pipeline_code: str, + src_pipeline_version: dict[str, Any], + host_versions_by_number: dict[int, str], +) -> str: + """Return the target pipelineVersion id for the given source pipeline version. + + Reuses an existing version on the host pipeline if one with the same + versionNumber exists; otherwise uploads the source zipfile and + records the new id in `host_versions_by_number`. + """ + src_vnum = src_pipeline_version["versionNumber"] + if src_vnum in host_versions_by_number: + return host_versions_by_number[src_vnum] + + if not src_pipeline_version.get("zipfile"): + raise GraphQLError( + f"source pipeline version v{src_vnum} has no zipfile; cannot upload" + ) + + up = _upload_version(target, ws_slug, pipeline_code, src_pipeline_version) + print(f" uploaded host pipeline version v{src_vnum} -> {up['versionName']}") + host_versions_by_number[up["versionNumber"]] = up["id"] + return up["id"] + + +# --------------------------------------------------------------------------- +# Source / target fetch helpers +# --------------------------------------------------------------------------- + + +def _list_pipelines_by_name(target: Client, ws_slug: str) -> dict[str, str]: + """List pipelines in `ws_slug`, return {pipeline.name: pipeline.code}.""" + by_name: dict[str, str] = {} + page = 1 + while True: + result = target.pipelines(workspace_slug=ws_slug, page=page, per_page=100) + for item in result.items: + if item.name: + by_name[item.name] = item.code + if page >= result.total_pages or result.total_pages == 0: + break + page += 1 + return by_name + + +def _list_all_source_templates(source: Client) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + page = 1 + while True: + data = gql( + source, + LIST_SOURCE_TEMPLATES_QUERY, + {"page": page, "perPage": TEMPLATES_PAGE_SIZE}, + "ListSourceTemplates", + ) + page_data = data["pipelineTemplates"] + items.extend(page_data["items"]) + if page >= page_data["totalPages"] or page_data["totalPages"] == 0: + break + page += 1 + return items + + +def _list_all_target_templates(target: Client) -> dict[str, dict[str, Any]]: + """Return target templates keyed by name (globally unique).""" + by_name: dict[str, dict[str, Any]] = {} + page = 1 + while True: + data = gql( + target, + LIST_TARGET_TEMPLATES_QUERY, + {"page": page, "perPage": TEMPLATES_PAGE_SIZE}, + "ListTargetTemplates", + ) + page_data = data["pipelineTemplates"] + for item in page_data["items"]: + by_name[item["name"]] = item + if page >= page_data["totalPages"] or page_data["totalPages"] == 0: + break + page += 1 + return by_name + + +def _fetch_source_template_versions(source: Client, code: str) -> list[dict[str, Any]]: + versions: list[dict[str, Any]] = [] + page = 1 + while True: + data = gql( + source, + TEMPLATE_VERSIONS_QUERY, + {"code": code, "page": page, "perPage": TEMPLATE_VERSIONS_PAGE_SIZE}, + "TemplateVersions", + ) + t = data["templateByCode"] + if t is None: + return [] + pg = t["versions"] + versions.extend(pg["items"]) + if page >= pg["totalPages"] or pg["totalPages"] == 0: + break + page += 1 + return sorted(versions, key=lambda v: v["versionNumber"]) + + +def _fetch_target_template_version_numbers(target: Client, code: str) -> set[int]: + out: set[int] = set() + page = 1 + while True: + data = gql( + target, + TARGET_TEMPLATE_VERSIONS_QUERY, + {"code": code, "page": page, "perPage": TEMPLATE_VERSIONS_PAGE_SIZE}, + "TargetTemplateVersions", + ) + t = data["templateByCode"] + if t is None: + return out + pg = t["versions"] + for v in pg["items"]: + out.add(v["versionNumber"]) + if page >= pg["totalPages"] or pg["totalPages"] == 0: + break + page += 1 + return out + + +def _fetch_pipeline_versions_by_number( + target: Client, pipeline_id: str +) -> dict[int, str]: + out: dict[int, str] = {} + page = 1 + while True: + data = gql( + target, + HOST_PIPELINE_VERSIONS_QUERY, + {"id": pipeline_id, "page": page, "perPage": PIPELINE_VERSIONS_PAGE_SIZE}, + "HostPipelineVersions", + ) + p = data["pipeline"] + if p is None: + return out + pg = p["versions"] + for v in pg["items"]: + out[v["versionNumber"]] = v["id"] + if page >= pg["totalPages"] or pg["totalPages"] == 0: + break + page += 1 + return out + + +# --------------------------------------------------------------------------- +# Metadata +# --------------------------------------------------------------------------- + + +def _apply_metadata( + target: Client, src_template: dict[str, Any], target_template_id: str +) -> None: + tags = [t["name"] for t in (src_template.get("tags") or [])] + input_: dict[str, Any] = { + "id": target_template_id, + "description": src_template.get("description") or None, + "functionalType": src_template.get("functionalType") or None, + "tags": tags or None, + } + input_ = {k: v for k, v in input_.items() if v is not None} + if list(input_.keys()) == ["id"]: + return + + data = gql( + target, + UPDATE_TEMPLATE_MUTATION, + {"input": input_}, + "UpdateTemplate", + ) + upd = data["updatePipelineTemplate"] + if not upd["success"]: + print( + f" warning: updatePipelineTemplate failed for " + f"'{src_template['name']}': " + ",".join(upd.get("errors") or []), + file=sys.stderr, + ) diff --git a/script/migrate_lib/transport.py b/script/migrate_lib/transport.py new file mode 100644 index 0000000..05aa27c --- /dev/null +++ b/script/migrate_lib/transport.py @@ -0,0 +1,96 @@ +"""GraphQL transport: SDK-backed clients, error type, request wrapper, debug.""" + +import sys +from typing import Any + +import httpx +from openhexa.graphql.graphql_client.client import Client + + +# Toggled by the CLI's --debug / -v. Module-global so the gql() wrapper and the +# SDK-call helpers don't need to thread it through every signature. +DEBUG = False + + +def _dbg(msg: str) -> None: + if DEBUG: + sys.stderr.write(f"[debug] {msg}\n") + + +def _short(value: Any, limit: int = 200) -> str: + """Render a value for debug output without dumping huge zipfiles.""" + if isinstance(value, dict): + return ( + "{" + ", ".join(f"{k}={_short(v, limit)}" for k, v in value.items()) + "}" + ) + if isinstance(value, list): + head = ", ".join(_short(v, limit) for v in value[:3]) + return f"[{head}{', ...' if len(value) > 3 else ''}] (n={len(value)})" + s = repr(value) + return ( + s if len(s) <= limit else s[:limit] + f"... " + ) + + +class GraphQLError(RuntimeError): + pass + + +def gql( + client: Client, + query: str, + variables: dict[str, Any] | None = None, + operation_name: str | None = None, +) -> dict[str, Any]: + """Execute a raw GraphQL query through the SDK client and return data.""" + if DEBUG: + _dbg(f"-> {operation_name or ''} @ {client.url}") + _dbg(f" variables: {_short(variables or {})}") + resp = client.execute( + query=query, variables=variables or {}, operation_name=operation_name + ) + if DEBUG: + _dbg(f"<- HTTP {resp.status_code} ({len(resp.content)} bytes)") + if not resp.is_success: + # Surface the body — the SDK's __str__ only shows the status code, + # which loses the actual GraphQL/Django error. + raise GraphQLError( + f"{operation_name or ''} returned HTTP {resp.status_code}: " + f"{resp.text[:2000]}" + ) + return client.get_data(resp) + + +def build_client(server_url: str, email: str, password: str, *, label: str) -> Client: + """Authenticate against an OpenHEXA server via the GraphQL Login mutation. + + `label` is used only to make the error message ("source"/"target") clearer. + """ + # 120s read timeout: createPipelineTemplateVersion can fan out to + # auto-update every pipeline derived from the template, which is slow + # on prod and exceeds httpx's 5s default. + http = httpx.Client( + headers={"User-Agent": "openhexa-migrate/1.0"}, + timeout=httpx.Timeout(120.0), + ) + # Prime CSRF cookie. Defensive — GraphQLView is csrf_exempt on the + # current backend, but a future change would otherwise silently + # break every mutation. + http.get(server_url) + csrf = http.cookies.get("csrftoken") + if csrf: + http.headers["X-CSRFToken"] = csrf + http.headers["Referer"] = server_url + + client = Client(url=server_url, http_client=http) + data = gql( + client, + "mutation Login($input: LoginInput!) { login(input: $input) { success errors } }", + {"input": {"email": email, "password": password}}, + "Login", + ) + if not data["login"]["success"]: + raise GraphQLError( + f"{label} login failed: " + ",".join(data["login"]["errors"] or []) + ) + return client diff --git a/script/migrate_lib/workspaces.py b/script/migrate_lib/workspaces.py new file mode 100644 index 0000000..9910275 --- /dev/null +++ b/script/migrate_lib/workspaces.py @@ -0,0 +1,54 @@ +"""Workspace-level migration (workspace metadata; later: members, settings).""" + +import sys +from typing import Any + +from openhexa.graphql.graphql_client.client import Client +from openhexa.graphql.graphql_client.input_types import ( + CountryInput, + CreateWorkspaceInput, + UpdateWorkspaceInput, +) + +from .transport import GraphQLError + + +def create(target: Client, src_ws: Any, organization_id: str | None = None) -> str: + """Create the workspace on target, returning the slug the server picked. + + Note: the server (see resolve_create_workspace + create_workspace_slug + in openhexa-app) ignores any `slug` passed in the input — it derives + the slug from the name with a random suffix. So we never pass a slug + and always read the actual slug back from the response. + """ + countries = [ + CountryInput(code=c.code, alpha3=c.alpha_3, name=c.name, flag=c.flag) + for c in (src_ws.countries or []) + ] + result = target.create_workspace( + input=CreateWorkspaceInput( + name=src_ws.name, + description=src_ws.description or "", + countries=countries, + load_sample_data=False, + configuration=src_ws.configuration or {}, + organization_id=organization_id, + ) + ) + if not result.success or result.workspace is None: + raise GraphQLError("createWorkspace failed: " + ",".join(result.errors or [])) + created_slug = result.workspace.slug + + if src_ws.docker_image: + upd = target.update_workspace( + input=UpdateWorkspaceInput( + slug=created_slug, docker_image=src_ws.docker_image + ) + ) + if not upd.success: + print( + f" warning: could not set dockerImage=" + f"'{src_ws.docker_image}': " + ",".join(upd.errors or []), + file=sys.stderr, + ) + return created_slug diff --git a/script/migrate_templates.py b/script/migrate_templates.py new file mode 100755 index 0000000..004b691 --- /dev/null +++ b/script/migrate_templates.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Migrate all pipeline templates from one OpenHEXA server to another. + +Templates are globally visible on a server, so this script runs once per +target server (independently of any workspace migration). For each +source template it: + + - finds or creates a host pipeline on target (the source pipeline in + its original workspace if already migrated, otherwise inside a + "Template pipelines" workspace this script creates as needed), + - uploads any missing pipeline versions, + - calls ``createPipelineTemplateVersion`` per missing template version + (matched against target by name + ``versionNumber``), + - applies ``description`` / ``functionalType`` / ``tags`` via + ``updatePipelineTemplate``. + +Re-runnable. Auth: Django superuser email/password on both ends, same +shape as ``migrate_workspace.py``. + +Out of scope: ``validatedAt`` (not settable via the API; warned about); +relinking pipelines that were created from templates on source back to +those templates on target. +""" + +import argparse +import sys + +try: + from openhexa.graphql.graphql_client.exceptions import ( + GraphQLClientGraphQLMultiError, + GraphQLClientHttpError, + ) +except ImportError: + sys.stderr.write( + "error: this script requires the 'openhexa.sdk' package " + "(install with: pip install openhexa.sdk)\n" + ) + sys.exit(1) + +from migrate_lib import templates, transport +from migrate_lib.templates import TemplatesResult +from migrate_lib.transport import GraphQLError, build_client + + +DEFAULT_SOURCE_URL = "https://api.openhexa.org/graphql/" +DEFAULT_TARGET_URL = "http://localhost:8000/graphql/" + + +def _print_summary(result: TemplatesResult) -> None: + print("\n=== Template migration summary ===") + print(f"Templates created on target: {len(result.created)}") + for name in result.created: + print(f" * {name}") + if result.versions_added: + print( + f"Template versions added: {sum(len(v) for _, v in result.versions_added)}" + ) + for name, nums in result.versions_added: + nums_str = ", ".join(f"v{n}" for n in nums) + print(f" * {name}: {nums_str}") + if result.skipped_unchanged: + print( + f"Templates already up to date (skipped): {len(result.skipped_unchanged)}" + ) + for name in result.skipped_unchanged: + print(f" * {name}") + if result.warnings: + print("Warnings:") + for w in result.warnings: + print(f" - {w}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Migrate every pipeline template from a source " + "OpenHEXA server to a target server.", + ) + parser.add_argument( + "--source-url", + default=DEFAULT_SOURCE_URL, + help=f"Source GraphQL endpoint (default: {DEFAULT_SOURCE_URL}).", + ) + parser.add_argument( + "--source-email", + required=True, + help="Email of the Django superuser on the source server.", + ) + parser.add_argument( + "--source-password", + required=True, + help="Password of the Django superuser on the source server.", + ) + parser.add_argument( + "--target-url", + default=DEFAULT_TARGET_URL, + help=f"Target GraphQL endpoint (default: {DEFAULT_TARGET_URL}).", + ) + parser.add_argument( + "--target-email", + required=True, + help="Email of the Django superuser on the target server.", + ) + parser.add_argument( + "--target-password", + required=True, + help="Password of the Django superuser on the target server.", + ) + parser.add_argument( + "--debug", + "-v", + action="store_true", + help="Print each GraphQL request (operation + variables) and response status.", + ) + args = parser.parse_args() + + transport.DEBUG = args.debug + + print(f"Source: {args.source_url}") + print(f"Target: {args.target_url}") + + try: + source = build_client( + args.source_url, args.source_email, args.source_password, label="source" + ) + target = build_client( + args.target_url, args.target_email, args.target_password, label="target" + ) + result = templates.migrate_all(source, target) + _print_summary(result) + except GraphQLClientHttpError as exc: + sys.stderr.write( + f"\nerror: HTTP {exc.status_code} from server:\n{exc.response.text[:4000]}\n" + ) + return 1 + except GraphQLClientGraphQLMultiError as exc: + sys.stderr.write("\nerror: GraphQL errors from server:\n") + for e in exc.errors: + sys.stderr.write(f" - {e.message}") + if e.path: + sys.stderr.write(f" (at path: {'.'.join(map(str, e.path))})") + sys.stderr.write("\n") + return 1 + except GraphQLError as exc: + sys.stderr.write(f"\nerror: {exc}\n") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py new file mode 100755 index 0000000..86dc721 --- /dev/null +++ b/script/migrate_workspace.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Migrate a workspace from one OpenHEXA server to another. + +Auth (both source and target): Django superuser email/password passed via +CLI flags, exchanged for a session cookie via the GraphQL ``login`` mutation. + +Scope: + - Workspace metadata (name, description, dockerImage, configuration, + countries) is migrated. + - All files in the workspace bucket are copied from source to target + (must run before pipelines, since notebook pipelines require their + .ipynb to exist on the target before createPipeline succeeds). + - Connections (including secret field values) are recreated via + createConnection. Secret values come through only because we + authenticate as a superuser; any that come back empty are created + empty and flagged in the summary for manual entry. + - Each pipeline is created via createPipeline; each zipfile-pipeline + version is uploaded via uploadPipeline; pipeline-level schedule / + config / webhookEnabled / scheduledPipelineVersionId are applied via + updatePipeline. + +Out of scope: members, invitations, recipients, templates, runs, +shortcuts, datasets, database tables. +""" + +import argparse +import sys + +try: + from openhexa.graphql.graphql_client.exceptions import ( + GraphQLClientGraphQLMultiError, + GraphQLClientHttpError, + ) +except ImportError: + sys.stderr.write( + "error: this script requires the 'openhexa.sdk' package " + "(install with: pip install openhexa.sdk)\n" + ) + sys.exit(1) + +from migrate_lib import connections, files, pipelines, transport, workspaces +from migrate_lib.connections import ConnectionsResult +from migrate_lib.files import FilesResult +from migrate_lib.pipelines import PipelinesResult +from migrate_lib.transport import GraphQLError, build_client + + +DEFAULT_SOURCE_URL = "https://api.openhexa.org/graphql/" +DEFAULT_TARGET_URL = "http://localhost:8000/graphql/" + + +def migrate( + source, target, source_slug: str, target_organization_id: str | None = None +) -> None: + print(f"=> Fetching source workspace '{source_slug}' ...") + src_ws = source.workspace(slug=source_slug) + if src_ws is None: + raise GraphQLError(f"source workspace '{source_slug}' not found") + print(f" name: {src_ws.name!r}") + + print("=> Creating target workspace ...") + target_slug = workspaces.create(target, src_ws, target_organization_id) + print(f" created with slug '{target_slug}'") + if target_slug != source_slug: + print( + f" note: the server picked its own slug — '{target_slug}' " + f"instead of source slug '{source_slug}'. The createWorkspace " + "mutation always derives the slug from the workspace name." + ) + + files_result = files.migrate_all(source, target, source_slug, target_slug) + connections_result = connections.migrate_all( + source, target, source_slug, target_slug + ) + pipelines_result = pipelines.migrate_all(source, target, source_slug, target_slug) + + _print_summary( + src_ws.name, target_slug, files_result, connections_result, pipelines_result + ) + + +def _print_summary( + src_ws_name: str, + target_slug: str, + files_result: FilesResult, + connections_result: ConnectionsResult, + pipelines_result: PipelinesResult, +) -> None: + print("\n=== Migration summary ===") + print(f"Workspace: {src_ws_name!r} -> slug '{target_slug}'") + total_bytes = sum(b for _, b in files_result.copied) + print(f"Files copied: {len(files_result.copied)} ({total_bytes} bytes)") + if files_result.failed: + print( + f"Files that could NOT be migrated " + f"({len(files_result.failed)} — handle manually):" + ) + for path in files_result.failed: + print(f" * {path}") + print(f"Connections created: {len(connections_result.created)}") + for slug, n_fields in connections_result.created: + print(f" * {slug} ({n_fields} field(s))") + if connections_result.skipped: + print( + f"Connections skipped (already existed): {len(connections_result.skipped)}" + ) + for slug in connections_result.skipped: + print(f" * {slug}") + if connections_result.failed: + print( + f"Connections that could NOT be migrated " + f"({len(connections_result.failed)} — handle manually):" + ) + for slug in connections_result.failed: + print(f" * {slug}") + if connections_result.warnings: + print("Connection warnings:") + for w in connections_result.warnings: + print(f" - {w}") + print(f"Pipelines created: {len(pipelines_result.created)}") + for code, vnames in pipelines_result.created: + print(f" * {code}") + for vn in vnames: + print(f" - {vn}") + if pipelines_result.skipped: + print(f"Pipelines skipped (already existed): {len(pipelines_result.skipped)}") + for code in pipelines_result.skipped: + print(f" * {code}") + if pipelines_result.warnings: + print("Warnings:") + for w in pipelines_result.warnings: + print(f" - {w}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Migrate a workspace from a remote OpenHEXA server " + "to the local Docker setup.", + ) + parser.add_argument( + "--slug", + required=True, + help="Slug of the source workspace on the remote server.", + ) + parser.add_argument( + "--source-url", + default=DEFAULT_SOURCE_URL, + help=f"Source GraphQL endpoint (default: {DEFAULT_SOURCE_URL}).", + ) + parser.add_argument( + "--source-email", + required=True, + help="Email of the Django superuser on the source server.", + ) + parser.add_argument( + "--source-password", + required=True, + help="Password of the Django superuser on the source server.", + ) + parser.add_argument( + "--target-url", + default=DEFAULT_TARGET_URL, + help=f"Local GraphQL endpoint (default: {DEFAULT_TARGET_URL}).", + ) + parser.add_argument( + "--target-email", + required=True, + help="Email of the Django superuser on the target server.", + ) + parser.add_argument( + "--target-password", + required=True, + help="Password of the Django superuser on the target server.", + ) + parser.add_argument( + "--target-organization", + default=None, + help="Optional UUID of the organization on the target server to create " + "the workspace under. If omitted, the workspace is created without an " + "organization.", + ) + parser.add_argument( + "--debug", + "-v", + action="store_true", + help="Print each GraphQL request (operation + variables) and response status.", + ) + args = parser.parse_args() + + transport.DEBUG = args.debug + + print(f"Source: {args.source_url}") + print(f"Target: {args.target_url}") + + try: + source = build_client( + args.source_url, args.source_email, args.source_password, label="source" + ) + target = build_client( + args.target_url, args.target_email, args.target_password, label="target" + ) + migrate(source, target, args.slug, args.target_organization) + except GraphQLClientHttpError as exc: + # SDK's __str__ only includes the status code. Print the body too. + sys.stderr.write( + f"\nerror: HTTP {exc.status_code} from server:\n{exc.response.text[:4000]}\n" + ) + return 1 + except GraphQLClientGraphQLMultiError as exc: + sys.stderr.write("\nerror: GraphQL errors from server:\n") + for e in exc.errors: + sys.stderr.write(f" - {e.message}") + if e.path: + sys.stderr.write(f" (at path: {'.'.join(map(str, e.path))})") + sys.stderr.write("\n") + return 1 + except GraphQLError as exc: + sys.stderr.write(f"\nerror: {exc}\n") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())