From 4f26c59d7588e1d79993c357d22bc4026463c56b Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Fri, 22 May 2026 15:45:09 +0200 Subject: [PATCH 01/14] First attempt --- script/migrate_workspace.py | 651 ++++++++++++++++++++++++++++++++++++ 1 file changed, 651 insertions(+) create mode 100755 script/migrate_workspace.py diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py new file mode 100755 index 0000000..8add0e3 --- /dev/null +++ b/script/migrate_workspace.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +"""Migrate a workspace from a remote OpenHEXA server to the local Docker setup. + +Source auth: WorkspaceMembership.access_token (the same token format the +OpenHEXA CLI uses), sent as ``Authorization: Bearer ``. + +Target auth: Django superuser email/password from .env, exchanged for a +session cookie via the GraphQL ``login`` mutation. + +Scope: + - Workspace metadata (name, description, dockerImage, configuration, + countries) is migrated. + - All pipelines and all their versions (zipfile bytes + parameters + + per-version metadata) are migrated. + - Pipeline-level fields (schedule, config, webhookEnabled, tags, + functionalType, scheduledPipelineVersion) are applied after upload. + +Out of scope: members, invitations, connections, recipients, templates, +runs, shortcuts, workspace files, datasets, database tables. +""" + +import argparse +import base64 +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urljoin, urlparse + +try: + import requests +except ImportError: + sys.stderr.write( + "error: this script requires the 'requests' package " + "(install with: pip install requests)\n" + ) + sys.exit(1) + + +DEFAULT_SOURCE_URL = "https://api.openhexa.org/graphql/" +PIPELINES_PAGE_SIZE = 50 +VERSIONS_PAGE_SIZE = 50 + + +# --------------------------------------------------------------------------- +# .env loading +# --------------------------------------------------------------------------- + +def parse_env_file(path: Path) -> dict[str, str]: + env: dict[str, str] = {} + for raw in path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + value = value.strip() + if (value.startswith("'") and value.endswith("'")) or ( + value.startswith('"') and value.endswith('"') + ): + value = value[1:-1] + env[key.strip()] = value + return env + + +# --------------------------------------------------------------------------- +# GraphQL client +# --------------------------------------------------------------------------- + +class GraphQLError(RuntimeError): + pass + + +class GraphQLClient: + def __init__(self, url: str, label: str): + self.url = url + self.label = label + self.session = requests.Session() + self.session.headers["Accept"] = "application/json" + + def _csrf_priming_get(self) -> None: + """Trigger Django to set the csrftoken cookie.""" + self.session.get(self.url, timeout=30) + + def _maybe_csrf_header(self) -> dict[str, str]: + token = self.session.cookies.get("csrftoken") + if not token: + return {} + return { + "X-CSRFToken": token, + "Referer": f"{urlparse(self.url).scheme}://{urlparse(self.url).netloc}/", + } + + def execute( + self, query: str, variables: dict[str, Any] | None = None + ) -> dict[str, Any]: + payload = {"query": query, "variables": variables or {}} + headers = {"Content-Type": "application/json", **self._maybe_csrf_header()} + resp = self.session.post( + self.url, data=json.dumps(payload), headers=headers, timeout=120 + ) + if resp.status_code != 200: + raise GraphQLError( + f"[{self.label}] HTTP {resp.status_code}: {resp.text[:500]}" + ) + body = resp.json() + if body.get("errors"): + raise GraphQLError( + f"[{self.label}] GraphQL errors: " + + json.dumps(body["errors"], indent=2) + ) + return body["data"] + + +def build_source_client(url: str, token: str) -> GraphQLClient: + client = GraphQLClient(url, label="source") + client.session.headers["Authorization"] = f"Bearer {token}" + # Bearer-auth POSTs don't need a CSRF token (no session cookie), but + # priming costs nothing and is harmless. + client._csrf_priming_get() + return client + + +def build_target_client(url: str, email: str, password: str) -> GraphQLClient: + client = GraphQLClient(url, label="target") + client._csrf_priming_get() + data = client.execute( + """ + mutation Login($input: LoginInput!) { + login(input: $input) { success errors } + } + """, + {"input": {"email": email, "password": password}}, + ) + result = data["login"] + if not result["success"]: + raise GraphQLError( + "target login failed: " + ",".join(result.get("errors") or []) + ) + return client + + +# --------------------------------------------------------------------------- +# Fetch from source +# --------------------------------------------------------------------------- + +WORKSPACE_QUERY = """ +query SourceWorkspace($slug: String!) { + workspace(slug: $slug) { + slug + name + description + dockerImage + configuration + countries { code alpha3 name flag } + } +} +""" + +PIPELINES_QUERY = """ +query SourcePipelines($slug: String!, $page: Int!, $perPage: Int!) { + pipelines(workspaceSlug: $slug, page: $page, perPage: $perPage) { + pageNumber + totalPages + totalItems + items { + id + code + name + description + type + functionalType + notebookPath + schedule + config + webhookEnabled + tags { name } + scheduledPipelineVersion { id versionNumber } + } + } +} +""" + +VERSIONS_QUERY = """ +query SourcePipelineVersions($id: UUID!, $page: Int!, $perPage: Int!) { + pipeline(id: $id) { + versions(page: $page, perPage: $perPage) { + pageNumber + totalPages + totalItems + items { + id + versionNumber + name + description + externalLink + config + timeout + zipfile + parameters { + code + name + type + multiple + required + default + help + widget + connection + choices + choicesFromFile { path format column } + directory + } + } + } + } +} +""" + + +def fetch_source_workspace(client: GraphQLClient, slug: str) -> dict[str, Any]: + data = client.execute(WORKSPACE_QUERY, {"slug": slug}) + if data["workspace"] is None: + raise GraphQLError(f"source workspace '{slug}' not found") + return data["workspace"] + + +def fetch_source_pipelines( + client: GraphQLClient, slug: str +) -> list[dict[str, Any]]: + pipelines: list[dict[str, Any]] = [] + page = 1 + while True: + data = client.execute( + PIPELINES_QUERY, + {"slug": slug, "page": page, "perPage": PIPELINES_PAGE_SIZE}, + ) + result = data["pipelines"] + pipelines.extend(result["items"]) + if page >= result["totalPages"] or result["totalPages"] == 0: + break + page += 1 + return pipelines + + +def fetch_pipeline_versions( + client: GraphQLClient, pipeline_id: str +) -> list[dict[str, Any]]: + versions: list[dict[str, Any]] = [] + page = 1 + while True: + data = client.execute( + VERSIONS_QUERY, + {"id": pipeline_id, "page": page, "perPage": VERSIONS_PAGE_SIZE}, + ) + result = data["pipeline"]["versions"] + versions.extend(result["items"]) + if page >= result["totalPages"] or result["totalPages"] == 0: + break + page += 1 + # Upload oldest first so versionNumber order is preserved locally. + versions.sort(key=lambda v: v["versionNumber"]) + return versions + + +# --------------------------------------------------------------------------- +# Apply to target +# --------------------------------------------------------------------------- + +TARGET_WORKSPACE_QUERY = """ +query TargetWorkspace($slug: String!) { + workspace(slug: $slug) { slug } +} +""" + +TARGET_PIPELINE_QUERY = """ +query TargetPipeline($slug: String!, $code: String!) { + pipelineByCode(workspaceSlug: $slug, code: $code) { id code } +} +""" + +CREATE_WORKSPACE = """ +mutation CreateWorkspace($input: CreateWorkspaceInput!) { + createWorkspace(input: $input) { + success errors + workspace { slug name } + } +} +""" + +UPDATE_WORKSPACE = """ +mutation UpdateWorkspace($input: UpdateWorkspaceInput!) { + updateWorkspace(input: $input) { + success errors + workspace { slug } + } +} +""" + +UPLOAD_PIPELINE = """ +mutation UploadPipeline($input: UploadPipelineInput!) { + uploadPipeline(input: $input) { + success errors details + pipelineVersion { id versionNumber versionName } + } +} +""" + +UPDATE_PIPELINE = """ +mutation UpdatePipeline($input: UpdatePipelineInput!) { + updatePipeline(input: $input) { + success errors + pipeline { id code name } + } +} +""" + + +def target_workspace_exists(client: GraphQLClient, slug: str) -> bool: + try: + data = client.execute(TARGET_WORKSPACE_QUERY, {"slug": slug}) + except GraphQLError: + return False + return data.get("workspace") is not None + + +def target_pipeline_get( + client: GraphQLClient, slug: str, code: str +) -> dict[str, Any] | None: + try: + data = client.execute( + TARGET_PIPELINE_QUERY, {"slug": slug, "code": code} + ) + except GraphQLError: + return None + return data.get("pipelineByCode") + + +def create_target_workspace( + client: GraphQLClient, src_ws: dict[str, Any], target_slug: str +) -> dict[str, Any]: + countries = [ + { + "code": c["code"], + "alpha3": c.get("alpha3"), + "name": c.get("name"), + "flag": c.get("flag"), + } + for c in (src_ws.get("countries") or []) + ] + input_ = { + "name": src_ws["name"], + "slug": target_slug, + "description": src_ws.get("description") or "", + "countries": countries, + "loadSampleData": False, + "configuration": src_ws.get("configuration") or {}, + } + data = client.execute(CREATE_WORKSPACE, {"input": input_}) + result = data["createWorkspace"] + if not result["success"]: + raise GraphQLError( + "createWorkspace failed: " + ",".join(result.get("errors") or []) + ) + created_slug = result["workspace"]["slug"] + + docker_image = src_ws.get("dockerImage") + if docker_image: + data = client.execute( + UPDATE_WORKSPACE, + { + "input": { + "slug": created_slug, + "dockerImage": docker_image, + } + }, + ) + upd = data["updateWorkspace"] + if not upd["success"]: + print( + f" warning: could not set dockerImage='{docker_image}': " + + ",".join(upd.get("errors") or []), + file=sys.stderr, + ) + return result["workspace"] + + +def _clean_parameters(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Strip nulls that the input type doesn't accept verbatim.""" + cleaned = [] + for p in parameters: + item = {k: v for k, v in p.items() if v is not None} + cfile = item.get("choicesFromFile") + if cfile is not None: + item["choicesFromFile"] = { + k: v for k, v in cfile.items() if v is not None + } + cleaned.append(item) + return cleaned + + +def upload_pipeline_version( + client: GraphQLClient, + workspace_slug: str, + pipeline_code: str, + version: dict[str, Any], + pipeline_meta: dict[str, Any], +) -> dict[str, Any]: + # zipfile from source is already base64-encoded (per schema). + # We forward it verbatim, but defensively re-validate it decodes. + try: + base64.b64decode(version["zipfile"], validate=True) + except Exception as exc: + raise GraphQLError( + f"version {version['versionNumber']} has invalid base64 zipfile: {exc}" + ) + + tags = [t["name"] for t in (pipeline_meta.get("tags") or [])] + input_: dict[str, Any] = { + "workspaceSlug": workspace_slug, + "pipelineCode": pipeline_code, + "name": version.get("name"), + "description": version.get("description"), + "externalLink": version.get("externalLink"), + "parameters": _clean_parameters(version.get("parameters") or []), + "zipfile": version["zipfile"], + "config": version.get("config") or {}, + "timeout": version.get("timeout"), + "tags": tags, + "functionalType": pipeline_meta.get("functionalType"), + } + input_ = {k: v for k, v in input_.items() if v is not None} + data = client.execute(UPLOAD_PIPELINE, {"input": input_}) + result = data["uploadPipeline"] + if not result["success"]: + raise GraphQLError( + f"uploadPipeline failed for {pipeline_code} v{version['versionNumber']}: " + + ",".join(result.get("errors") or []) + + (f" ({result.get('details')})" if result.get("details") else "") + ) + return result["pipelineVersion"] + + +def update_pipeline_after_upload( + client: GraphQLClient, + target_pipeline_id: str, + src_pipeline: dict[str, Any], + scheduled_version_id: str | None, +) -> None: + tags = [t["name"] for t in (src_pipeline.get("tags") or [])] + input_: dict[str, Any] = { + "id": target_pipeline_id, + "name": src_pipeline.get("name"), + "description": src_pipeline.get("description"), + "config": src_pipeline.get("config") or {}, + "schedule": src_pipeline.get("schedule"), + "webhookEnabled": src_pipeline.get("webhookEnabled"), + "autoUpdateFromTemplate": False, + "tags": tags, + "functionalType": src_pipeline.get("functionalType"), + "scheduledPipelineVersionId": scheduled_version_id, + } + input_ = {k: v for k, v in input_.items() if v is not None} + data = client.execute(UPDATE_PIPELINE, {"input": input_}) + result = data["updatePipeline"] + if not result["success"]: + raise GraphQLError( + f"updatePipeline failed for {src_pipeline['code']}: " + + ",".join(result.get("errors") or []) + ) + + +# --------------------------------------------------------------------------- +# Orchestration +# --------------------------------------------------------------------------- + +def migrate( + src: GraphQLClient, + dst: GraphQLClient, + source_slug: str, + target_slug: str, +) -> None: + print(f"=> Fetching source workspace '{source_slug}' ...") + src_ws = fetch_source_workspace(src, source_slug) + print(f" name: {src_ws['name']!r}") + + if target_workspace_exists(dst, target_slug): + raise SystemExit( + f"error: target workspace '{target_slug}' already exists; aborting." + ) + + print(f"=> Creating target workspace '{target_slug}' ...") + create_target_workspace(dst, src_ws, target_slug) + + print("=> Listing source pipelines ...") + src_pipelines = fetch_source_pipelines(src, source_slug) + print(f" found {len(src_pipelines)} pipeline(s)") + + created_summary: list[tuple[str, list[str]]] = [] + skipped: list[str] = [] + warnings: list[str] = [] + + for pipeline in src_pipelines: + code = pipeline["code"] + existing = target_pipeline_get(dst, target_slug, code) + if existing is not None: + msg = f"pipeline '{code}' already exists on target — skipping" + print(f" - {msg}") + skipped.append(code) + continue + + if pipeline.get("type") == "notebook": + warnings.append( + f"pipeline '{code}' is a notebook pipeline; its source file " + f"at notebookPath='{pipeline.get('notebookPath')}' must be " + "copied into the local workspace files for it to run." + ) + + print(f" - migrating pipeline '{code}' ({pipeline['name']!r})") + versions = fetch_pipeline_versions(src, pipeline["id"]) + if not versions: + warnings.append( + f"pipeline '{code}' has no versions on source; nothing uploaded." + ) + continue + + uploaded_versions: list[dict[str, Any]] = [] + scheduled_version_id: str | None = None + scheduled_src = pipeline.get("scheduledPipelineVersion") or {} + scheduled_src_number = scheduled_src.get("versionNumber") + + for v in versions: + up = upload_pipeline_version( + dst, target_slug, code, v, pipeline + ) + uploaded_versions.append(up) + print( + f" uploaded version v{v['versionNumber']} " + f"-> {up['versionName']}" + ) + if ( + scheduled_src_number is not None + and up.get("versionNumber") == scheduled_src_number + ): + scheduled_version_id = up["id"] + + target_pipeline = target_pipeline_get(dst, target_slug, code) + if target_pipeline is None: + raise GraphQLError( + f"could not find pipeline '{code}' on target after upload" + ) + update_pipeline_after_upload( + dst, target_pipeline["id"], pipeline, scheduled_version_id + ) + + created_summary.append( + (code, [v["versionName"] for v in uploaded_versions]) + ) + + # ----------- summary ----------- + print("\n=== Migration summary ===") + print(f"Workspace: {src_ws['name']!r} -> slug '{target_slug}'") + print(f"Pipelines created: {len(created_summary)}") + for code, vnames in created_summary: + print(f" * {code}") + for vn in vnames: + print(f" - {vn}") + if skipped: + print(f"Pipelines skipped (already existed): {len(skipped)}") + for code in skipped: + print(f" * {code}") + if warnings: + print("Warnings:") + for w in warnings: + print(f" - {w}") + print("Note: connections were not migrated; recreate them locally if " + "any pipeline depends on connection-typed parameters.") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> int: + parser = argparse.ArgumentParser( + description="Migrate a workspace from a remote OpenHEXA server " + "to the local Docker setup.", + ) + parser.add_argument( + "--token", required=True, + help="WorkspaceMembership access token from the source server " + "(same token used by the OpenHEXA CLI).", + ) + parser.add_argument( + "--slug", required=True, + help="Slug of the source workspace on the remote server.", + ) + parser.add_argument( + "--target-slug", + help="Slug to use locally (defaults to the source slug).", + ) + parser.add_argument( + "--source-url", default=DEFAULT_SOURCE_URL, + help=f"Source GraphQL endpoint (default: {DEFAULT_SOURCE_URL}).", + ) + parser.add_argument( + "--env-file", + default=str(Path(__file__).resolve().parent.parent / ".env"), + help="Path to the .env file (default: repo .env).", + ) + parser.add_argument( + "--target-url", + help="Override the local GraphQL endpoint (default derived from " + "APP_PORT in .env: http://localhost:${APP_PORT}/graphql/).", + ) + args = parser.parse_args() + + env_path = Path(args.env_file) + if not env_path.exists(): + sys.stderr.write(f"error: env file not found: {env_path}\n") + return 2 + env = parse_env_file(env_path) + + superuser = env.get("DJANGO_SUPERUSER_USERNAME") + password = env.get("DJANGO_SUPERUSER_PASSWORD") + if not superuser or not password: + sys.stderr.write( + "error: DJANGO_SUPERUSER_USERNAME and DJANGO_SUPERUSER_PASSWORD " + "must be set in the env file.\n" + ) + return 2 + + target_url = args.target_url or urljoin( + f"http://localhost:{env.get('APP_PORT', '8000')}/", "graphql/" + ) + target_slug = args.target_slug or args.slug + + print(f"Source: {args.source_url}") + print(f"Target: {target_url}") + + try: + src = build_source_client(args.source_url, args.token) + dst = build_target_client(target_url, superuser, password) + migrate(src, dst, args.slug, target_slug) + except GraphQLError as exc: + sys.stderr.write(f"\nerror: {exc}\n") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From f7ac87abc1fd25b8cf9a83202f31bea80f3507f7 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Fri, 22 May 2026 18:58:18 +0200 Subject: [PATCH 02/14] Add openhexa sdk to migration script --- script/migrate_workspace.py | 606 ++++++++++++++++-------------------- 1 file changed, 262 insertions(+), 344 deletions(-) diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index 8add0e3..60519e3 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -10,29 +10,42 @@ Scope: - Workspace metadata (name, description, dockerImage, configuration, countries) is migrated. - - All pipelines and all their versions (zipfile bytes + parameters + - per-version metadata) are migrated. - - Pipeline-level fields (schedule, config, webhookEnabled, tags, - functionalType, scheduledPipelineVersion) are applied after upload. + - Each pipeline is created via createPipeline; each zipfile-pipeline + version is uploaded via uploadPipeline; pipeline-level schedule / + config / webhookEnabled / scheduledPipelineVersionId are applied via + updatePipeline. Out of scope: members, invitations, connections, recipients, templates, -runs, shortcuts, workspace files, datasets, database tables. +runs, shortcuts, workspace files, datasets, database tables. Notebook +pipelines are created with their notebookPath but the notebook file +itself is not migrated (a warning is emitted). """ import argparse import base64 -import json import sys from pathlib import Path from typing import Any -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin try: - import requests + import httpx + from openhexa.graphql.graphql_client.client import Client + from openhexa.graphql.graphql_client.exceptions import ( + GraphQLClientGraphQLMultiError, + GraphQLClientHttpError, + ) + from openhexa.graphql.graphql_client.input_types import ( + CountryInput, + CreatePipelineInput, + CreateWorkspaceInput, + UpdateWorkspaceInput, + ) + from openhexa.sdk.utils import OpenHexaClient except ImportError: sys.stderr.write( - "error: this script requires the 'requests' package " - "(install with: pip install requests)\n" + "error: this script requires the 'openhexa.sdk' package " + "(install with: pip install openhexa.sdk)\n" ) sys.exit(1) @@ -46,6 +59,7 @@ # .env loading # --------------------------------------------------------------------------- + def parse_env_file(path: Path) -> dict[str, str]: env: dict[str, str] = {} for raw in path.read_text().splitlines(): @@ -63,150 +77,76 @@ def parse_env_file(path: Path) -> dict[str, str]: # --------------------------------------------------------------------------- -# GraphQL client +# GraphQL transport (SDK-backed) # --------------------------------------------------------------------------- + class GraphQLError(RuntimeError): pass -class GraphQLClient: - def __init__(self, url: str, label: str): - self.url = url - self.label = label - self.session = requests.Session() - self.session.headers["Accept"] = "application/json" - - def _csrf_priming_get(self) -> None: - """Trigger Django to set the csrftoken cookie.""" - self.session.get(self.url, timeout=30) - - def _maybe_csrf_header(self) -> dict[str, str]: - token = self.session.cookies.get("csrftoken") - if not token: - return {} - return { - "X-CSRFToken": token, - "Referer": f"{urlparse(self.url).scheme}://{urlparse(self.url).netloc}/", - } +def gql( + client: Client, + query: str, + variables: dict[str, Any] | None = None, + operation_name: str | None = None, +) -> dict[str, Any]: + """Execute a raw GraphQL query through the SDK client and return data.""" + resp = client.execute( + query=query, variables=variables or {}, operation_name=operation_name + ) + return client.get_data(resp) - def execute( - self, query: str, variables: dict[str, Any] | None = None - ) -> dict[str, Any]: - payload = {"query": query, "variables": variables or {}} - headers = {"Content-Type": "application/json", **self._maybe_csrf_header()} - resp = self.session.post( - self.url, data=json.dumps(payload), headers=headers, timeout=120 - ) - if resp.status_code != 200: - raise GraphQLError( - f"[{self.label}] HTTP {resp.status_code}: {resp.text[:500]}" - ) - body = resp.json() - if body.get("errors"): - raise GraphQLError( - f"[{self.label}] GraphQL errors: " - + json.dumps(body["errors"], indent=2) - ) - return body["data"] +def build_source(server_url: str, token: str) -> OpenHexaClient: + return OpenHexaClient(token=token, server_url=server_url) -def build_source_client(url: str, token: str) -> GraphQLClient: - client = GraphQLClient(url, label="source") - client.session.headers["Authorization"] = f"Bearer {token}" - # Bearer-auth POSTs don't need a CSRF token (no session cookie), but - # priming costs nothing and is harmless. - client._csrf_priming_get() - return client +def build_target(target_url: str, email: str, password: str) -> Client: + http = httpx.Client(headers={"User-Agent": "openhexa-migrate/1.0"}) + # Prime CSRF cookie. Defensive — GraphQLView is csrf_exempt on the + # current backend, but a future change would otherwise silently + # break every mutation. + http.get(target_url) + csrf = http.cookies.get("csrftoken") + if csrf: + http.headers["X-CSRFToken"] = csrf + http.headers["Referer"] = target_url -def build_target_client(url: str, email: str, password: str) -> GraphQLClient: - client = GraphQLClient(url, label="target") - client._csrf_priming_get() - data = client.execute( - """ - mutation Login($input: LoginInput!) { - login(input: $input) { success errors } - } - """, + client = Client(url=target_url, http_client=http) + data = gql( + client, + "mutation Login($input: LoginInput!) { login(input: $input) { success errors } }", {"input": {"email": email, "password": password}}, + "Login", ) - result = data["login"] - if not result["success"]: + if not data["login"]["success"]: raise GraphQLError( - "target login failed: " + ",".join(result.get("errors") or []) + "target login failed: " + ",".join(data["login"]["errors"] or []) ) return client # --------------------------------------------------------------------------- -# Fetch from source +# Source fetch helpers # --------------------------------------------------------------------------- -WORKSPACE_QUERY = """ -query SourceWorkspace($slug: String!) { - workspace(slug: $slug) { - slug - name - description - dockerImage - configuration - countries { code alpha3 name flag } - } -} -""" - -PIPELINES_QUERY = """ -query SourcePipelines($slug: String!, $page: Int!, $perPage: Int!) { - pipelines(workspaceSlug: $slug, page: $page, perPage: $perPage) { - pageNumber - totalPages - totalItems - items { - id - code - name - description - type - functionalType - notebookPath - schedule - config - webhookEnabled - tags { name } - scheduledPipelineVersion { id versionNumber } - } - } -} -""" - -VERSIONS_QUERY = """ -query SourcePipelineVersions($id: UUID!, $page: Int!, $perPage: Int!) { +PIPELINE_DETAIL_QUERY = """ +query PipelineDetail($id: UUID!, $vPage: Int!, $vPerPage: Int!) { pipeline(id: $id) { - versions(page: $page, perPage: $perPage) { + id code name description type functionalType notebookPath + schedule config webhookEnabled + tags { name } + scheduledPipelineVersion { id versionNumber } + versions(page: $vPage, perPage: $vPerPage) { pageNumber totalPages - totalItems items { - id - versionNumber - name - description - externalLink - config - timeout + id versionNumber name description externalLink config timeout zipfile parameters { - code - name - type - multiple - required - default - help - widget - connection - choices + code name type multiple required default help + widget connection choices choicesFromFile { path format column } directory } @@ -217,86 +157,55 @@ def build_target_client(url: str, email: str, password: str) -> GraphQLClient: """ -def fetch_source_workspace(client: GraphQLClient, slug: str) -> dict[str, Any]: - data = client.execute(WORKSPACE_QUERY, {"slug": slug}) - if data["workspace"] is None: - raise GraphQLError(f"source workspace '{slug}' not found") - return data["workspace"] - - -def fetch_source_pipelines( - client: GraphQLClient, slug: str -) -> list[dict[str, Any]]: - pipelines: list[dict[str, Any]] = [] +def fetch_source_pipeline_ids( + source: OpenHexaClient, slug: str +) -> list[tuple[str, str]]: + """Return [(pipeline_id, pipeline_code), ...] across all pages.""" + pairs: list[tuple[str, str]] = [] page = 1 while True: - data = client.execute( - PIPELINES_QUERY, - {"slug": slug, "page": page, "perPage": PIPELINES_PAGE_SIZE}, + result = source.pipelines( + workspace_slug=slug, page=page, per_page=PIPELINES_PAGE_SIZE ) - result = data["pipelines"] - pipelines.extend(result["items"]) - if page >= result["totalPages"] or result["totalPages"] == 0: + pairs.extend((str(item.id), item.code) for item in result.items) + if page >= result.total_pages or result.total_pages == 0: break page += 1 - return pipelines + return pairs -def fetch_pipeline_versions( - client: GraphQLClient, pipeline_id: str -) -> list[dict[str, Any]]: - versions: list[dict[str, Any]] = [] - page = 1 - while True: - data = client.execute( - VERSIONS_QUERY, - {"id": pipeline_id, "page": page, "perPage": VERSIONS_PAGE_SIZE}, +def fetch_source_pipeline_detail( + source: OpenHexaClient, pipeline_id: str +) -> dict[str, Any]: + """Fetch full pipeline data + all versions for one pipeline.""" + first = gql( + source, + PIPELINE_DETAIL_QUERY, + {"id": pipeline_id, "vPage": 1, "vPerPage": VERSIONS_PAGE_SIZE}, + "PipelineDetail", + ) + detail = first["pipeline"] + if detail is None: + raise GraphQLError(f"source pipeline id={pipeline_id} disappeared") + versions = list(detail["versions"]["items"]) + total_pages = detail["versions"]["totalPages"] + for vpage in range(2, total_pages + 1): + more = gql( + source, + PIPELINE_DETAIL_QUERY, + {"id": pipeline_id, "vPage": vpage, "vPerPage": VERSIONS_PAGE_SIZE}, + "PipelineDetail", ) - result = data["pipeline"]["versions"] - versions.extend(result["items"]) - if page >= result["totalPages"] or result["totalPages"] == 0: - break - page += 1 - # Upload oldest first so versionNumber order is preserved locally. - versions.sort(key=lambda v: v["versionNumber"]) - return versions + versions.extend(more["pipeline"]["versions"]["items"]) + detail["versions"] = sorted(versions, key=lambda v: v["versionNumber"]) + return detail # --------------------------------------------------------------------------- -# Apply to target +# Target apply helpers # --------------------------------------------------------------------------- -TARGET_WORKSPACE_QUERY = """ -query TargetWorkspace($slug: String!) { - workspace(slug: $slug) { slug } -} -""" - -TARGET_PIPELINE_QUERY = """ -query TargetPipeline($slug: String!, $code: String!) { - pipelineByCode(workspaceSlug: $slug, code: $code) { id code } -} -""" - -CREATE_WORKSPACE = """ -mutation CreateWorkspace($input: CreateWorkspaceInput!) { - createWorkspace(input: $input) { - success errors - workspace { slug name } - } -} -""" - -UPDATE_WORKSPACE = """ -mutation UpdateWorkspace($input: UpdateWorkspaceInput!) { - updateWorkspace(input: $input) { - success errors - workspace { slug } - } -} -""" - -UPLOAD_PIPELINE = """ +UPLOAD_PIPELINE_MUTATION = """ mutation UploadPipeline($input: UploadPipelineInput!) { uploadPipeline(input: $input) { success errors details @@ -305,83 +214,82 @@ def fetch_pipeline_versions( } """ -UPDATE_PIPELINE = """ +UPDATE_PIPELINE_MUTATION = """ mutation UpdatePipeline($input: UpdatePipelineInput!) { updatePipeline(input: $input) { success errors - pipeline { id code name } + pipeline { id code } } } """ -def target_workspace_exists(client: GraphQLClient, slug: str) -> bool: - try: - data = client.execute(TARGET_WORKSPACE_QUERY, {"slug": slug}) - except GraphQLError: - return False - return data.get("workspace") is not None - - -def target_pipeline_get( - client: GraphQLClient, slug: str, code: str -) -> dict[str, Any] | None: - try: - data = client.execute( - TARGET_PIPELINE_QUERY, {"slug": slug, "code": code} - ) - except GraphQLError: - return None - return data.get("pipelineByCode") - - -def create_target_workspace( - client: GraphQLClient, src_ws: dict[str, Any], target_slug: str -) -> dict[str, Any]: +def create_target_workspace(target: Client, src_ws: Any, target_slug: str) -> str: + """Create the workspace on target, returning the created slug.""" countries = [ - { - "code": c["code"], - "alpha3": c.get("alpha3"), - "name": c.get("name"), - "flag": c.get("flag"), - } - for c in (src_ws.get("countries") or []) + CountryInput(code=c.code, alpha3=c.alpha_3, name=c.name, flag=c.flag) + for c in (src_ws.countries or []) ] - input_ = { - "name": src_ws["name"], - "slug": target_slug, - "description": src_ws.get("description") or "", - "countries": countries, - "loadSampleData": False, - "configuration": src_ws.get("configuration") or {}, - } - data = client.execute(CREATE_WORKSPACE, {"input": input_}) - result = data["createWorkspace"] - if not result["success"]: - raise GraphQLError( - "createWorkspace failed: " + ",".join(result.get("errors") or []) + result = target.create_workspace( + input=CreateWorkspaceInput( + name=src_ws.name, + slug=target_slug, + description=src_ws.description or "", + countries=countries, + load_sample_data=False, + configuration=src_ws.configuration or {}, ) - created_slug = result["workspace"]["slug"] - - docker_image = src_ws.get("dockerImage") - if docker_image: - data = client.execute( - UPDATE_WORKSPACE, - { - "input": { - "slug": created_slug, - "dockerImage": docker_image, - } - }, + ) + if not result.success or result.workspace is None: + raise GraphQLError("createWorkspace failed: " + ",".join(result.errors or [])) + created_slug = result.workspace.slug + + if src_ws.docker_image: + upd = target.update_workspace( + input=UpdateWorkspaceInput( + slug=created_slug, docker_image=src_ws.docker_image + ) ) - upd = data["updateWorkspace"] - if not upd["success"]: + if not upd.success: print( - f" warning: could not set dockerImage='{docker_image}': " - + ",".join(upd.get("errors") or []), + f" warning: could not set dockerImage=" + f"'{src_ws.docker_image}': " + ",".join(upd.errors or []), file=sys.stderr, ) - return result["workspace"] + return created_slug + + +def create_target_pipeline( + target: Client, target_slug: str, src_pipeline: dict[str, Any] +) -> str: + """Create an empty pipeline on the target and return its UUID.""" + is_notebook = src_pipeline.get("type") == "notebook" + tags = [t["name"] for t in (src_pipeline.get("tags") or [])] + create_input = CreatePipelineInput( + name=src_pipeline["name"] or src_pipeline["code"], + description=src_pipeline.get("description") or None, + workspace_slug=target_slug, + notebook_path=src_pipeline.get("notebookPath") if is_notebook else None, + functional_type=src_pipeline.get("functionalType"), + tags=tags or None, + ) + result = target.create_pipeline(input=create_input) + if not result.success or result.pipeline is None: + raise GraphQLError( + f"createPipeline failed for '{src_pipeline['code']}': " + + ",".join(e.value for e in (result.errors or [])) + ) + + # createPipeline returns only the code; fetch by code to get the id + # needed for subsequent updatePipeline. + created = target.pipeline( + workspace_slug=target_slug, pipeline_code=result.pipeline.code + ) + if created is None: + raise GraphQLError( + f"could not look up created pipeline '{result.pipeline.code}'" + ) + return str(created.id) def _clean_parameters(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -391,22 +299,18 @@ def _clean_parameters(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]: item = {k: v for k, v in p.items() if v is not None} cfile = item.get("choicesFromFile") if cfile is not None: - item["choicesFromFile"] = { - k: v for k, v in cfile.items() if v is not None - } + item["choicesFromFile"] = {k: v for k, v in cfile.items() if v is not None} cleaned.append(item) return cleaned def upload_pipeline_version( - client: GraphQLClient, + target: Client, workspace_slug: str, pipeline_code: str, version: dict[str, Any], - pipeline_meta: dict[str, Any], ) -> dict[str, Any]: - # zipfile from source is already base64-encoded (per schema). - # We forward it verbatim, but defensively re-validate it decodes. + """Upload one version to an existing target pipeline.""" try: base64.b64decode(version["zipfile"], validate=True) except Exception as exc: @@ -414,7 +318,6 @@ def upload_pipeline_version( f"version {version['versionNumber']} has invalid base64 zipfile: {exc}" ) - tags = [t["name"] for t in (pipeline_meta.get("tags") or [])] input_: dict[str, Any] = { "workspaceSlug": workspace_slug, "pipelineCode": pipeline_code, @@ -425,42 +328,46 @@ def upload_pipeline_version( "zipfile": version["zipfile"], "config": version.get("config") or {}, "timeout": version.get("timeout"), - "tags": tags, - "functionalType": pipeline_meta.get("functionalType"), } input_ = {k: v for k, v in input_.items() if v is not None} - data = client.execute(UPLOAD_PIPELINE, {"input": input_}) + data = gql(target, UPLOAD_PIPELINE_MUTATION, {"input": input_}, "UploadPipeline") result = data["uploadPipeline"] if not result["success"]: raise GraphQLError( f"uploadPipeline failed for {pipeline_code} v{version['versionNumber']}: " + ",".join(result.get("errors") or []) - + (f" ({result.get('details')})" if result.get("details") else "") + + (f" ({result['details']})" if result.get("details") else "") ) return result["pipelineVersion"] -def update_pipeline_after_upload( - client: GraphQLClient, +def update_pipeline_settings( + target: Client, target_pipeline_id: str, src_pipeline: dict[str, Any], scheduled_version_id: str | None, ) -> None: - tags = [t["name"] for t in (src_pipeline.get("tags") or [])] + """Apply pipeline-level fields that createPipeline/uploadPipeline cannot set.""" input_: dict[str, Any] = { "id": target_pipeline_id, - "name": src_pipeline.get("name"), - "description": src_pipeline.get("description"), - "config": src_pipeline.get("config") or {}, "schedule": src_pipeline.get("schedule"), "webhookEnabled": src_pipeline.get("webhookEnabled"), - "autoUpdateFromTemplate": False, - "tags": tags, - "functionalType": src_pipeline.get("functionalType"), + "config": src_pipeline.get("config") or None, "scheduledPipelineVersionId": scheduled_version_id, + "autoUpdateFromTemplate": False, } + # Only call updatePipeline if at least one migrated field has a value. + meaningful = { + k: v + for k, v in input_.items() + if k != "id" + and k != "autoUpdateFromTemplate" + and v not in (None, False, {}, "") + } + if not meaningful: + return input_ = {k: v for k, v in input_.items() if v is not None} - data = client.execute(UPDATE_PIPELINE, {"input": input_}) + data = gql(target, UPDATE_PIPELINE_MUTATION, {"input": input_}, "UpdatePipeline") result = data["updatePipeline"] if not result["success"]: raise GraphQLError( @@ -473,92 +380,93 @@ def update_pipeline_after_upload( # Orchestration # --------------------------------------------------------------------------- + def migrate( - src: GraphQLClient, - dst: GraphQLClient, + source: OpenHexaClient, + target: Client, source_slug: str, target_slug: str, ) -> None: print(f"=> Fetching source workspace '{source_slug}' ...") - src_ws = fetch_source_workspace(src, source_slug) - print(f" name: {src_ws['name']!r}") + src_ws = source.workspace(slug=source_slug) + if src_ws is None: + raise GraphQLError(f"source workspace '{source_slug}' not found") + print(f" name: {src_ws.name!r}") - if target_workspace_exists(dst, target_slug): + if target.workspace(slug=target_slug) is not None: raise SystemExit( f"error: target workspace '{target_slug}' already exists; aborting." ) print(f"=> Creating target workspace '{target_slug}' ...") - create_target_workspace(dst, src_ws, target_slug) + create_target_workspace(target, src_ws, target_slug) print("=> Listing source pipelines ...") - src_pipelines = fetch_source_pipelines(src, source_slug) - print(f" found {len(src_pipelines)} pipeline(s)") + pairs = fetch_source_pipeline_ids(source, source_slug) + print(f" found {len(pairs)} pipeline(s)") created_summary: list[tuple[str, list[str]]] = [] skipped: list[str] = [] warnings: list[str] = [] - for pipeline in src_pipelines: - code = pipeline["code"] - existing = target_pipeline_get(dst, target_slug, code) + for pipeline_id, code in pairs: + existing = target.pipeline(workspace_slug=target_slug, pipeline_code=code) if existing is not None: - msg = f"pipeline '{code}' already exists on target — skipping" - print(f" - {msg}") + print(f" - pipeline '{code}' already exists on target — skipping") skipped.append(code) continue - if pipeline.get("type") == "notebook": - warnings.append( - f"pipeline '{code}' is a notebook pipeline; its source file " - f"at notebookPath='{pipeline.get('notebookPath')}' must be " - "copied into the local workspace files for it to run." - ) + print(f" - migrating pipeline '{code}' ...") + detail = fetch_source_pipeline_detail(source, pipeline_id) + is_notebook = detail.get("type") == "notebook" - print(f" - migrating pipeline '{code}' ({pipeline['name']!r})") - versions = fetch_pipeline_versions(src, pipeline["id"]) - if not versions: - warnings.append( - f"pipeline '{code}' has no versions on source; nothing uploaded." - ) - continue + target_pid = create_target_pipeline(target, target_slug, detail) + print(f" created pipeline (id={target_pid})") - uploaded_versions: list[dict[str, Any]] = [] + uploaded_names: list[str] = [] scheduled_version_id: str | None = None - scheduled_src = pipeline.get("scheduledPipelineVersion") or {} + scheduled_src = detail.get("scheduledPipelineVersion") or {} scheduled_src_number = scheduled_src.get("versionNumber") - for v in versions: - up = upload_pipeline_version( - dst, target_slug, code, v, pipeline - ) - uploaded_versions.append(up) - print( - f" uploaded version v{v['versionNumber']} " - f"-> {up['versionName']}" - ) - if ( - scheduled_src_number is not None - and up.get("versionNumber") == scheduled_src_number - ): - scheduled_version_id = up["id"] - - target_pipeline = target_pipeline_get(dst, target_slug, code) - if target_pipeline is None: - raise GraphQLError( - f"could not find pipeline '{code}' on target after upload" + if is_notebook: + warnings.append( + f"pipeline '{code}' is a notebook pipeline; its source file " + f"at notebookPath='{detail.get('notebookPath')}' must be " + "copied into the local workspace files for it to run." ) - update_pipeline_after_upload( - dst, target_pipeline["id"], pipeline, scheduled_version_id - ) - - created_summary.append( - (code, [v["versionName"] for v in uploaded_versions]) - ) + if detail.get("versions"): + warnings.append( + f"pipeline '{code}' has {len(detail['versions'])} version(s) " + "on the source, but uploadPipeline is not supported for " + "notebook pipelines — versions were not migrated." + ) + else: + versions = detail.get("versions") or [] + if not versions: + warnings.append( + f"pipeline '{code}' has no versions on source; " + "created with no version." + ) + for v in versions: + up = upload_pipeline_version(target, target_slug, code, v) + uploaded_names.append(up["versionName"]) + print( + f" uploaded version v{v['versionNumber']} " + f"-> {up['versionName']}" + ) + if ( + scheduled_src_number is not None + and up.get("versionNumber") == scheduled_src_number + ): + scheduled_version_id = up["id"] + + update_pipeline_settings(target, target_pid, detail, scheduled_version_id) + + created_summary.append((code, uploaded_names)) # ----------- summary ----------- print("\n=== Migration summary ===") - print(f"Workspace: {src_ws['name']!r} -> slug '{target_slug}'") + print(f"Workspace: {src_ws.name!r} -> slug '{target_slug}'") print(f"Pipelines created: {len(created_summary)}") for code, vnames in created_summary: print(f" * {code}") @@ -572,26 +480,31 @@ def migrate( print("Warnings:") for w in warnings: print(f" - {w}") - print("Note: connections were not migrated; recreate them locally if " - "any pipeline depends on connection-typed parameters.") + print( + "Note: connections were not migrated; recreate them locally if " + "any pipeline depends on connection-typed parameters." + ) # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- + def main() -> int: parser = argparse.ArgumentParser( description="Migrate a workspace from a remote OpenHEXA server " "to the local Docker setup.", ) parser.add_argument( - "--token", required=True, + "--token", + required=True, help="WorkspaceMembership access token from the source server " - "(same token used by the OpenHEXA CLI).", + "(same token used by the OpenHEXA CLI).", ) parser.add_argument( - "--slug", required=True, + "--slug", + required=True, help="Slug of the source workspace on the remote server.", ) parser.add_argument( @@ -599,7 +512,8 @@ def main() -> int: help="Slug to use locally (defaults to the source slug).", ) parser.add_argument( - "--source-url", default=DEFAULT_SOURCE_URL, + "--source-url", + default=DEFAULT_SOURCE_URL, help=f"Source GraphQL endpoint (default: {DEFAULT_SOURCE_URL}).", ) parser.add_argument( @@ -610,7 +524,7 @@ def main() -> int: parser.add_argument( "--target-url", help="Override the local GraphQL endpoint (default derived from " - "APP_PORT in .env: http://localhost:${APP_PORT}/graphql/).", + "APP_PORT in .env: http://localhost:${APP_PORT}/graphql/).", ) args = parser.parse_args() @@ -638,10 +552,14 @@ def main() -> int: print(f"Target: {target_url}") try: - src = build_source_client(args.source_url, args.token) - dst = build_target_client(target_url, superuser, password) - migrate(src, dst, args.slug, target_slug) - except GraphQLError as exc: + source = build_source(args.source_url, args.token) + target = build_target(target_url, superuser, password) + migrate(source, target, args.slug, target_slug) + except ( + GraphQLError, + GraphQLClientGraphQLMultiError, + GraphQLClientHttpError, + ) as exc: sys.stderr.write(f"\nerror: {exc}\n") return 1 return 0 From 0756e1550d23f7eb24be3331440692142f051eb5 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Fri, 22 May 2026 20:39:19 +0200 Subject: [PATCH 03/14] Add debugging and a few fixes --- script/migrate_workspace.py | 143 ++++++++++++++++++++++++++---------- 1 file changed, 103 insertions(+), 40 deletions(-) diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index 60519e3..83c6c45 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -54,6 +54,26 @@ PIPELINES_PAGE_SIZE = 50 VERSIONS_PAGE_SIZE = 50 +# Toggled by --debug / -v. Module-global so the gql() and SDK-call wrappers +# don't need to thread it through every signature. +DEBUG = False + + +def _dbg(msg: str) -> None: + if DEBUG: + sys.stderr.write(f"[debug] {msg}\n") + + +def _short(value: Any, limit: int = 200) -> str: + """Render a value for debug output without dumping huge zipfiles.""" + if isinstance(value, dict): + return "{" + ", ".join(f"{k}={_short(v, limit)}" for k, v in value.items()) + "}" + if isinstance(value, list): + head = ", ".join(_short(v, limit) for v in value[:3]) + return f"[{head}{', ...' if len(value) > 3 else ''}] (n={len(value)})" + s = repr(value) + return s if len(s) <= limit else s[:limit] + f"... " + # --------------------------------------------------------------------------- # .env loading @@ -92,9 +112,21 @@ def gql( operation_name: str | None = None, ) -> dict[str, Any]: """Execute a raw GraphQL query through the SDK client and return data.""" + if DEBUG: + _dbg(f"-> {operation_name or ''} @ {client.url}") + _dbg(f" variables: {_short(variables or {})}") resp = client.execute( query=query, variables=variables or {}, operation_name=operation_name ) + if DEBUG: + _dbg(f"<- HTTP {resp.status_code} ({len(resp.content)} bytes)") + if not resp.is_success: + # Surface the body — the SDK's __str__ only shows the status code, + # which loses the actual GraphQL/Django error. + raise GraphQLError( + f"{operation_name or ''} returned HTTP {resp.status_code}: " + f"{resp.text[:2000]}" + ) return client.get_data(resp) @@ -147,8 +179,6 @@ def build_target(target_url: str, email: str, password: str) -> Client: parameters { code name type multiple required default help widget connection choices - choicesFromFile { path format column } - directory } } } @@ -224,8 +254,14 @@ def fetch_source_pipeline_detail( """ -def create_target_workspace(target: Client, src_ws: Any, target_slug: str) -> str: - """Create the workspace on target, returning the created slug.""" +def create_target_workspace(target: Client, src_ws: Any) -> str: + """Create the workspace on target, returning the slug the server picked. + + Note: the server (see resolve_create_workspace + create_workspace_slug + in openhexa-app) ignores any `slug` passed in the input — it derives + the slug from the name with a random suffix. So we never pass a slug + and always read the actual slug back from the response. + """ countries = [ CountryInput(code=c.code, alpha3=c.alpha_3, name=c.name, flag=c.flag) for c in (src_ws.countries or []) @@ -233,7 +269,6 @@ def create_target_workspace(target: Client, src_ws: Any, target_slug: str) -> st result = target.create_workspace( input=CreateWorkspaceInput( name=src_ws.name, - slug=target_slug, description=src_ws.description or "", countries=countries, load_sample_data=False, @@ -261,8 +296,15 @@ def create_target_workspace(target: Client, src_ws: Any, target_slug: str) -> st def create_target_pipeline( target: Client, target_slug: str, src_pipeline: dict[str, Any] -) -> str: - """Create an empty pipeline on the target and return its UUID.""" +) -> tuple[str, str]: + """Create an empty pipeline on the target and return (id, code). + + The server (see ``Pipeline.objects.create_if_has_perm`` in + openhexa-app/.../pipelines/models.py) auto-generates the pipeline + code from ``slugify(name)`` with a collision suffix and rejects any + code the caller tries to pass. So we never pass a code; we always + read the actual one back from the response. + """ is_notebook = src_pipeline.get("type") == "notebook" tags = [t["name"] for t in (src_pipeline.get("tags") or [])] create_input = CreatePipelineInput( @@ -279,17 +321,18 @@ def create_target_pipeline( f"createPipeline failed for '{src_pipeline['code']}': " + ",".join(e.value for e in (result.errors or [])) ) + created_code = result.pipeline.code # createPipeline returns only the code; fetch by code to get the id # needed for subsequent updatePipeline. created = target.pipeline( - workspace_slug=target_slug, pipeline_code=result.pipeline.code + workspace_slug=target_slug, pipeline_code=created_code ) if created is None: raise GraphQLError( - f"could not look up created pipeline '{result.pipeline.code}'" + f"could not look up created pipeline '{created_code}'" ) - return str(created.id) + return str(created.id), created_code def _clean_parameters(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -385,7 +428,6 @@ def migrate( source: OpenHexaClient, target: Client, source_slug: str, - target_slug: str, ) -> None: print(f"=> Fetching source workspace '{source_slug}' ...") src_ws = source.workspace(slug=source_slug) @@ -393,14 +435,16 @@ def migrate( raise GraphQLError(f"source workspace '{source_slug}' not found") print(f" name: {src_ws.name!r}") - if target.workspace(slug=target_slug) is not None: - raise SystemExit( - f"error: target workspace '{target_slug}' already exists; aborting." + print("=> Creating target workspace ...") + target_slug = create_target_workspace(target, src_ws) + print(f" created with slug '{target_slug}'") + if target_slug != source_slug: + print( + f" note: the server picked its own slug — '{target_slug}' " + f"instead of source slug '{source_slug}'. The createWorkspace " + "mutation always derives the slug from the workspace name." ) - print(f"=> Creating target workspace '{target_slug}' ...") - create_target_workspace(target, src_ws, target_slug) - print("=> Listing source pipelines ...") pairs = fetch_source_pipeline_ids(source, source_slug) print(f" found {len(pairs)} pipeline(s)") @@ -409,19 +453,25 @@ def migrate( skipped: list[str] = [] warnings: list[str] = [] - for pipeline_id, code in pairs: - existing = target.pipeline(workspace_slug=target_slug, pipeline_code=code) + for pipeline_id, src_code in pairs: + existing = target.pipeline(workspace_slug=target_slug, pipeline_code=src_code) if existing is not None: - print(f" - pipeline '{code}' already exists on target — skipping") - skipped.append(code) + print(f" - pipeline '{src_code}' already exists on target — skipping") + skipped.append(src_code) continue - print(f" - migrating pipeline '{code}' ...") + print(f" - migrating pipeline '{src_code}' ...") detail = fetch_source_pipeline_detail(source, pipeline_id) is_notebook = detail.get("type") == "notebook" - target_pid = create_target_pipeline(target, target_slug, detail) - print(f" created pipeline (id={target_pid})") + target_pid, target_code = create_target_pipeline(target, target_slug, detail) + if target_code != src_code: + print( + f" created pipeline as '{target_code}' (source was " + f"'{src_code}'; server re-derives code from name) — id={target_pid}" + ) + else: + print(f" created pipeline '{target_code}' (id={target_pid})") uploaded_names: list[str] = [] scheduled_version_id: str | None = None @@ -430,13 +480,13 @@ def migrate( if is_notebook: warnings.append( - f"pipeline '{code}' is a notebook pipeline; its source file " + f"pipeline '{target_code}' is a notebook pipeline; its source file " f"at notebookPath='{detail.get('notebookPath')}' must be " "copied into the local workspace files for it to run." ) if detail.get("versions"): warnings.append( - f"pipeline '{code}' has {len(detail['versions'])} version(s) " + f"pipeline '{target_code}' has {len(detail['versions'])} version(s) " "on the source, but uploadPipeline is not supported for " "notebook pipelines — versions were not migrated." ) @@ -444,11 +494,11 @@ def migrate( versions = detail.get("versions") or [] if not versions: warnings.append( - f"pipeline '{code}' has no versions on source; " + f"pipeline '{target_code}' has no versions on source; " "created with no version." ) for v in versions: - up = upload_pipeline_version(target, target_slug, code, v) + up = upload_pipeline_version(target, target_slug, target_code, v) uploaded_names.append(up["versionName"]) print( f" uploaded version v{v['versionNumber']} " @@ -462,7 +512,7 @@ def migrate( update_pipeline_settings(target, target_pid, detail, scheduled_version_id) - created_summary.append((code, uploaded_names)) + created_summary.append((target_code, uploaded_names)) # ----------- summary ----------- print("\n=== Migration summary ===") @@ -507,10 +557,6 @@ def main() -> int: required=True, help="Slug of the source workspace on the remote server.", ) - parser.add_argument( - "--target-slug", - help="Slug to use locally (defaults to the source slug).", - ) parser.add_argument( "--source-url", default=DEFAULT_SOURCE_URL, @@ -526,8 +572,16 @@ def main() -> int: help="Override the local GraphQL endpoint (default derived from " "APP_PORT in .env: http://localhost:${APP_PORT}/graphql/).", ) + parser.add_argument( + "--debug", "-v", + action="store_true", + help="Print each GraphQL request (operation + variables) and response status.", + ) args = parser.parse_args() + global DEBUG + DEBUG = args.debug + env_path = Path(args.env_file) if not env_path.exists(): sys.stderr.write(f"error: env file not found: {env_path}\n") @@ -546,7 +600,6 @@ def main() -> int: target_url = args.target_url or urljoin( f"http://localhost:{env.get('APP_PORT', '8000')}/", "graphql/" ) - target_slug = args.target_slug or args.slug print(f"Source: {args.source_url}") print(f"Target: {target_url}") @@ -554,12 +607,22 @@ def main() -> int: try: source = build_source(args.source_url, args.token) target = build_target(target_url, superuser, password) - migrate(source, target, args.slug, target_slug) - except ( - GraphQLError, - GraphQLClientGraphQLMultiError, - GraphQLClientHttpError, - ) as exc: + migrate(source, target, args.slug) + except GraphQLClientHttpError as exc: + # SDK's __str__ only includes the status code. Print the body too. + sys.stderr.write( + f"\nerror: HTTP {exc.status_code} from server:\n{exc.response.text[:4000]}\n" + ) + return 1 + except GraphQLClientGraphQLMultiError as exc: + sys.stderr.write("\nerror: GraphQL errors from server:\n") + for e in exc.errors: + sys.stderr.write(f" - {e.message}") + if e.path: + sys.stderr.write(f" (at path: {'.'.join(map(str, e.path))})") + sys.stderr.write("\n") + return 1 + except GraphQLError as exc: sys.stderr.write(f"\nerror: {exc}\n") return 1 return 0 From 1d588495b8a8cd44b49026c26cad6d008f69498e Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Sat, 23 May 2026 15:22:19 +0200 Subject: [PATCH 04/14] Add file upload to support notebook pipeline migration --- script/migrate_workspace.py | 126 ++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index 83c6c45..c48656e 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -231,6 +231,100 @@ def fetch_source_pipeline_detail( return detail +# --------------------------------------------------------------------------- +# Workspace file transfer (used to migrate notebook files) +# --------------------------------------------------------------------------- + +PREPARE_DOWNLOAD_MUTATION = """ +mutation PrepareDownload($input: PrepareObjectDownloadInput!) { + prepareObjectDownload(input: $input) { + success errors downloadUrl + } +} +""" + +PREPARE_UPLOAD_MUTATION = """ +mutation PrepareUpload($input: PrepareObjectUploadInput!) { + prepareObjectUpload(input: $input) { + success errors uploadUrl headers + } +} +""" + + +def download_workspace_file( + source: OpenHexaClient, ws_slug: str, file_path: str +) -> bytes: + """Download a file from the source workspace via a presigned URL.""" + data = gql( + source, + PREPARE_DOWNLOAD_MUTATION, + { + "input": { + "workspaceSlug": ws_slug, + "objectKey": file_path, + "forceAttachment": False, + } + }, + "PrepareDownload", + ) + result = data["prepareObjectDownload"] + if not result["success"] or not result.get("downloadUrl"): + raise GraphQLError( + f"prepareObjectDownload failed for '{file_path}': " + + ",".join(result.get("errors") or []) + ) + url = result["downloadUrl"] + _dbg(f"download {file_path} <- {url}") + with httpx.Client(timeout=300) as c: + resp = c.get(url) + if not resp.is_success: + raise GraphQLError( + f"download of '{file_path}' returned HTTP {resp.status_code}: " + f"{resp.text[:500]}" + ) + return resp.content + + +def upload_workspace_file( + target: Client, + ws_slug: str, + file_path: str, + content: bytes, + content_type: str = "application/octet-stream", +) -> None: + """Upload bytes to the target workspace at the given object key.""" + data = gql( + target, + PREPARE_UPLOAD_MUTATION, + { + "input": { + "workspaceSlug": ws_slug, + "objectKey": file_path, + "contentType": content_type, + } + }, + "PrepareUpload", + ) + result = data["prepareObjectUpload"] + if not result["success"] or not result.get("uploadUrl"): + raise GraphQLError( + f"prepareObjectUpload failed for '{file_path}': " + + ",".join(result.get("errors") or []) + ) + url = result["uploadUrl"] + headers = dict(result.get("headers") or {}) + headers.setdefault("Content-Type", content_type) + _dbg(f"upload {file_path} ({len(content)} bytes) -> {url}") + with httpx.Client(timeout=300) as c: + resp = c.put(url, content=content, headers=headers) + if not resp.is_success: + raise GraphQLError( + f"upload of '{file_path}' returned HTTP {resp.status_code}: " + f"{resp.text[:500]}" + ) + + # --------------------------------------------------------------------------- # Target apply helpers # --------------------------------------------------------------------------- @@ -464,6 +558,38 @@ def migrate( detail = fetch_source_pipeline_detail(source, pipeline_id) is_notebook = detail.get("type") == "notebook" + if is_notebook: + # createPipeline for notebook pipelines requires the notebook + # file to already exist in the workspace bucket + # (openhexa-app pipelines/schema/mutations.py:82). Copy it + # over before creating the pipeline. + nbpath = detail.get("notebookPath") + if not nbpath: + warnings.append( + f"notebook pipeline '{src_code}' has no notebookPath; " + "skipped." + ) + skipped.append(src_code) + continue + try: + print(f" fetching notebook '{nbpath}' from source ...") + nb_bytes = download_workspace_file(source, source_slug, nbpath) + print(f" uploading notebook to target ({len(nb_bytes)} bytes) ...") + upload_workspace_file( + target, + target_slug, + nbpath, + nb_bytes, + content_type="application/x-ipynb+json", + ) + except GraphQLError as exc: + warnings.append( + f"notebook pipeline '{src_code}': could not migrate " + f"notebook file '{nbpath}' ({exc}); pipeline skipped." + ) + skipped.append(src_code) + continue + target_pid, target_code = create_target_pipeline(target, target_slug, detail) if target_code != src_code: print( From 575496aecc225a4457f98bd8df9ef88406cdd163 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Sat, 23 May 2026 15:25:23 +0200 Subject: [PATCH 05/14] Refac: Add arguments for target user/pass --- script/migrate_workspace.py | 68 +++++++++---------------------------- 1 file changed, 16 insertions(+), 52 deletions(-) diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index c48656e..f15dc66 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -4,8 +4,8 @@ Source auth: WorkspaceMembership.access_token (the same token format the OpenHEXA CLI uses), sent as ``Authorization: Bearer ``. -Target auth: Django superuser email/password from .env, exchanged for a -session cookie via the GraphQL ``login`` mutation. +Target auth: Django superuser email/password passed via CLI flags, +exchanged for a session cookie via the GraphQL ``login`` mutation. Scope: - Workspace metadata (name, description, dockerImage, configuration, @@ -24,9 +24,7 @@ import argparse import base64 import sys -from pathlib import Path from typing import Any -from urllib.parse import urljoin try: import httpx @@ -51,6 +49,7 @@ DEFAULT_SOURCE_URL = "https://api.openhexa.org/graphql/" +DEFAULT_TARGET_URL = "http://localhost:8000/graphql/" PIPELINES_PAGE_SIZE = 50 VERSIONS_PAGE_SIZE = 50 @@ -75,27 +74,6 @@ def _short(value: Any, limit: int = 200) -> str: return s if len(s) <= limit else s[:limit] + f"... " -# --------------------------------------------------------------------------- -# .env loading -# --------------------------------------------------------------------------- - - -def parse_env_file(path: Path) -> dict[str, str]: - env: dict[str, str] = {} - for raw in path.read_text().splitlines(): - line = raw.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, _, value = line.partition("=") - value = value.strip() - if (value.startswith("'") and value.endswith("'")) or ( - value.startswith('"') and value.endswith('"') - ): - value = value[1:-1] - env[key.strip()] = value - return env - - # --------------------------------------------------------------------------- # GraphQL transport (SDK-backed) # --------------------------------------------------------------------------- @@ -689,14 +667,19 @@ def main() -> int: help=f"Source GraphQL endpoint (default: {DEFAULT_SOURCE_URL}).", ) parser.add_argument( - "--env-file", - default=str(Path(__file__).resolve().parent.parent / ".env"), - help="Path to the .env file (default: repo .env).", + "--target-url", + default=DEFAULT_TARGET_URL, + help=f"Local GraphQL endpoint (default: {DEFAULT_TARGET_URL}).", ) parser.add_argument( - "--target-url", - help="Override the local GraphQL endpoint (default derived from " - "APP_PORT in .env: http://localhost:${APP_PORT}/graphql/).", + "--target-email", + required=True, + help="Email of the Django superuser on the target server.", + ) + parser.add_argument( + "--target-password", + required=True, + help="Password of the Django superuser on the target server.", ) parser.add_argument( "--debug", "-v", @@ -708,31 +691,12 @@ def main() -> int: global DEBUG DEBUG = args.debug - env_path = Path(args.env_file) - if not env_path.exists(): - sys.stderr.write(f"error: env file not found: {env_path}\n") - return 2 - env = parse_env_file(env_path) - - superuser = env.get("DJANGO_SUPERUSER_USERNAME") - password = env.get("DJANGO_SUPERUSER_PASSWORD") - if not superuser or not password: - sys.stderr.write( - "error: DJANGO_SUPERUSER_USERNAME and DJANGO_SUPERUSER_PASSWORD " - "must be set in the env file.\n" - ) - return 2 - - target_url = args.target_url or urljoin( - f"http://localhost:{env.get('APP_PORT', '8000')}/", "graphql/" - ) - print(f"Source: {args.source_url}") - print(f"Target: {target_url}") + print(f"Target: {args.target_url}") try: source = build_source(args.source_url, args.token) - target = build_target(target_url, superuser, password) + target = build_target(args.target_url, args.target_email, args.target_password) migrate(source, target, args.slug) except GraphQLClientHttpError as exc: # SDK's __str__ only includes the status code. Print the body too. From aac39062af42ac112baab72e07c75078111cae88 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Tue, 26 May 2026 10:17:01 +0200 Subject: [PATCH 06/14] Refactor: Split big script into a small lib --- .gitignore | 1 + script/migrate_lib/__init__.py | 0 script/migrate_lib/files.py | 95 +++++ script/migrate_lib/pipelines.py | 388 ++++++++++++++++++++ script/migrate_lib/transport.py | 87 +++++ script/migrate_lib/workspaces.py | 53 +++ script/migrate_workspace.py | 600 ++----------------------------- 7 files changed, 647 insertions(+), 577 deletions(-) create mode 100644 script/migrate_lib/__init__.py create mode 100644 script/migrate_lib/files.py create mode 100644 script/migrate_lib/pipelines.py create mode 100644 script/migrate_lib/transport.py create mode 100644 script/migrate_lib/workspaces.py diff --git a/.gitignore b/.gitignore index 229853e..10350df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +__pycache__/ build/ .env .envlocal diff --git a/script/migrate_lib/__init__.py b/script/migrate_lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/script/migrate_lib/files.py b/script/migrate_lib/files.py new file mode 100644 index 0000000..ce16b1f --- /dev/null +++ b/script/migrate_lib/files.py @@ -0,0 +1,95 @@ +"""Workspace file transfer (low-level utility used by other resource modules).""" + +import httpx +from openhexa.graphql.graphql_client.client import Client +from openhexa.sdk.utils import OpenHexaClient + +from .transport import GraphQLError, _dbg, gql + + +PREPARE_DOWNLOAD_MUTATION = """ +mutation PrepareDownload($input: PrepareObjectDownloadInput!) { + prepareObjectDownload(input: $input) { + success errors downloadUrl + } +} +""" + +PREPARE_UPLOAD_MUTATION = """ +mutation PrepareUpload($input: PrepareObjectUploadInput!) { + prepareObjectUpload(input: $input) { + success errors uploadUrl headers + } +} +""" + + +def download(source: OpenHexaClient, ws_slug: str, file_path: str) -> bytes: + """Download a file from the source workspace via a presigned URL.""" + data = gql( + source, + PREPARE_DOWNLOAD_MUTATION, + { + "input": { + "workspaceSlug": ws_slug, + "objectKey": file_path, + "forceAttachment": False, + } + }, + "PrepareDownload", + ) + result = data["prepareObjectDownload"] + if not result["success"] or not result.get("downloadUrl"): + raise GraphQLError( + f"prepareObjectDownload failed for '{file_path}': " + + ",".join(result.get("errors") or []) + ) + url = result["downloadUrl"] + _dbg(f"download {file_path} <- {url}") + with httpx.Client(timeout=300) as c: + resp = c.get(url) + if not resp.is_success: + raise GraphQLError( + f"download of '{file_path}' returned HTTP {resp.status_code}: " + f"{resp.text[:500]}" + ) + return resp.content + + +def upload( + target: Client, + ws_slug: str, + file_path: str, + content: bytes, + content_type: str = "application/octet-stream", +) -> None: + """Upload bytes to the target workspace at the given object key.""" + data = gql( + target, + PREPARE_UPLOAD_MUTATION, + { + "input": { + "workspaceSlug": ws_slug, + "objectKey": file_path, + "contentType": content_type, + } + }, + "PrepareUpload", + ) + result = data["prepareObjectUpload"] + if not result["success"] or not result.get("uploadUrl"): + raise GraphQLError( + f"prepareObjectUpload failed for '{file_path}': " + + ",".join(result.get("errors") or []) + ) + url = result["uploadUrl"] + headers = dict(result.get("headers") or {}) + headers.setdefault("Content-Type", content_type) + _dbg(f"upload {file_path} ({len(content)} bytes) -> {url}") + with httpx.Client(timeout=300) as c: + resp = c.put(url, content=content, headers=headers) + if not resp.is_success: + raise GraphQLError( + f"upload of '{file_path}' returned HTTP {resp.status_code}: " + f"{resp.text[:500]}" + ) diff --git a/script/migrate_lib/pipelines.py b/script/migrate_lib/pipelines.py new file mode 100644 index 0000000..6e2dd49 --- /dev/null +++ b/script/migrate_lib/pipelines.py @@ -0,0 +1,388 @@ +"""Pipeline migration: list source pipelines, copy each (with versions) to target.""" + +import base64 +from dataclasses import dataclass, field +from typing import Any + +from openhexa.graphql.graphql_client.client import Client +from openhexa.graphql.graphql_client.input_types import CreatePipelineInput +from openhexa.sdk.utils import OpenHexaClient + +from . import files +from .transport import GraphQLError, gql + + +PIPELINES_PAGE_SIZE = 50 +VERSIONS_PAGE_SIZE = 50 + + +PIPELINE_DETAIL_QUERY = """ +query PipelineDetail($id: UUID!, $vPage: Int!, $vPerPage: Int!) { + pipeline(id: $id) { + id code name description type functionalType notebookPath + schedule config webhookEnabled + tags { name } + scheduledPipelineVersion { id versionNumber } + versions(page: $vPage, perPage: $vPerPage) { + pageNumber + totalPages + items { + id versionNumber name description externalLink config timeout + zipfile + parameters { + code name type multiple required default help + widget connection choices + } + } + } + } +} +""" + +UPLOAD_PIPELINE_MUTATION = """ +mutation UploadPipeline($input: UploadPipelineInput!) { + uploadPipeline(input: $input) { + success errors details + pipelineVersion { id versionNumber versionName } + } +} +""" + +UPDATE_PIPELINE_MUTATION = """ +mutation UpdatePipeline($input: UpdatePipelineInput!) { + updatePipeline(input: $input) { + success errors + pipeline { id code } + } +} +""" + + +@dataclass +class PipelinesResult: + """What migrate_all() did, for the orchestrator to print.""" + + created: list[tuple[str, list[str]]] = field(default_factory=list) + """(pipeline_code, [version_name, ...]) for each pipeline created on target.""" + + skipped: list[str] = field(default_factory=list) + """Pipeline codes that already existed on target or could not be migrated.""" + + warnings: list[str] = field(default_factory=list) + """Human-readable warnings to print in the summary.""" + + +# --------------------------------------------------------------------------- +# Source fetch +# --------------------------------------------------------------------------- + + +def _list_source_ids(source: OpenHexaClient, slug: str) -> list[tuple[str, str]]: + """Return [(pipeline_id, pipeline_code), ...] across all pages.""" + pairs: list[tuple[str, str]] = [] + page = 1 + while True: + result = source.pipelines( + workspace_slug=slug, page=page, per_page=PIPELINES_PAGE_SIZE + ) + pairs.extend((str(item.id), item.code) for item in result.items) + if page >= result.total_pages or result.total_pages == 0: + break + page += 1 + return pairs + + +def _fetch_source_detail(source: OpenHexaClient, pipeline_id: str) -> dict[str, Any]: + """Fetch full pipeline data + all versions for one pipeline.""" + first = gql( + source, + PIPELINE_DETAIL_QUERY, + {"id": pipeline_id, "vPage": 1, "vPerPage": VERSIONS_PAGE_SIZE}, + "PipelineDetail", + ) + detail = first["pipeline"] + if detail is None: + raise GraphQLError(f"source pipeline id={pipeline_id} disappeared") + versions = list(detail["versions"]["items"]) + total_pages = detail["versions"]["totalPages"] + for vpage in range(2, total_pages + 1): + more = gql( + source, + PIPELINE_DETAIL_QUERY, + {"id": pipeline_id, "vPage": vpage, "vPerPage": VERSIONS_PAGE_SIZE}, + "PipelineDetail", + ) + versions.extend(more["pipeline"]["versions"]["items"]) + detail["versions"] = sorted(versions, key=lambda v: v["versionNumber"]) + return detail + + +# --------------------------------------------------------------------------- +# Target writes +# --------------------------------------------------------------------------- + + +def _create_on_target( + target: Client, target_slug: str, src_pipeline: dict[str, Any] +) -> tuple[str, str]: + """Create an empty pipeline on the target and return (id, code). + + The server (see ``Pipeline.objects.create_if_has_perm`` in + openhexa-app/.../pipelines/models.py) auto-generates the pipeline + code from ``slugify(name)`` with a collision suffix and rejects any + code the caller tries to pass. So we never pass a code; we always + read the actual one back from the response. + """ + is_notebook = src_pipeline.get("type") == "notebook" + tags = [t["name"] for t in (src_pipeline.get("tags") or [])] + create_input = CreatePipelineInput( + name=src_pipeline["name"] or src_pipeline["code"], + description=src_pipeline.get("description") or None, + workspace_slug=target_slug, + notebook_path=src_pipeline.get("notebookPath") if is_notebook else None, + functional_type=src_pipeline.get("functionalType"), + tags=tags or None, + ) + result = target.create_pipeline(input=create_input) + if not result.success or result.pipeline is None: + raise GraphQLError( + f"createPipeline failed for '{src_pipeline['code']}': " + + ",".join(e.value for e in (result.errors or [])) + ) + created_code = result.pipeline.code + + # createPipeline returns only the code; fetch by code to get the id + # needed for subsequent updatePipeline. + created = target.pipeline(workspace_slug=target_slug, pipeline_code=created_code) + if created is None: + raise GraphQLError(f"could not look up created pipeline '{created_code}'") + return str(created.id), created_code + + +def _clean_parameters(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Strip nulls that the input type doesn't accept verbatim.""" + cleaned = [] + for p in parameters: + item = {k: v for k, v in p.items() if v is not None} + cfile = item.get("choicesFromFile") + if cfile is not None: + item["choicesFromFile"] = {k: v for k, v in cfile.items() if v is not None} + cleaned.append(item) + return cleaned + + +def _upload_version( + target: Client, + workspace_slug: str, + pipeline_code: str, + version: dict[str, Any], +) -> dict[str, Any]: + """Upload one version to an existing target pipeline.""" + try: + base64.b64decode(version["zipfile"], validate=True) + except Exception as exc: + raise GraphQLError( + f"version {version['versionNumber']} has invalid base64 zipfile: {exc}" + ) + + input_: dict[str, Any] = { + "workspaceSlug": workspace_slug, + "pipelineCode": pipeline_code, + "name": version.get("name"), + "description": version.get("description"), + "externalLink": version.get("externalLink"), + "parameters": _clean_parameters(version.get("parameters") or []), + "zipfile": version["zipfile"], + "config": version.get("config") or {}, + "timeout": version.get("timeout"), + } + input_ = {k: v for k, v in input_.items() if v is not None} + data = gql(target, UPLOAD_PIPELINE_MUTATION, {"input": input_}, "UploadPipeline") + result = data["uploadPipeline"] + if not result["success"]: + raise GraphQLError( + f"uploadPipeline failed for {pipeline_code} v{version['versionNumber']}: " + + ",".join(result.get("errors") or []) + + (f" ({result['details']})" if result.get("details") else "") + ) + return result["pipelineVersion"] + + +def _update_settings( + target: Client, + target_pipeline_id: str, + src_pipeline: dict[str, Any], + scheduled_version_id: str | None, +) -> None: + """Apply pipeline-level fields that createPipeline/uploadPipeline cannot set.""" + input_: dict[str, Any] = { + "id": target_pipeline_id, + "schedule": src_pipeline.get("schedule"), + "webhookEnabled": src_pipeline.get("webhookEnabled"), + "config": src_pipeline.get("config") or None, + "scheduledPipelineVersionId": scheduled_version_id, + "autoUpdateFromTemplate": False, + } + # Only call updatePipeline if at least one migrated field has a value. + meaningful = { + k: v + for k, v in input_.items() + if k != "id" + and k != "autoUpdateFromTemplate" + and v not in (None, False, {}, "") + } + if not meaningful: + return + input_ = {k: v for k, v in input_.items() if v is not None} + data = gql(target, UPDATE_PIPELINE_MUTATION, {"input": input_}, "UpdatePipeline") + result = data["updatePipeline"] + if not result["success"]: + raise GraphQLError( + f"updatePipeline failed for {src_pipeline['code']}: " + + ",".join(result.get("errors") or []) + ) + + +# --------------------------------------------------------------------------- +# Orchestration (one resource) +# --------------------------------------------------------------------------- + + +def migrate_all( + source: OpenHexaClient, + target: Client, + source_slug: str, + target_slug: str, +) -> PipelinesResult: + """Migrate every pipeline from `source_slug` into `target_slug`.""" + result = PipelinesResult() + + print("=> Listing source pipelines ...") + pairs = _list_source_ids(source, source_slug) + print(f" found {len(pairs)} pipeline(s)") + + for pipeline_id, src_code in pairs: + existing = target.pipeline(workspace_slug=target_slug, pipeline_code=src_code) + if existing is not None: + print(f" - pipeline '{src_code}' already exists on target — skipping") + result.skipped.append(src_code) + continue + + print(f" - migrating pipeline '{src_code}' ...") + detail = _fetch_source_detail(source, pipeline_id) + is_notebook = detail.get("type") == "notebook" + + if is_notebook and not _copy_notebook_file( + source, target, source_slug, target_slug, src_code, detail, result + ): + continue + + target_pid, target_code = _create_on_target(target, target_slug, detail) + if target_code != src_code: + print( + f" created pipeline as '{target_code}' (source was " + f"'{src_code}'; server re-derives code from name) — id={target_pid}" + ) + else: + print(f" created pipeline '{target_code}' (id={target_pid})") + + uploaded_names, scheduled_version_id = _upload_versions( + target, target_slug, target_code, detail, is_notebook, result + ) + + _update_settings(target, target_pid, detail, scheduled_version_id) + result.created.append((target_code, uploaded_names)) + + return result + + +def _copy_notebook_file( + source: OpenHexaClient, + target: Client, + source_slug: str, + target_slug: str, + src_code: str, + detail: dict[str, Any], + result: PipelinesResult, +) -> bool: + """Copy a notebook pipeline's .ipynb from source to target bucket. + + Returns True on success (caller proceeds to create the pipeline), False + if the notebook is missing or transfer failed (caller skips the pipeline). + + createPipeline for notebook pipelines requires the notebook file to + already exist in the workspace bucket (openhexa-app + pipelines/schema/mutations.py:82) — hence this happens before the + createPipeline call. + """ + nbpath = detail.get("notebookPath") + if not nbpath: + result.warnings.append( + f"notebook pipeline '{src_code}' has no notebookPath; skipped." + ) + result.skipped.append(src_code) + return False + try: + print(f" fetching notebook '{nbpath}' from source ...") + nb_bytes = files.download(source, source_slug, nbpath) + print(f" uploading notebook to target ({len(nb_bytes)} bytes) ...") + files.upload( + target, + target_slug, + nbpath, + nb_bytes, + content_type="application/x-ipynb+json", + ) + except GraphQLError as exc: + result.warnings.append( + f"notebook pipeline '{src_code}': could not migrate notebook file " + f"'{nbpath}' ({exc}); pipeline skipped." + ) + result.skipped.append(src_code) + return False + return True + + +def _upload_versions( + target: Client, + target_slug: str, + target_code: str, + detail: dict[str, Any], + is_notebook: bool, + result: PipelinesResult, +) -> tuple[list[str], str | None]: + """Upload all zipfile versions; return (uploaded_version_names, scheduled_version_id).""" + uploaded_names: list[str] = [] + scheduled_version_id: str | None = None + scheduled_src = detail.get("scheduledPipelineVersion") or {} + scheduled_src_number = scheduled_src.get("versionNumber") + + if is_notebook: + result.warnings.append( + f"pipeline '{target_code}' is a notebook pipeline; its source file " + f"at notebookPath='{detail.get('notebookPath')}' must be " + "copied into the local workspace files for it to run." + ) + if detail.get("versions"): + result.warnings.append( + f"pipeline '{target_code}' has {len(detail['versions'])} version(s) " + "on the source, but uploadPipeline is not supported for " + "notebook pipelines — versions were not migrated." + ) + return uploaded_names, scheduled_version_id + + versions = detail.get("versions") or [] + if not versions: + result.warnings.append( + f"pipeline '{target_code}' has no versions on source; created with no version." + ) + for v in versions: + up = _upload_version(target, target_slug, target_code, v) + uploaded_names.append(up["versionName"]) + print(f" uploaded version v{v['versionNumber']} -> {up['versionName']}") + if ( + scheduled_src_number is not None + and up.get("versionNumber") == scheduled_src_number + ): + scheduled_version_id = up["id"] + return uploaded_names, scheduled_version_id diff --git a/script/migrate_lib/transport.py b/script/migrate_lib/transport.py new file mode 100644 index 0000000..65b1cb2 --- /dev/null +++ b/script/migrate_lib/transport.py @@ -0,0 +1,87 @@ +"""GraphQL transport: SDK-backed clients, error type, request wrapper, debug.""" + +import sys +from typing import Any + +import httpx +from openhexa.graphql.graphql_client.client import Client +from openhexa.sdk.utils import OpenHexaClient + + +# Toggled by the CLI's --debug / -v. Module-global so the gql() wrapper and the +# SDK-call helpers don't need to thread it through every signature. +DEBUG = False + + +def _dbg(msg: str) -> None: + if DEBUG: + sys.stderr.write(f"[debug] {msg}\n") + + +def _short(value: Any, limit: int = 200) -> str: + """Render a value for debug output without dumping huge zipfiles.""" + if isinstance(value, dict): + return "{" + ", ".join(f"{k}={_short(v, limit)}" for k, v in value.items()) + "}" + if isinstance(value, list): + head = ", ".join(_short(v, limit) for v in value[:3]) + return f"[{head}{', ...' if len(value) > 3 else ''}] (n={len(value)})" + s = repr(value) + return s if len(s) <= limit else s[:limit] + f"... " + + +class GraphQLError(RuntimeError): + pass + + +def gql( + client: Client, + query: str, + variables: dict[str, Any] | None = None, + operation_name: str | None = None, +) -> dict[str, Any]: + """Execute a raw GraphQL query through the SDK client and return data.""" + if DEBUG: + _dbg(f"-> {operation_name or ''} @ {client.url}") + _dbg(f" variables: {_short(variables or {})}") + resp = client.execute( + query=query, variables=variables or {}, operation_name=operation_name + ) + if DEBUG: + _dbg(f"<- HTTP {resp.status_code} ({len(resp.content)} bytes)") + if not resp.is_success: + # Surface the body — the SDK's __str__ only shows the status code, + # which loses the actual GraphQL/Django error. + raise GraphQLError( + f"{operation_name or ''} returned HTTP {resp.status_code}: " + f"{resp.text[:2000]}" + ) + return client.get_data(resp) + + +def build_source(server_url: str, token: str) -> OpenHexaClient: + return OpenHexaClient(token=token, server_url=server_url) + + +def build_target(target_url: str, email: str, password: str) -> Client: + http = httpx.Client(headers={"User-Agent": "openhexa-migrate/1.0"}) + # Prime CSRF cookie. Defensive — GraphQLView is csrf_exempt on the + # current backend, but a future change would otherwise silently + # break every mutation. + http.get(target_url) + csrf = http.cookies.get("csrftoken") + if csrf: + http.headers["X-CSRFToken"] = csrf + http.headers["Referer"] = target_url + + client = Client(url=target_url, http_client=http) + data = gql( + client, + "mutation Login($input: LoginInput!) { login(input: $input) { success errors } }", + {"input": {"email": email, "password": password}}, + "Login", + ) + if not data["login"]["success"]: + raise GraphQLError( + "target login failed: " + ",".join(data["login"]["errors"] or []) + ) + return client diff --git a/script/migrate_lib/workspaces.py b/script/migrate_lib/workspaces.py new file mode 100644 index 0000000..2d722fd --- /dev/null +++ b/script/migrate_lib/workspaces.py @@ -0,0 +1,53 @@ +"""Workspace-level migration (workspace metadata; later: members, settings).""" + +import sys +from typing import Any + +from openhexa.graphql.graphql_client.client import Client +from openhexa.graphql.graphql_client.input_types import ( + CountryInput, + CreateWorkspaceInput, + UpdateWorkspaceInput, +) + +from .transport import GraphQLError + + +def create(target: Client, src_ws: Any) -> str: + """Create the workspace on target, returning the slug the server picked. + + Note: the server (see resolve_create_workspace + create_workspace_slug + in openhexa-app) ignores any `slug` passed in the input — it derives + the slug from the name with a random suffix. So we never pass a slug + and always read the actual slug back from the response. + """ + countries = [ + CountryInput(code=c.code, alpha3=c.alpha_3, name=c.name, flag=c.flag) + for c in (src_ws.countries or []) + ] + result = target.create_workspace( + input=CreateWorkspaceInput( + name=src_ws.name, + description=src_ws.description or "", + countries=countries, + load_sample_data=False, + configuration=src_ws.configuration or {}, + ) + ) + if not result.success or result.workspace is None: + raise GraphQLError("createWorkspace failed: " + ",".join(result.errors or [])) + created_slug = result.workspace.slug + + if src_ws.docker_image: + upd = target.update_workspace( + input=UpdateWorkspaceInput( + slug=created_slug, docker_image=src_ws.docker_image + ) + ) + if not upd.success: + print( + f" warning: could not set dockerImage=" + f"'{src_ws.docker_image}': " + ",".join(upd.errors or []), + file=sys.stderr, + ) + return created_slug diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index f15dc66..e90011d 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Migrate a workspace from a remote OpenHEXA server to the local Docker setup. +"""Migrate a workspace from one OpenHEXA server to another. Source auth: WorkspaceMembership.access_token (the same token format the OpenHEXA CLI uses), sent as ``Authorization: Bearer ``. @@ -22,24 +22,13 @@ """ import argparse -import base64 import sys -from typing import Any try: - import httpx - from openhexa.graphql.graphql_client.client import Client from openhexa.graphql.graphql_client.exceptions import ( GraphQLClientGraphQLMultiError, GraphQLClientHttpError, ) - from openhexa.graphql.graphql_client.input_types import ( - CountryInput, - CreatePipelineInput, - CreateWorkspaceInput, - UpdateWorkspaceInput, - ) - from openhexa.sdk.utils import OpenHexaClient except ImportError: sys.stderr.write( "error: this script requires the 'openhexa.sdk' package " @@ -47,460 +36,16 @@ ) sys.exit(1) +from migrate_lib import pipelines, transport, workspaces +from migrate_lib.pipelines import PipelinesResult +from migrate_lib.transport import GraphQLError, build_source, build_target + DEFAULT_SOURCE_URL = "https://api.openhexa.org/graphql/" DEFAULT_TARGET_URL = "http://localhost:8000/graphql/" -PIPELINES_PAGE_SIZE = 50 -VERSIONS_PAGE_SIZE = 50 - -# Toggled by --debug / -v. Module-global so the gql() and SDK-call wrappers -# don't need to thread it through every signature. -DEBUG = False - - -def _dbg(msg: str) -> None: - if DEBUG: - sys.stderr.write(f"[debug] {msg}\n") - - -def _short(value: Any, limit: int = 200) -> str: - """Render a value for debug output without dumping huge zipfiles.""" - if isinstance(value, dict): - return "{" + ", ".join(f"{k}={_short(v, limit)}" for k, v in value.items()) + "}" - if isinstance(value, list): - head = ", ".join(_short(v, limit) for v in value[:3]) - return f"[{head}{', ...' if len(value) > 3 else ''}] (n={len(value)})" - s = repr(value) - return s if len(s) <= limit else s[:limit] + f"... " - - -# --------------------------------------------------------------------------- -# GraphQL transport (SDK-backed) -# --------------------------------------------------------------------------- - - -class GraphQLError(RuntimeError): - pass - - -def gql( - client: Client, - query: str, - variables: dict[str, Any] | None = None, - operation_name: str | None = None, -) -> dict[str, Any]: - """Execute a raw GraphQL query through the SDK client and return data.""" - if DEBUG: - _dbg(f"-> {operation_name or ''} @ {client.url}") - _dbg(f" variables: {_short(variables or {})}") - resp = client.execute( - query=query, variables=variables or {}, operation_name=operation_name - ) - if DEBUG: - _dbg(f"<- HTTP {resp.status_code} ({len(resp.content)} bytes)") - if not resp.is_success: - # Surface the body — the SDK's __str__ only shows the status code, - # which loses the actual GraphQL/Django error. - raise GraphQLError( - f"{operation_name or ''} returned HTTP {resp.status_code}: " - f"{resp.text[:2000]}" - ) - return client.get_data(resp) - - -def build_source(server_url: str, token: str) -> OpenHexaClient: - return OpenHexaClient(token=token, server_url=server_url) - - -def build_target(target_url: str, email: str, password: str) -> Client: - http = httpx.Client(headers={"User-Agent": "openhexa-migrate/1.0"}) - # Prime CSRF cookie. Defensive — GraphQLView is csrf_exempt on the - # current backend, but a future change would otherwise silently - # break every mutation. - http.get(target_url) - csrf = http.cookies.get("csrftoken") - if csrf: - http.headers["X-CSRFToken"] = csrf - http.headers["Referer"] = target_url - - client = Client(url=target_url, http_client=http) - data = gql( - client, - "mutation Login($input: LoginInput!) { login(input: $input) { success errors } }", - {"input": {"email": email, "password": password}}, - "Login", - ) - if not data["login"]["success"]: - raise GraphQLError( - "target login failed: " + ",".join(data["login"]["errors"] or []) - ) - return client - - -# --------------------------------------------------------------------------- -# Source fetch helpers -# --------------------------------------------------------------------------- - -PIPELINE_DETAIL_QUERY = """ -query PipelineDetail($id: UUID!, $vPage: Int!, $vPerPage: Int!) { - pipeline(id: $id) { - id code name description type functionalType notebookPath - schedule config webhookEnabled - tags { name } - scheduledPipelineVersion { id versionNumber } - versions(page: $vPage, perPage: $vPerPage) { - pageNumber - totalPages - items { - id versionNumber name description externalLink config timeout - zipfile - parameters { - code name type multiple required default help - widget connection choices - } - } - } - } -} -""" - - -def fetch_source_pipeline_ids( - source: OpenHexaClient, slug: str -) -> list[tuple[str, str]]: - """Return [(pipeline_id, pipeline_code), ...] across all pages.""" - pairs: list[tuple[str, str]] = [] - page = 1 - while True: - result = source.pipelines( - workspace_slug=slug, page=page, per_page=PIPELINES_PAGE_SIZE - ) - pairs.extend((str(item.id), item.code) for item in result.items) - if page >= result.total_pages or result.total_pages == 0: - break - page += 1 - return pairs - - -def fetch_source_pipeline_detail( - source: OpenHexaClient, pipeline_id: str -) -> dict[str, Any]: - """Fetch full pipeline data + all versions for one pipeline.""" - first = gql( - source, - PIPELINE_DETAIL_QUERY, - {"id": pipeline_id, "vPage": 1, "vPerPage": VERSIONS_PAGE_SIZE}, - "PipelineDetail", - ) - detail = first["pipeline"] - if detail is None: - raise GraphQLError(f"source pipeline id={pipeline_id} disappeared") - versions = list(detail["versions"]["items"]) - total_pages = detail["versions"]["totalPages"] - for vpage in range(2, total_pages + 1): - more = gql( - source, - PIPELINE_DETAIL_QUERY, - {"id": pipeline_id, "vPage": vpage, "vPerPage": VERSIONS_PAGE_SIZE}, - "PipelineDetail", - ) - versions.extend(more["pipeline"]["versions"]["items"]) - detail["versions"] = sorted(versions, key=lambda v: v["versionNumber"]) - return detail - - -# --------------------------------------------------------------------------- -# Workspace file transfer (used to migrate notebook files) -# --------------------------------------------------------------------------- - -PREPARE_DOWNLOAD_MUTATION = """ -mutation PrepareDownload($input: PrepareObjectDownloadInput!) { - prepareObjectDownload(input: $input) { - success errors downloadUrl - } -} -""" - -PREPARE_UPLOAD_MUTATION = """ -mutation PrepareUpload($input: PrepareObjectUploadInput!) { - prepareObjectUpload(input: $input) { - success errors uploadUrl headers - } -} -""" - - -def download_workspace_file( - source: OpenHexaClient, ws_slug: str, file_path: str -) -> bytes: - """Download a file from the source workspace via a presigned URL.""" - data = gql( - source, - PREPARE_DOWNLOAD_MUTATION, - { - "input": { - "workspaceSlug": ws_slug, - "objectKey": file_path, - "forceAttachment": False, - } - }, - "PrepareDownload", - ) - result = data["prepareObjectDownload"] - if not result["success"] or not result.get("downloadUrl"): - raise GraphQLError( - f"prepareObjectDownload failed for '{file_path}': " - + ",".join(result.get("errors") or []) - ) - url = result["downloadUrl"] - _dbg(f"download {file_path} <- {url}") - with httpx.Client(timeout=300) as c: - resp = c.get(url) - if not resp.is_success: - raise GraphQLError( - f"download of '{file_path}' returned HTTP {resp.status_code}: " - f"{resp.text[:500]}" - ) - return resp.content - - -def upload_workspace_file( - target: Client, - ws_slug: str, - file_path: str, - content: bytes, - content_type: str = "application/octet-stream", -) -> None: - """Upload bytes to the target workspace at the given object key.""" - data = gql( - target, - PREPARE_UPLOAD_MUTATION, - { - "input": { - "workspaceSlug": ws_slug, - "objectKey": file_path, - "contentType": content_type, - } - }, - "PrepareUpload", - ) - result = data["prepareObjectUpload"] - if not result["success"] or not result.get("uploadUrl"): - raise GraphQLError( - f"prepareObjectUpload failed for '{file_path}': " - + ",".join(result.get("errors") or []) - ) - url = result["uploadUrl"] - headers = dict(result.get("headers") or {}) - headers.setdefault("Content-Type", content_type) - _dbg(f"upload {file_path} ({len(content)} bytes) -> {url}") - with httpx.Client(timeout=300) as c: - resp = c.put(url, content=content, headers=headers) - if not resp.is_success: - raise GraphQLError( - f"upload of '{file_path}' returned HTTP {resp.status_code}: " - f"{resp.text[:500]}" - ) - - -# --------------------------------------------------------------------------- -# Target apply helpers -# --------------------------------------------------------------------------- - -UPLOAD_PIPELINE_MUTATION = """ -mutation UploadPipeline($input: UploadPipelineInput!) { - uploadPipeline(input: $input) { - success errors details - pipelineVersion { id versionNumber versionName } - } -} -""" - -UPDATE_PIPELINE_MUTATION = """ -mutation UpdatePipeline($input: UpdatePipelineInput!) { - updatePipeline(input: $input) { - success errors - pipeline { id code } - } -} -""" - - -def create_target_workspace(target: Client, src_ws: Any) -> str: - """Create the workspace on target, returning the slug the server picked. - - Note: the server (see resolve_create_workspace + create_workspace_slug - in openhexa-app) ignores any `slug` passed in the input — it derives - the slug from the name with a random suffix. So we never pass a slug - and always read the actual slug back from the response. - """ - countries = [ - CountryInput(code=c.code, alpha3=c.alpha_3, name=c.name, flag=c.flag) - for c in (src_ws.countries or []) - ] - result = target.create_workspace( - input=CreateWorkspaceInput( - name=src_ws.name, - description=src_ws.description or "", - countries=countries, - load_sample_data=False, - configuration=src_ws.configuration or {}, - ) - ) - if not result.success or result.workspace is None: - raise GraphQLError("createWorkspace failed: " + ",".join(result.errors or [])) - created_slug = result.workspace.slug - - if src_ws.docker_image: - upd = target.update_workspace( - input=UpdateWorkspaceInput( - slug=created_slug, docker_image=src_ws.docker_image - ) - ) - if not upd.success: - print( - f" warning: could not set dockerImage=" - f"'{src_ws.docker_image}': " + ",".join(upd.errors or []), - file=sys.stderr, - ) - return created_slug - - -def create_target_pipeline( - target: Client, target_slug: str, src_pipeline: dict[str, Any] -) -> tuple[str, str]: - """Create an empty pipeline on the target and return (id, code). - - The server (see ``Pipeline.objects.create_if_has_perm`` in - openhexa-app/.../pipelines/models.py) auto-generates the pipeline - code from ``slugify(name)`` with a collision suffix and rejects any - code the caller tries to pass. So we never pass a code; we always - read the actual one back from the response. - """ - is_notebook = src_pipeline.get("type") == "notebook" - tags = [t["name"] for t in (src_pipeline.get("tags") or [])] - create_input = CreatePipelineInput( - name=src_pipeline["name"] or src_pipeline["code"], - description=src_pipeline.get("description") or None, - workspace_slug=target_slug, - notebook_path=src_pipeline.get("notebookPath") if is_notebook else None, - functional_type=src_pipeline.get("functionalType"), - tags=tags or None, - ) - result = target.create_pipeline(input=create_input) - if not result.success or result.pipeline is None: - raise GraphQLError( - f"createPipeline failed for '{src_pipeline['code']}': " - + ",".join(e.value for e in (result.errors or [])) - ) - created_code = result.pipeline.code - - # createPipeline returns only the code; fetch by code to get the id - # needed for subsequent updatePipeline. - created = target.pipeline( - workspace_slug=target_slug, pipeline_code=created_code - ) - if created is None: - raise GraphQLError( - f"could not look up created pipeline '{created_code}'" - ) - return str(created.id), created_code - -def _clean_parameters(parameters: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Strip nulls that the input type doesn't accept verbatim.""" - cleaned = [] - for p in parameters: - item = {k: v for k, v in p.items() if v is not None} - cfile = item.get("choicesFromFile") - if cfile is not None: - item["choicesFromFile"] = {k: v for k, v in cfile.items() if v is not None} - cleaned.append(item) - return cleaned - -def upload_pipeline_version( - target: Client, - workspace_slug: str, - pipeline_code: str, - version: dict[str, Any], -) -> dict[str, Any]: - """Upload one version to an existing target pipeline.""" - try: - base64.b64decode(version["zipfile"], validate=True) - except Exception as exc: - raise GraphQLError( - f"version {version['versionNumber']} has invalid base64 zipfile: {exc}" - ) - - input_: dict[str, Any] = { - "workspaceSlug": workspace_slug, - "pipelineCode": pipeline_code, - "name": version.get("name"), - "description": version.get("description"), - "externalLink": version.get("externalLink"), - "parameters": _clean_parameters(version.get("parameters") or []), - "zipfile": version["zipfile"], - "config": version.get("config") or {}, - "timeout": version.get("timeout"), - } - input_ = {k: v for k, v in input_.items() if v is not None} - data = gql(target, UPLOAD_PIPELINE_MUTATION, {"input": input_}, "UploadPipeline") - result = data["uploadPipeline"] - if not result["success"]: - raise GraphQLError( - f"uploadPipeline failed for {pipeline_code} v{version['versionNumber']}: " - + ",".join(result.get("errors") or []) - + (f" ({result['details']})" if result.get("details") else "") - ) - return result["pipelineVersion"] - - -def update_pipeline_settings( - target: Client, - target_pipeline_id: str, - src_pipeline: dict[str, Any], - scheduled_version_id: str | None, -) -> None: - """Apply pipeline-level fields that createPipeline/uploadPipeline cannot set.""" - input_: dict[str, Any] = { - "id": target_pipeline_id, - "schedule": src_pipeline.get("schedule"), - "webhookEnabled": src_pipeline.get("webhookEnabled"), - "config": src_pipeline.get("config") or None, - "scheduledPipelineVersionId": scheduled_version_id, - "autoUpdateFromTemplate": False, - } - # Only call updatePipeline if at least one migrated field has a value. - meaningful = { - k: v - for k, v in input_.items() - if k != "id" - and k != "autoUpdateFromTemplate" - and v not in (None, False, {}, "") - } - if not meaningful: - return - input_ = {k: v for k, v in input_.items() if v is not None} - data = gql(target, UPDATE_PIPELINE_MUTATION, {"input": input_}, "UpdatePipeline") - result = data["updatePipeline"] - if not result["success"]: - raise GraphQLError( - f"updatePipeline failed for {src_pipeline['code']}: " - + ",".join(result.get("errors") or []) - ) - - -# --------------------------------------------------------------------------- -# Orchestration -# --------------------------------------------------------------------------- - - -def migrate( - source: OpenHexaClient, - target: Client, - source_slug: str, -) -> None: +def migrate(source, target, source_slug: str) -> None: print(f"=> Fetching source workspace '{source_slug}' ...") src_ws = source.workspace(slug=source_slug) if src_ws is None: @@ -508,7 +53,7 @@ def migrate( print(f" name: {src_ws.name!r}") print("=> Creating target workspace ...") - target_slug = create_target_workspace(target, src_ws) + target_slug = workspaces.create(target, src_ws) print(f" created with slug '{target_slug}'") if target_slug != source_slug: print( @@ -517,122 +62,28 @@ def migrate( "mutation always derives the slug from the workspace name." ) - print("=> Listing source pipelines ...") - pairs = fetch_source_pipeline_ids(source, source_slug) - print(f" found {len(pairs)} pipeline(s)") - - created_summary: list[tuple[str, list[str]]] = [] - skipped: list[str] = [] - warnings: list[str] = [] - - for pipeline_id, src_code in pairs: - existing = target.pipeline(workspace_slug=target_slug, pipeline_code=src_code) - if existing is not None: - print(f" - pipeline '{src_code}' already exists on target — skipping") - skipped.append(src_code) - continue - - print(f" - migrating pipeline '{src_code}' ...") - detail = fetch_source_pipeline_detail(source, pipeline_id) - is_notebook = detail.get("type") == "notebook" - - if is_notebook: - # createPipeline for notebook pipelines requires the notebook - # file to already exist in the workspace bucket - # (openhexa-app pipelines/schema/mutations.py:82). Copy it - # over before creating the pipeline. - nbpath = detail.get("notebookPath") - if not nbpath: - warnings.append( - f"notebook pipeline '{src_code}' has no notebookPath; " - "skipped." - ) - skipped.append(src_code) - continue - try: - print(f" fetching notebook '{nbpath}' from source ...") - nb_bytes = download_workspace_file(source, source_slug, nbpath) - print(f" uploading notebook to target ({len(nb_bytes)} bytes) ...") - upload_workspace_file( - target, - target_slug, - nbpath, - nb_bytes, - content_type="application/x-ipynb+json", - ) - except GraphQLError as exc: - warnings.append( - f"notebook pipeline '{src_code}': could not migrate " - f"notebook file '{nbpath}' ({exc}); pipeline skipped." - ) - skipped.append(src_code) - continue + pipelines_result = pipelines.migrate_all(source, target, source_slug, target_slug) - target_pid, target_code = create_target_pipeline(target, target_slug, detail) - if target_code != src_code: - print( - f" created pipeline as '{target_code}' (source was " - f"'{src_code}'; server re-derives code from name) — id={target_pid}" - ) - else: - print(f" created pipeline '{target_code}' (id={target_pid})") + _print_summary(src_ws.name, target_slug, pipelines_result) - uploaded_names: list[str] = [] - scheduled_version_id: str | None = None - scheduled_src = detail.get("scheduledPipelineVersion") or {} - scheduled_src_number = scheduled_src.get("versionNumber") - if is_notebook: - warnings.append( - f"pipeline '{target_code}' is a notebook pipeline; its source file " - f"at notebookPath='{detail.get('notebookPath')}' must be " - "copied into the local workspace files for it to run." - ) - if detail.get("versions"): - warnings.append( - f"pipeline '{target_code}' has {len(detail['versions'])} version(s) " - "on the source, but uploadPipeline is not supported for " - "notebook pipelines — versions were not migrated." - ) - else: - versions = detail.get("versions") or [] - if not versions: - warnings.append( - f"pipeline '{target_code}' has no versions on source; " - "created with no version." - ) - for v in versions: - up = upload_pipeline_version(target, target_slug, target_code, v) - uploaded_names.append(up["versionName"]) - print( - f" uploaded version v{v['versionNumber']} " - f"-> {up['versionName']}" - ) - if ( - scheduled_src_number is not None - and up.get("versionNumber") == scheduled_src_number - ): - scheduled_version_id = up["id"] - - update_pipeline_settings(target, target_pid, detail, scheduled_version_id) - - created_summary.append((target_code, uploaded_names)) - - # ----------- summary ----------- +def _print_summary( + src_ws_name: str, target_slug: str, pipelines_result: PipelinesResult +) -> None: print("\n=== Migration summary ===") - print(f"Workspace: {src_ws.name!r} -> slug '{target_slug}'") - print(f"Pipelines created: {len(created_summary)}") - for code, vnames in created_summary: + print(f"Workspace: {src_ws_name!r} -> slug '{target_slug}'") + print(f"Pipelines created: {len(pipelines_result.created)}") + for code, vnames in pipelines_result.created: print(f" * {code}") for vn in vnames: print(f" - {vn}") - if skipped: - print(f"Pipelines skipped (already existed): {len(skipped)}") - for code in skipped: + if pipelines_result.skipped: + print(f"Pipelines skipped (already existed): {len(pipelines_result.skipped)}") + for code in pipelines_result.skipped: print(f" * {code}") - if warnings: + if pipelines_result.warnings: print("Warnings:") - for w in warnings: + for w in pipelines_result.warnings: print(f" - {w}") print( "Note: connections were not migrated; recreate them locally if " @@ -640,11 +91,6 @@ def migrate( ) -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - - def main() -> int: parser = argparse.ArgumentParser( description="Migrate a workspace from a remote OpenHEXA server " @@ -682,14 +128,14 @@ def main() -> int: help="Password of the Django superuser on the target server.", ) parser.add_argument( - "--debug", "-v", + "--debug", + "-v", action="store_true", help="Print each GraphQL request (operation + variables) and response status.", ) args = parser.parse_args() - global DEBUG - DEBUG = args.debug + transport.DEBUG = args.debug print(f"Source: {args.source_url}") print(f"Target: {args.target_url}") From 9449ed3bbed0f70af1aae56db75272bb89bb9369 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Tue, 26 May 2026 10:38:31 +0200 Subject: [PATCH 07/14] Refac: Align source auth to same as target --- script/migrate_lib/files.py | 3 +-- script/migrate_lib/pipelines.py | 9 ++++----- script/migrate_lib/transport.py | 17 ++++++++--------- script/migrate_workspace.py | 33 +++++++++++++++++++-------------- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/script/migrate_lib/files.py b/script/migrate_lib/files.py index ce16b1f..8764ef8 100644 --- a/script/migrate_lib/files.py +++ b/script/migrate_lib/files.py @@ -2,7 +2,6 @@ import httpx from openhexa.graphql.graphql_client.client import Client -from openhexa.sdk.utils import OpenHexaClient from .transport import GraphQLError, _dbg, gql @@ -24,7 +23,7 @@ """ -def download(source: OpenHexaClient, ws_slug: str, file_path: str) -> bytes: +def download(source: Client, ws_slug: str, file_path: str) -> bytes: """Download a file from the source workspace via a presigned URL.""" data = gql( source, diff --git a/script/migrate_lib/pipelines.py b/script/migrate_lib/pipelines.py index 6e2dd49..f81ec61 100644 --- a/script/migrate_lib/pipelines.py +++ b/script/migrate_lib/pipelines.py @@ -6,7 +6,6 @@ from openhexa.graphql.graphql_client.client import Client from openhexa.graphql.graphql_client.input_types import CreatePipelineInput -from openhexa.sdk.utils import OpenHexaClient from . import files from .transport import GraphQLError, gql @@ -77,7 +76,7 @@ class PipelinesResult: # --------------------------------------------------------------------------- -def _list_source_ids(source: OpenHexaClient, slug: str) -> list[tuple[str, str]]: +def _list_source_ids(source: Client, slug: str) -> list[tuple[str, str]]: """Return [(pipeline_id, pipeline_code), ...] across all pages.""" pairs: list[tuple[str, str]] = [] page = 1 @@ -92,7 +91,7 @@ def _list_source_ids(source: OpenHexaClient, slug: str) -> list[tuple[str, str]] return pairs -def _fetch_source_detail(source: OpenHexaClient, pipeline_id: str) -> dict[str, Any]: +def _fetch_source_detail(source: Client, pipeline_id: str) -> dict[str, Any]: """Fetch full pipeline data + all versions for one pipeline.""" first = gql( source, @@ -249,7 +248,7 @@ def _update_settings( def migrate_all( - source: OpenHexaClient, + source: Client, target: Client, source_slug: str, target_slug: str, @@ -297,7 +296,7 @@ def migrate_all( def _copy_notebook_file( - source: OpenHexaClient, + source: Client, target: Client, source_slug: str, target_slug: str, diff --git a/script/migrate_lib/transport.py b/script/migrate_lib/transport.py index 65b1cb2..f245315 100644 --- a/script/migrate_lib/transport.py +++ b/script/migrate_lib/transport.py @@ -5,7 +5,6 @@ import httpx from openhexa.graphql.graphql_client.client import Client -from openhexa.sdk.utils import OpenHexaClient # Toggled by the CLI's --debug / -v. Module-global so the gql() wrapper and the @@ -58,22 +57,22 @@ def gql( return client.get_data(resp) -def build_source(server_url: str, token: str) -> OpenHexaClient: - return OpenHexaClient(token=token, server_url=server_url) +def build_client(server_url: str, email: str, password: str, *, label: str) -> Client: + """Authenticate against an OpenHEXA server via the GraphQL Login mutation. - -def build_target(target_url: str, email: str, password: str) -> Client: + `label` is used only to make the error message ("source"/"target") clearer. + """ http = httpx.Client(headers={"User-Agent": "openhexa-migrate/1.0"}) # Prime CSRF cookie. Defensive — GraphQLView is csrf_exempt on the # current backend, but a future change would otherwise silently # break every mutation. - http.get(target_url) + http.get(server_url) csrf = http.cookies.get("csrftoken") if csrf: http.headers["X-CSRFToken"] = csrf - http.headers["Referer"] = target_url + http.headers["Referer"] = server_url - client = Client(url=target_url, http_client=http) + client = Client(url=server_url, http_client=http) data = gql( client, "mutation Login($input: LoginInput!) { login(input: $input) { success errors } }", @@ -82,6 +81,6 @@ def build_target(target_url: str, email: str, password: str) -> Client: ) if not data["login"]["success"]: raise GraphQLError( - "target login failed: " + ",".join(data["login"]["errors"] or []) + f"{label} login failed: " + ",".join(data["login"]["errors"] or []) ) return client diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index e90011d..bd308cc 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -1,11 +1,8 @@ #!/usr/bin/env python3 """Migrate a workspace from one OpenHEXA server to another. -Source auth: WorkspaceMembership.access_token (the same token format the -OpenHEXA CLI uses), sent as ``Authorization: Bearer ``. - -Target auth: Django superuser email/password passed via CLI flags, -exchanged for a session cookie via the GraphQL ``login`` mutation. +Auth (both source and target): Django superuser email/password passed via +CLI flags, exchanged for a session cookie via the GraphQL ``login`` mutation. Scope: - Workspace metadata (name, description, dockerImage, configuration, @@ -38,7 +35,7 @@ from migrate_lib import pipelines, transport, workspaces from migrate_lib.pipelines import PipelinesResult -from migrate_lib.transport import GraphQLError, build_source, build_target +from migrate_lib.transport import GraphQLError, build_client DEFAULT_SOURCE_URL = "https://api.openhexa.org/graphql/" @@ -96,12 +93,6 @@ def main() -> int: description="Migrate a workspace from a remote OpenHEXA server " "to the local Docker setup.", ) - parser.add_argument( - "--token", - required=True, - help="WorkspaceMembership access token from the source server " - "(same token used by the OpenHEXA CLI).", - ) parser.add_argument( "--slug", required=True, @@ -112,6 +103,16 @@ def main() -> int: default=DEFAULT_SOURCE_URL, help=f"Source GraphQL endpoint (default: {DEFAULT_SOURCE_URL}).", ) + parser.add_argument( + "--source-email", + required=True, + help="Email of the Django superuser on the source server.", + ) + parser.add_argument( + "--source-password", + required=True, + help="Password of the Django superuser on the source server.", + ) parser.add_argument( "--target-url", default=DEFAULT_TARGET_URL, @@ -141,8 +142,12 @@ def main() -> int: print(f"Target: {args.target_url}") try: - source = build_source(args.source_url, args.token) - target = build_target(args.target_url, args.target_email, args.target_password) + source = build_client( + args.source_url, args.source_email, args.source_password, label="source" + ) + target = build_client( + args.target_url, args.target_email, args.target_password, label="target" + ) migrate(source, target, args.slug) except GraphQLClientHttpError as exc: # SDK's __str__ only includes the status code. Print the body too. From 1763444a7c3bb2f7de82066f8218a6f7932aada2 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Tue, 26 May 2026 13:14:56 +0200 Subject: [PATCH 08/14] Add optional target organization --- script/migrate_lib/workspaces.py | 3 ++- script/migrate_workspace.py | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/script/migrate_lib/workspaces.py b/script/migrate_lib/workspaces.py index 2d722fd..9910275 100644 --- a/script/migrate_lib/workspaces.py +++ b/script/migrate_lib/workspaces.py @@ -13,7 +13,7 @@ from .transport import GraphQLError -def create(target: Client, src_ws: Any) -> str: +def create(target: Client, src_ws: Any, organization_id: str | None = None) -> str: """Create the workspace on target, returning the slug the server picked. Note: the server (see resolve_create_workspace + create_workspace_slug @@ -32,6 +32,7 @@ def create(target: Client, src_ws: Any) -> str: countries=countries, load_sample_data=False, configuration=src_ws.configuration or {}, + organization_id=organization_id, ) ) if not result.success or result.workspace is None: diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index bd308cc..b99ac30 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -42,7 +42,9 @@ DEFAULT_TARGET_URL = "http://localhost:8000/graphql/" -def migrate(source, target, source_slug: str) -> None: +def migrate( + source, target, source_slug: str, target_organization_id: str | None = None +) -> None: print(f"=> Fetching source workspace '{source_slug}' ...") src_ws = source.workspace(slug=source_slug) if src_ws is None: @@ -50,7 +52,7 @@ def migrate(source, target, source_slug: str) -> None: print(f" name: {src_ws.name!r}") print("=> Creating target workspace ...") - target_slug = workspaces.create(target, src_ws) + target_slug = workspaces.create(target, src_ws, target_organization_id) print(f" created with slug '{target_slug}'") if target_slug != source_slug: print( @@ -128,6 +130,13 @@ def main() -> int: required=True, help="Password of the Django superuser on the target server.", ) + parser.add_argument( + "--target-organization", + default=None, + help="Optional UUID of the organization on the target server to create " + "the workspace under. If omitted, the workspace is created without an " + "organization.", + ) parser.add_argument( "--debug", "-v", @@ -148,7 +157,7 @@ def main() -> int: target = build_client( args.target_url, args.target_email, args.target_password, label="target" ) - migrate(source, target, args.slug) + migrate(source, target, args.slug, args.target_organization) except GraphQLClientHttpError as exc: # SDK's __str__ only includes the status code. Print the body too. sys.stderr.write( From f3216295864522896618af4b6783d1c995992672 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Tue, 26 May 2026 13:15:15 +0200 Subject: [PATCH 09/14] Remove unneeded warning --- script/migrate_lib/pipelines.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/script/migrate_lib/pipelines.py b/script/migrate_lib/pipelines.py index f81ec61..7aafde2 100644 --- a/script/migrate_lib/pipelines.py +++ b/script/migrate_lib/pipelines.py @@ -357,11 +357,6 @@ def _upload_versions( scheduled_src_number = scheduled_src.get("versionNumber") if is_notebook: - result.warnings.append( - f"pipeline '{target_code}' is a notebook pipeline; its source file " - f"at notebookPath='{detail.get('notebookPath')}' must be " - "copied into the local workspace files for it to run." - ) if detail.get("versions"): result.warnings.append( f"pipeline '{target_code}' has {len(detail['versions'])} version(s) " From c140067a821f468c8cb1a62f839c33b480deeee7 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Tue, 26 May 2026 13:42:10 +0200 Subject: [PATCH 10/14] Migrate all files v1 --- script/migrate_lib/files.py | 95 ++++++++++++++++++++++++++++++++- script/migrate_lib/pipelines.py | 56 ++----------------- script/migrate_workspace.py | 29 +++++++--- 3 files changed, 120 insertions(+), 60 deletions(-) diff --git a/script/migrate_lib/files.py b/script/migrate_lib/files.py index 8764ef8..ecf2fdb 100644 --- a/script/migrate_lib/files.py +++ b/script/migrate_lib/files.py @@ -1,4 +1,8 @@ -"""Workspace file transfer (low-level utility used by other resource modules).""" +"""Workspace file transfer: list / download / upload, plus full bucket migration.""" + +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import Any import httpx from openhexa.graphql.graphql_client.client import Client @@ -6,6 +10,9 @@ from .transport import GraphQLError, _dbg, gql +OBJECTS_PAGE_SIZE = 100 + + PREPARE_DOWNLOAD_MUTATION = """ mutation PrepareDownload($input: PrepareObjectDownloadInput!) { prepareObjectDownload(input: $input) { @@ -22,6 +29,34 @@ } """ +LIST_OBJECTS_QUERY = """ +query ListObjects($slug: String!, $prefix: String, $page: Int!, $perPage: Int!) { + workspace(slug: $slug) { + bucket { + objects(prefix: $prefix, page: $page, perPage: $perPage, ignoreHiddenFiles: false) { + hasNextPage + items { + key name path size type + } + } + } + } +} +""" + + +@dataclass +class FilesResult: + """What migrate_all() did, for the orchestrator to print.""" + + copied: list[tuple[str, int]] = field(default_factory=list) + """(object_key, byte_size) for each file copied to target.""" + + skipped: list[str] = field(default_factory=list) + """Object keys that could not be copied.""" + + warnings: list[str] = field(default_factory=list) + def download(source: Client, ws_slug: str, file_path: str) -> bytes: """Download a file from the source workspace via a presigned URL.""" @@ -92,3 +127,61 @@ def upload( f"upload of '{file_path}' returned HTTP {resp.status_code}: " f"{resp.text[:500]}" ) + + +def walk(client: Client, ws_slug: str, prefix: str = "") -> Iterator[dict[str, Any]]: + """Recursively yield FILE BucketObject dicts under `prefix`. + + The bucket.objects field is delimited (returns DIRECTORY entries rather + than recursing), so we walk each directory ourselves. + """ + # Buffer all directory entries at this level so we don't interleave + # recursive listings with this level's pagination. + subdirs: list[str] = [] + page = 1 + while True: + data = gql( + client, + LIST_OBJECTS_QUERY, + { + "slug": ws_slug, + "prefix": prefix or None, + "page": page, + "perPage": OBJECTS_PAGE_SIZE, + }, + "ListObjects", + ) + ws = data["workspace"] + if ws is None: + return + page_data = ws["bucket"]["objects"] + for obj in page_data["items"]: + if obj["type"] == "FILE": + yield obj + elif obj["type"] == "DIRECTORY": + subdirs.append(obj["key"]) + if not page_data["hasNextPage"]: + break + page += 1 + for sub in subdirs: + yield from walk(client, ws_slug, sub) + + +def migrate_all( + source: Client, target: Client, source_slug: str, target_slug: str +) -> FilesResult: + """Copy every file from the source workspace bucket to the target bucket.""" + result = FilesResult() + print("=> Listing source files ...") + for obj in walk(source, source_slug): + path = obj["key"] + size = obj.get("size") or 0 + try: + print(f" - copying '{path}' ({size} bytes) ...") + content = download(source, source_slug, path) + upload(target, target_slug, path, content) + result.copied.append((path, len(content))) + except GraphQLError as exc: + result.warnings.append(f"could not copy '{path}': {exc}") + result.skipped.append(path) + return result diff --git a/script/migrate_lib/pipelines.py b/script/migrate_lib/pipelines.py index 7aafde2..f5fb6a4 100644 --- a/script/migrate_lib/pipelines.py +++ b/script/migrate_lib/pipelines.py @@ -7,7 +7,6 @@ from openhexa.graphql.graphql_client.client import Client from openhexa.graphql.graphql_client.input_types import CreatePipelineInput -from . import files from .transport import GraphQLError, gql @@ -271,9 +270,11 @@ def migrate_all( detail = _fetch_source_detail(source, pipeline_id) is_notebook = detail.get("type") == "notebook" - if is_notebook and not _copy_notebook_file( - source, target, source_slug, target_slug, src_code, detail, result - ): + if is_notebook and not detail.get("notebookPath"): + result.warnings.append( + f"notebook pipeline '{src_code}' has no notebookPath; skipped." + ) + result.skipped.append(src_code) continue target_pid, target_code = _create_on_target(target, target_slug, detail) @@ -295,53 +296,6 @@ def migrate_all( return result -def _copy_notebook_file( - source: Client, - target: Client, - source_slug: str, - target_slug: str, - src_code: str, - detail: dict[str, Any], - result: PipelinesResult, -) -> bool: - """Copy a notebook pipeline's .ipynb from source to target bucket. - - Returns True on success (caller proceeds to create the pipeline), False - if the notebook is missing or transfer failed (caller skips the pipeline). - - createPipeline for notebook pipelines requires the notebook file to - already exist in the workspace bucket (openhexa-app - pipelines/schema/mutations.py:82) — hence this happens before the - createPipeline call. - """ - nbpath = detail.get("notebookPath") - if not nbpath: - result.warnings.append( - f"notebook pipeline '{src_code}' has no notebookPath; skipped." - ) - result.skipped.append(src_code) - return False - try: - print(f" fetching notebook '{nbpath}' from source ...") - nb_bytes = files.download(source, source_slug, nbpath) - print(f" uploading notebook to target ({len(nb_bytes)} bytes) ...") - files.upload( - target, - target_slug, - nbpath, - nb_bytes, - content_type="application/x-ipynb+json", - ) - except GraphQLError as exc: - result.warnings.append( - f"notebook pipeline '{src_code}': could not migrate notebook file " - f"'{nbpath}' ({exc}); pipeline skipped." - ) - result.skipped.append(src_code) - return False - return True - - def _upload_versions( target: Client, target_slug: str, diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index b99ac30..7e0c4d9 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -7,15 +7,16 @@ Scope: - Workspace metadata (name, description, dockerImage, configuration, countries) is migrated. + - All files in the workspace bucket are copied from source to target + (must run before pipelines, since notebook pipelines require their + .ipynb to exist on the target before createPipeline succeeds). - Each pipeline is created via createPipeline; each zipfile-pipeline version is uploaded via uploadPipeline; pipeline-level schedule / config / webhookEnabled / scheduledPipelineVersionId are applied via updatePipeline. Out of scope: members, invitations, connections, recipients, templates, -runs, shortcuts, workspace files, datasets, database tables. Notebook -pipelines are created with their notebookPath but the notebook file -itself is not migrated (a warning is emitted). +runs, shortcuts, datasets, database tables. """ import argparse @@ -33,7 +34,8 @@ ) sys.exit(1) -from migrate_lib import pipelines, transport, workspaces +from migrate_lib import files, pipelines, transport, workspaces +from migrate_lib.files import FilesResult from migrate_lib.pipelines import PipelinesResult from migrate_lib.transport import GraphQLError, build_client @@ -61,16 +63,26 @@ def migrate( "mutation always derives the slug from the workspace name." ) + files_result = files.migrate_all(source, target, source_slug, target_slug) pipelines_result = pipelines.migrate_all(source, target, source_slug, target_slug) - _print_summary(src_ws.name, target_slug, pipelines_result) + _print_summary(src_ws.name, target_slug, files_result, pipelines_result) def _print_summary( - src_ws_name: str, target_slug: str, pipelines_result: PipelinesResult + src_ws_name: str, + target_slug: str, + files_result: FilesResult, + pipelines_result: PipelinesResult, ) -> None: print("\n=== Migration summary ===") print(f"Workspace: {src_ws_name!r} -> slug '{target_slug}'") + total_bytes = sum(b for _, b in files_result.copied) + print(f"Files copied: {len(files_result.copied)} ({total_bytes} bytes)") + if files_result.skipped: + print(f"Files skipped: {len(files_result.skipped)}") + for path in files_result.skipped: + print(f" * {path}") print(f"Pipelines created: {len(pipelines_result.created)}") for code, vnames in pipelines_result.created: print(f" * {code}") @@ -80,9 +92,10 @@ def _print_summary( print(f"Pipelines skipped (already existed): {len(pipelines_result.skipped)}") for code in pipelines_result.skipped: print(f" * {code}") - if pipelines_result.warnings: + all_warnings = files_result.warnings + pipelines_result.warnings + if all_warnings: print("Warnings:") - for w in pipelines_result.warnings: + for w in all_warnings: print(f" - {w}") print( "Note: connections were not migrated; recreate them locally if " From 378f54ac2ffec725b0e81cc9082697fd568ec69a Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Tue, 26 May 2026 15:46:51 +0200 Subject: [PATCH 11/14] Files migration: improve failure messages --- script/migrate_lib/files.py | 16 +++++++++------- script/migrate_workspace.py | 14 ++++++++------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/script/migrate_lib/files.py b/script/migrate_lib/files.py index ecf2fdb..b43b293 100644 --- a/script/migrate_lib/files.py +++ b/script/migrate_lib/files.py @@ -1,5 +1,6 @@ """Workspace file transfer: list / download / upload, plus full bucket migration.""" +import sys from collections.abc import Iterator from dataclasses import dataclass, field from typing import Any @@ -52,10 +53,8 @@ class FilesResult: copied: list[tuple[str, int]] = field(default_factory=list) """(object_key, byte_size) for each file copied to target.""" - skipped: list[str] = field(default_factory=list) - """Object keys that could not be copied.""" - - warnings: list[str] = field(default_factory=list) + failed: list[str] = field(default_factory=list) + """Object keys whose download or upload failed; user must handle manually.""" def download(source: Client, ws_slug: str, file_path: str) -> bytes: @@ -181,7 +180,10 @@ def migrate_all( content = download(source, source_slug, path) upload(target, target_slug, path, content) result.copied.append((path, len(content))) - except GraphQLError as exc: - result.warnings.append(f"could not copy '{path}': {exc}") - result.skipped.append(path) + except GraphQLError: + # Surface the failure in the live log without dumping the (often + # HTML) server response. The full path goes into result.failed for + # the final summary so the user can re-attempt manually. + print(f"\tFAILED to copy '{path}'", file=sys.stderr) + result.failed.append(path) return result diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index 7e0c4d9..76dc1c1 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -79,9 +79,12 @@ def _print_summary( print(f"Workspace: {src_ws_name!r} -> slug '{target_slug}'") total_bytes = sum(b for _, b in files_result.copied) print(f"Files copied: {len(files_result.copied)} ({total_bytes} bytes)") - if files_result.skipped: - print(f"Files skipped: {len(files_result.skipped)}") - for path in files_result.skipped: + if files_result.failed: + print( + f"Files that could NOT be migrated " + f"({len(files_result.failed)} — handle manually):" + ) + for path in files_result.failed: print(f" * {path}") print(f"Pipelines created: {len(pipelines_result.created)}") for code, vnames in pipelines_result.created: @@ -92,10 +95,9 @@ def _print_summary( print(f"Pipelines skipped (already existed): {len(pipelines_result.skipped)}") for code in pipelines_result.skipped: print(f" * {code}") - all_warnings = files_result.warnings + pipelines_result.warnings - if all_warnings: + if pipelines_result.warnings: print("Warnings:") - for w in all_warnings: + for w in pipelines_result.warnings: print(f" - {w}") print( "Note: connections were not migrated; recreate them locally if " From c1cb3cbe6987207564375165e328a2f2c68e2706 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Tue, 26 May 2026 22:14:03 +0200 Subject: [PATCH 12/14] Add connections to migration script --- script/migrate_lib/connections.py | 141 ++++++++++++++++++++++++++++++ script/migrate_workspace.py | 43 +++++++-- 2 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 script/migrate_lib/connections.py diff --git a/script/migrate_lib/connections.py b/script/migrate_lib/connections.py new file mode 100644 index 0000000..c70f067 --- /dev/null +++ b/script/migrate_lib/connections.py @@ -0,0 +1,141 @@ +"""Connection migration: copy every workspace connection to the target. + +Secret field values come through the source API only because we authenticate +as a Django superuser: the ``value`` resolver redacts secret fields unless the +caller has ``workspaces.update_connection``, which a superuser short-circuits +to True. If a secret still comes back empty, we create the connection anyway +and warn — the user must set that secret manually on the target. + +Connection slugs are preserved: unlike createWorkspace / createPipeline (which +re-derive the slug/code server-side), createConnection honors a caller-supplied +slug and only suffixes it on collision. The target workspace is fresh, so the +source slug carries over intact, keeping pipeline parameters that reference a +connection by slug valid. +""" + +import sys +from dataclasses import dataclass, field +from typing import Any + +from openhexa.graphql.graphql_client.client import Client +from openhexa.graphql.graphql_client.input_types import ( + ConnectionFieldInput, + ConnectionType, + CreateConnectionInput, +) + +from .transport import GraphQLError, gql + + +# `workspace.connections` is a (non-paginated) field on Workspace returning the +# full list; the SDK's workspace() doesn't pull fields, so we query it raw. +LIST_CONNECTIONS_QUERY = """ +query ListConnections($slug: String!) { + workspace(slug: $slug) { + connections { + id name slug description type + fields { code value secret } + } + } +} +""" + + +@dataclass +class ConnectionsResult: + """What migrate_all() did, for the orchestrator to print.""" + + created: list[tuple[str, int]] = field(default_factory=list) + """(slug, field_count) for each connection created on target.""" + + skipped: list[str] = field(default_factory=list) + """Slugs that already existed on target.""" + + failed: list[str] = field(default_factory=list) + """Slugs whose creation failed; user must handle manually.""" + + warnings: list[str] = field(default_factory=list) + """Human-readable warnings (e.g. secret fields created with no value).""" + + +def _list_connections(client: Client, slug: str) -> list[dict[str, Any]] | None: + """Return the workspace's connections, or None if the workspace is absent.""" + data = gql(client, LIST_CONNECTIONS_QUERY, {"slug": slug}, "ListConnections") + ws = data["workspace"] + if ws is None: + return None + return list(ws["connections"]) + + +def _build_fields( + conn: dict[str, Any], result: ConnectionsResult +) -> list[ConnectionFieldInput]: + """Map source fields to ConnectionFieldInput, warning on empty secrets.""" + fields_in: list[ConnectionFieldInput] = [] + for f in conn.get("fields") or []: + value = f.get("value") + if f.get("secret") and not value: + result.warnings.append( + f"connection '{conn['slug']}' field '{f['code']}' is a secret " + "with no readable value on source — created empty; set it " + "manually on the target." + ) + fields_in.append( + ConnectionFieldInput( + code=f["code"], secret=bool(f.get("secret")), value=value + ) + ) + return fields_in + + +def migrate_all( + source: Client, + target: Client, + source_slug: str, + target_slug: str, +) -> ConnectionsResult: + """Copy every connection from `source_slug` into `target_slug`.""" + result = ConnectionsResult() + + print("=> Listing source connections ...") + conns = _list_connections(source, source_slug) + if conns is None: + raise GraphQLError( + f"source workspace '{source_slug}' not found while listing connections" + ) + print(f" found {len(conns)} connection(s)") + + existing = {c["slug"] for c in (_list_connections(target, target_slug) or [])} + + for conn in conns: + slug = conn["slug"] + if slug in existing: + print(f" - connection '{slug}' already exists on target — skipping") + result.skipped.append(slug) + continue + try: + print(f" - migrating connection '{slug}' ...") + fields_in = _build_fields(conn, result) + res = target.create_connection( + input=CreateConnectionInput( + workspace_slug=target_slug, + name=conn["name"], + slug=slug, + type=ConnectionType(conn["type"]), + description=conn.get("description") or "", + fields=fields_in, + ) + ) + if not res.success or res.connection is None: + raise GraphQLError( + f"createConnection failed for '{slug}': " + + ",".join(e.value for e in (res.errors or [])) + ) + result.created.append((slug, len(fields_in))) + except GraphQLError as exc: + # Collect and continue (like files.py) so one bad connection + # doesn't abort the rest of the migration. + print(f"\tFAILED to migrate connection '{slug}': {exc}", file=sys.stderr) + result.failed.append(slug) + + return result diff --git a/script/migrate_workspace.py b/script/migrate_workspace.py index 76dc1c1..86dc721 100755 --- a/script/migrate_workspace.py +++ b/script/migrate_workspace.py @@ -10,13 +10,17 @@ - All files in the workspace bucket are copied from source to target (must run before pipelines, since notebook pipelines require their .ipynb to exist on the target before createPipeline succeeds). + - Connections (including secret field values) are recreated via + createConnection. Secret values come through only because we + authenticate as a superuser; any that come back empty are created + empty and flagged in the summary for manual entry. - Each pipeline is created via createPipeline; each zipfile-pipeline version is uploaded via uploadPipeline; pipeline-level schedule / config / webhookEnabled / scheduledPipelineVersionId are applied via updatePipeline. -Out of scope: members, invitations, connections, recipients, templates, -runs, shortcuts, datasets, database tables. +Out of scope: members, invitations, recipients, templates, runs, +shortcuts, datasets, database tables. """ import argparse @@ -34,7 +38,8 @@ ) sys.exit(1) -from migrate_lib import files, pipelines, transport, workspaces +from migrate_lib import connections, files, pipelines, transport, workspaces +from migrate_lib.connections import ConnectionsResult from migrate_lib.files import FilesResult from migrate_lib.pipelines import PipelinesResult from migrate_lib.transport import GraphQLError, build_client @@ -64,15 +69,21 @@ def migrate( ) files_result = files.migrate_all(source, target, source_slug, target_slug) + connections_result = connections.migrate_all( + source, target, source_slug, target_slug + ) pipelines_result = pipelines.migrate_all(source, target, source_slug, target_slug) - _print_summary(src_ws.name, target_slug, files_result, pipelines_result) + _print_summary( + src_ws.name, target_slug, files_result, connections_result, pipelines_result + ) def _print_summary( src_ws_name: str, target_slug: str, files_result: FilesResult, + connections_result: ConnectionsResult, pipelines_result: PipelinesResult, ) -> None: print("\n=== Migration summary ===") @@ -86,6 +97,26 @@ def _print_summary( ) for path in files_result.failed: print(f" * {path}") + print(f"Connections created: {len(connections_result.created)}") + for slug, n_fields in connections_result.created: + print(f" * {slug} ({n_fields} field(s))") + if connections_result.skipped: + print( + f"Connections skipped (already existed): {len(connections_result.skipped)}" + ) + for slug in connections_result.skipped: + print(f" * {slug}") + if connections_result.failed: + print( + f"Connections that could NOT be migrated " + f"({len(connections_result.failed)} — handle manually):" + ) + for slug in connections_result.failed: + print(f" * {slug}") + if connections_result.warnings: + print("Connection warnings:") + for w in connections_result.warnings: + print(f" - {w}") print(f"Pipelines created: {len(pipelines_result.created)}") for code, vnames in pipelines_result.created: print(f" * {code}") @@ -99,10 +130,6 @@ def _print_summary( print("Warnings:") for w in pipelines_result.warnings: print(f" - {w}") - print( - "Note: connections were not migrated; recreate them locally if " - "any pipeline depends on connection-typed parameters." - ) def main() -> int: From 31e1347969af78ed0f99e3ab7ca5a046bc21464f Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Wed, 27 May 2026 10:29:00 +0200 Subject: [PATCH 13/14] Add script to migrate template pipelines --- script/migrate_lib/templates.py | 604 ++++++++++++++++++++++++++++++++ script/migrate_lib/transport.py | 16 +- script/migrate_templates.py | 150 ++++++++ 3 files changed, 767 insertions(+), 3 deletions(-) create mode 100644 script/migrate_lib/templates.py create mode 100755 script/migrate_templates.py diff --git a/script/migrate_lib/templates.py b/script/migrate_lib/templates.py new file mode 100644 index 0000000..c94630a --- /dev/null +++ b/script/migrate_lib/templates.py @@ -0,0 +1,604 @@ +"""Pipeline template migration: server-wide templates copied to target. + +Templates are globally visible, so this runs once per target server, +independently of any workspace migration. Each source template is recreated +on the target by: + +1. Locating a host pipeline that can back the template on target: + - Prefer the *already-migrated* source pipeline (in its original + workspace on target) if it exists — keeps the underlying pipeline + visible inside its real workspace. + - Otherwise create a host pipeline inside a dedicated + "Template pipelines" workspace. +2. Ensuring each template version's underlying pipeline version exists on + the host pipeline (upload the source zipfile if missing). +3. Calling ``createPipelineTemplateVersion`` per missing template version, + in source ``versionNumber`` order. The first call also creates the + ``PipelineTemplate`` on target (via ``Pipeline.get_or_create_template``). +4. Applying metadata (description / functionalType / tags) via + ``updatePipelineTemplate``. + +Idempotent: matches templates by name (globally unique) and existing +versions by ``versionNumber``. +""" + +import sys +import time +from dataclasses import dataclass, field +from typing import Any + +from openhexa.graphql.graphql_client.client import Client +from openhexa.graphql.graphql_client.input_types import ( + CreatePipelineInput, + CreateWorkspaceInput, +) + +from .pipelines import _upload_version +from .transport import GraphQLError, gql + + +TEMPLATES_PAGE_SIZE = 50 +TEMPLATE_VERSIONS_PAGE_SIZE = 50 +PIPELINE_VERSIONS_PAGE_SIZE = 100 + +HOST_WORKSPACE_NAME = "Template pipelines" + +# Back-off between createPipelineTemplateVersion calls. Each one triggers +# auto-update of every dependent pipeline server-side, which can be heavy. +PER_VERSION_DELAY_SECONDS = 0.5 + + +LIST_SOURCE_TEMPLATES_QUERY = """ +query ListSourceTemplates($page: Int!, $perPage: Int!) { + pipelineTemplates(page: $page, perPage: $perPage) { + pageNumber totalPages totalItems + items { + id name code description functionalType + validatedAt + tags { name } + sourcePipeline { id code name workspace { slug } } + } + } +} +""" + +LIST_TARGET_TEMPLATES_QUERY = """ +query ListTargetTemplates($page: Int!, $perPage: Int!) { + pipelineTemplates(page: $page, perPage: $perPage) { + pageNumber totalPages + items { + id name code + sourcePipeline { id code workspace { slug } } + } + } +} +""" + +TEMPLATE_VERSIONS_QUERY = """ +query TemplateVersions($code: String!, $page: Int!, $perPage: Int!) { + templateByCode(code: $code) { + id name + versions(page: $page, perPage: $perPage) { + pageNumber totalPages + items { + id versionNumber changelog + sourcePipelineVersion { + id versionNumber name description externalLink config timeout + zipfile + parameters { + code name type multiple required default help + widget connection choices + } + } + } + } + } +} +""" + +TARGET_TEMPLATE_VERSIONS_QUERY = """ +query TargetTemplateVersions($code: String!, $page: Int!, $perPage: Int!) { + templateByCode(code: $code) { + id + versions(page: $page, perPage: $perPage) { + pageNumber totalPages + items { id versionNumber } + } + } +} +""" + +HOST_PIPELINE_VERSIONS_QUERY = """ +query HostPipelineVersions($id: UUID!, $page: Int!, $perPage: Int!) { + pipeline(id: $id) { + id + versions(page: $page, perPage: $perPage) { + pageNumber totalPages + items { id versionNumber } + } + } +} +""" + +CREATE_TEMPLATE_VERSION_MUTATION = """ +mutation CreateTemplateVersion($input: CreatePipelineTemplateVersionInput!) { + createPipelineTemplateVersion(input: $input) { + success errors + pipelineTemplate { id name code } + } +} +""" + +UPDATE_TEMPLATE_MUTATION = """ +mutation UpdateTemplate($input: UpdateTemplateInput!) { + updatePipelineTemplate(input: $input) { + success errors + template { id } + } +} +""" + + +@dataclass +class TemplatesResult: + """What migrate_all() did, for the orchestrator to print.""" + + created: list[str] = field(default_factory=list) + """Names of templates newly created on target (didn't exist before).""" + + versions_added: list[tuple[str, list[int]]] = field(default_factory=list) + """(template_name, [versionNumber, ...]) for versions added this run.""" + + skipped_unchanged: list[str] = field(default_factory=list) + """Names of templates that were already fully present on target.""" + + warnings: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Orchestration +# --------------------------------------------------------------------------- + + +def migrate_all(source: Client, target: Client) -> TemplatesResult: + """Migrate every template from `source` into `target`.""" + result = TemplatesResult() + + print(f"=> Ensuring '{HOST_WORKSPACE_NAME}' workspace exists on target ...") + host_ws_slug = _ensure_host_workspace(target) + print(f" host workspace slug: '{host_ws_slug}'") + + # Listed once so re-runs find host pipelines whose code the server + # re-derived from name (e.g. underscores -> dashes), not the source's + # raw code. + host_pipelines_by_name = _list_pipelines_by_name(target, host_ws_slug) + + print("=> Listing source templates ...") + src_templates = _list_all_source_templates(source) + print(f" found {len(src_templates)} template(s)") + + print("=> Listing existing target templates ...") + target_by_name = _list_all_target_templates(target) + print(f" target already has {len(target_by_name)} template(s)") + + for src in src_templates: + try: + _migrate_one( + source, + target, + src, + target_by_name, + host_ws_slug, + host_pipelines_by_name, + result, + ) + except GraphQLError as exc: + msg = f"template '{src['name']}' (code='{src['code']}'): {exc}" + result.warnings.append(msg) + print(f" FAILED: {msg}", file=sys.stderr) + + return result + + +def _migrate_one( + source: Client, + target: Client, + src_template: dict[str, Any], + target_by_name: dict[str, dict[str, Any]], + host_ws_slug: str, + host_pipelines_by_name: dict[str, str], + result: TemplatesResult, +) -> None: + name = src_template["name"] + code = src_template["code"] + print(f" - template '{name}' (code='{code}') ...") + + src_pipe = src_template.get("sourcePipeline") + if src_pipe is None: + result.warnings.append( + f"template '{name}' has no source pipeline on source — skipped." + ) + return + + existing_target = target_by_name.get(name) + + # 1. Decide host (workspace_slug, pipeline_code) and existing version numbers. + if existing_target is not None: + host_pipe_code = existing_target["sourcePipeline"]["code"] + host_pipe_ws = existing_target["sourcePipeline"]["workspace"]["slug"] + existing_version_numbers = _fetch_target_template_version_numbers(target, code) + print( + f" already exists on target at " + f"'{host_pipe_ws}/{host_pipe_code}'; " + f"{len(existing_version_numbers)} version(s) present" + ) + else: + host_pipe_code, host_pipe_ws = _resolve_or_create_host_pipeline( + target, src_pipe, host_ws_slug, host_pipelines_by_name + ) + existing_version_numbers = set() + + # 2. Resolve host pipeline id and the versions it already has (by number). + host_pipe = target.pipeline( + workspace_slug=host_pipe_ws, pipeline_code=host_pipe_code + ) + if host_pipe is None: + raise GraphQLError( + f"host pipeline '{host_pipe_ws}/{host_pipe_code}' could not be looked up" + ) + host_pipe_id = str(host_pipe.id) + host_versions_by_number = _fetch_pipeline_versions_by_number(target, host_pipe_id) + + # 3. Add missing template versions. + src_versions = _fetch_source_template_versions(source, code) + if not src_versions: + result.warnings.append( + f"template '{name}' has no versions on source — skipped." + ) + return + + added: list[int] = [] + target_template_id: str | None = ( + existing_target["id"] if existing_target is not None else None + ) + + for ver in src_versions: + vnum = ver["versionNumber"] + if vnum in existing_version_numbers: + continue + + src_pipe_ver = ver["sourcePipelineVersion"] + target_pv_id = _ensure_host_pipeline_version( + target, + host_pipe_ws, + host_pipe_code, + src_pipe_ver, + host_versions_by_number, + ) + + input_: dict[str, Any] = { + "workspaceSlug": host_pipe_ws, + "pipelineId": host_pipe_id, + "pipelineVersionId": target_pv_id, + "name": name, + "code": code, + "description": src_template.get("description") or "", + "changelog": ver.get("changelog") or None, + # versionName / documentation are not queryable on older source + # servers, so let the target extract documentation from the + # zipfile's README on its own (server default behavior). + } + input_ = {k: v for k, v in input_.items() if v is not None} + + data = gql( + target, + CREATE_TEMPLATE_VERSION_MUTATION, + {"input": input_}, + "CreateTemplateVersion", + ) + cptv = data["createPipelineTemplateVersion"] + if not cptv["success"]: + raise GraphQLError( + f"createPipelineTemplateVersion v{vnum} failed: " + + ",".join(cptv.get("errors") or []) + ) + if target_template_id is None and cptv.get("pipelineTemplate"): + target_template_id = cptv["pipelineTemplate"]["id"] + + print(f" added template version v{vnum}") + added.append(vnum) + time.sleep(PER_VERSION_DELAY_SECONDS) + + if added: + if existing_target is None: + result.created.append(name) + result.versions_added.append((name, added)) + else: + result.skipped_unchanged.append(name) + + # 4. Apply metadata (idempotent re-application is fine). + if target_template_id is not None: + _apply_metadata(target, src_template, target_template_id) + + # 5. Warn on validated-on-source. + if src_template.get("validatedAt"): + result.warnings.append( + f"template '{name}' is validated on source; target stays community " + "(validatedAt is not settable via the API)." + ) + + +# --------------------------------------------------------------------------- +# Host workspace & host pipeline +# --------------------------------------------------------------------------- + + +def _ensure_host_workspace(target: Client) -> str: + """Return the slug of an existing target workspace named HOST_WORKSPACE_NAME, creating it if needed.""" + page = 1 + while True: + page_result = target.workspaces(page=page, per_page=100) + for ws in page_result.items: + if ws.name == HOST_WORKSPACE_NAME: + return ws.slug + if page >= page_result.total_pages or page_result.total_pages == 0: + break + page += 1 + + created = target.create_workspace( + input=CreateWorkspaceInput( + name=HOST_WORKSPACE_NAME, + description=( + "Host workspace for source pipelines backing migrated templates. " + "Auto-created by migrate_templates.py." + ), + countries=[], + load_sample_data=False, + configuration={}, + ) + ) + if not created.success or created.workspace is None: + raise GraphQLError( + f"could not create '{HOST_WORKSPACE_NAME}' workspace: " + + ",".join(created.errors or []) + ) + return created.workspace.slug + + +def _resolve_or_create_host_pipeline( + target: Client, + src_pipeline: dict[str, Any], + host_ws_slug: str, + host_pipelines_by_name: dict[str, str], +) -> tuple[str, str]: + """Return (pipeline_code, workspace_slug) of the host pipeline to use. + + Reuses the source pipeline in its original workspace on target if it + exists there (because the workspace migration already brought it + over). Otherwise reuses or creates a host pipeline in + HOST_WORKSPACE_NAME, matched by source pipeline name (since the + server re-derives code from name on create, e.g. dropping + underscores). + """ + src_code = src_pipeline["code"] + src_name = src_pipeline["name"] or src_code + src_ws = src_pipeline.get("workspace") or {} + src_ws_slug = src_ws.get("slug") + + if src_ws_slug: + existing = target.pipeline(workspace_slug=src_ws_slug, pipeline_code=src_code) + if existing is not None: + print( + f" reusing already-migrated source pipeline " + f"'{src_ws_slug}/{existing.code}'" + ) + return existing.code, src_ws_slug + + # Fallback: ensure pipeline exists inside the dedicated host workspace. + if src_name in host_pipelines_by_name: + host_code = host_pipelines_by_name[src_name] + print(f" reusing existing host pipeline '{host_ws_slug}/{host_code}'") + return host_code, host_ws_slug + + create_result = target.create_pipeline( + input=CreatePipelineInput( + name=src_name, + workspace_slug=host_ws_slug, + ) + ) + if not create_result.success or create_result.pipeline is None: + raise GraphQLError( + f"createPipeline for template host '{src_code}' failed: " + + ",".join(e.value for e in (create_result.errors or [])) + ) + new_code = create_result.pipeline.code + host_pipelines_by_name[src_name] = new_code + print(f" created host pipeline '{host_ws_slug}/{new_code}'") + return new_code, host_ws_slug + + +def _ensure_host_pipeline_version( + target: Client, + ws_slug: str, + pipeline_code: str, + src_pipeline_version: dict[str, Any], + host_versions_by_number: dict[int, str], +) -> str: + """Return the target pipelineVersion id for the given source pipeline version. + + Reuses an existing version on the host pipeline if one with the same + versionNumber exists; otherwise uploads the source zipfile and + records the new id in `host_versions_by_number`. + """ + src_vnum = src_pipeline_version["versionNumber"] + if src_vnum in host_versions_by_number: + return host_versions_by_number[src_vnum] + + if not src_pipeline_version.get("zipfile"): + raise GraphQLError( + f"source pipeline version v{src_vnum} has no zipfile; cannot upload" + ) + + up = _upload_version(target, ws_slug, pipeline_code, src_pipeline_version) + print(f" uploaded host pipeline version v{src_vnum} -> {up['versionName']}") + host_versions_by_number[up["versionNumber"]] = up["id"] + return up["id"] + + +# --------------------------------------------------------------------------- +# Source / target fetch helpers +# --------------------------------------------------------------------------- + + +def _list_pipelines_by_name(target: Client, ws_slug: str) -> dict[str, str]: + """List pipelines in `ws_slug`, return {pipeline.name: pipeline.code}.""" + by_name: dict[str, str] = {} + page = 1 + while True: + result = target.pipelines(workspace_slug=ws_slug, page=page, per_page=100) + for item in result.items: + if item.name: + by_name[item.name] = item.code + if page >= result.total_pages or result.total_pages == 0: + break + page += 1 + return by_name + + +def _list_all_source_templates(source: Client) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + page = 1 + while True: + data = gql( + source, + LIST_SOURCE_TEMPLATES_QUERY, + {"page": page, "perPage": TEMPLATES_PAGE_SIZE}, + "ListSourceTemplates", + ) + page_data = data["pipelineTemplates"] + items.extend(page_data["items"]) + if page >= page_data["totalPages"] or page_data["totalPages"] == 0: + break + page += 1 + return items + + +def _list_all_target_templates(target: Client) -> dict[str, dict[str, Any]]: + """Return target templates keyed by name (globally unique).""" + by_name: dict[str, dict[str, Any]] = {} + page = 1 + while True: + data = gql( + target, + LIST_TARGET_TEMPLATES_QUERY, + {"page": page, "perPage": TEMPLATES_PAGE_SIZE}, + "ListTargetTemplates", + ) + page_data = data["pipelineTemplates"] + for item in page_data["items"]: + by_name[item["name"]] = item + if page >= page_data["totalPages"] or page_data["totalPages"] == 0: + break + page += 1 + return by_name + + +def _fetch_source_template_versions(source: Client, code: str) -> list[dict[str, Any]]: + versions: list[dict[str, Any]] = [] + page = 1 + while True: + data = gql( + source, + TEMPLATE_VERSIONS_QUERY, + {"code": code, "page": page, "perPage": TEMPLATE_VERSIONS_PAGE_SIZE}, + "TemplateVersions", + ) + t = data["templateByCode"] + if t is None: + return [] + pg = t["versions"] + versions.extend(pg["items"]) + if page >= pg["totalPages"] or pg["totalPages"] == 0: + break + page += 1 + return sorted(versions, key=lambda v: v["versionNumber"]) + + +def _fetch_target_template_version_numbers(target: Client, code: str) -> set[int]: + out: set[int] = set() + page = 1 + while True: + data = gql( + target, + TARGET_TEMPLATE_VERSIONS_QUERY, + {"code": code, "page": page, "perPage": TEMPLATE_VERSIONS_PAGE_SIZE}, + "TargetTemplateVersions", + ) + t = data["templateByCode"] + if t is None: + return out + pg = t["versions"] + for v in pg["items"]: + out.add(v["versionNumber"]) + if page >= pg["totalPages"] or pg["totalPages"] == 0: + break + page += 1 + return out + + +def _fetch_pipeline_versions_by_number( + target: Client, pipeline_id: str +) -> dict[int, str]: + out: dict[int, str] = {} + page = 1 + while True: + data = gql( + target, + HOST_PIPELINE_VERSIONS_QUERY, + {"id": pipeline_id, "page": page, "perPage": PIPELINE_VERSIONS_PAGE_SIZE}, + "HostPipelineVersions", + ) + p = data["pipeline"] + if p is None: + return out + pg = p["versions"] + for v in pg["items"]: + out[v["versionNumber"]] = v["id"] + if page >= pg["totalPages"] or pg["totalPages"] == 0: + break + page += 1 + return out + + +# --------------------------------------------------------------------------- +# Metadata +# --------------------------------------------------------------------------- + + +def _apply_metadata( + target: Client, src_template: dict[str, Any], target_template_id: str +) -> None: + tags = [t["name"] for t in (src_template.get("tags") or [])] + input_: dict[str, Any] = { + "id": target_template_id, + "description": src_template.get("description") or None, + "functionalType": src_template.get("functionalType") or None, + "tags": tags or None, + } + input_ = {k: v for k, v in input_.items() if v is not None} + if list(input_.keys()) == ["id"]: + return + + data = gql( + target, + UPDATE_TEMPLATE_MUTATION, + {"input": input_}, + "UpdateTemplate", + ) + upd = data["updatePipelineTemplate"] + if not upd["success"]: + print( + f" warning: updatePipelineTemplate failed for " + f"'{src_template['name']}': " + ",".join(upd.get("errors") or []), + file=sys.stderr, + ) diff --git a/script/migrate_lib/transport.py b/script/migrate_lib/transport.py index f245315..05aa27c 100644 --- a/script/migrate_lib/transport.py +++ b/script/migrate_lib/transport.py @@ -20,12 +20,16 @@ def _dbg(msg: str) -> None: def _short(value: Any, limit: int = 200) -> str: """Render a value for debug output without dumping huge zipfiles.""" if isinstance(value, dict): - return "{" + ", ".join(f"{k}={_short(v, limit)}" for k, v in value.items()) + "}" + return ( + "{" + ", ".join(f"{k}={_short(v, limit)}" for k, v in value.items()) + "}" + ) if isinstance(value, list): head = ", ".join(_short(v, limit) for v in value[:3]) return f"[{head}{', ...' if len(value) > 3 else ''}] (n={len(value)})" s = repr(value) - return s if len(s) <= limit else s[:limit] + f"... " + return ( + s if len(s) <= limit else s[:limit] + f"... " + ) class GraphQLError(RuntimeError): @@ -62,7 +66,13 @@ def build_client(server_url: str, email: str, password: str, *, label: str) -> C `label` is used only to make the error message ("source"/"target") clearer. """ - http = httpx.Client(headers={"User-Agent": "openhexa-migrate/1.0"}) + # 120s read timeout: createPipelineTemplateVersion can fan out to + # auto-update every pipeline derived from the template, which is slow + # on prod and exceeds httpx's 5s default. + http = httpx.Client( + headers={"User-Agent": "openhexa-migrate/1.0"}, + timeout=httpx.Timeout(120.0), + ) # Prime CSRF cookie. Defensive — GraphQLView is csrf_exempt on the # current backend, but a future change would otherwise silently # break every mutation. diff --git a/script/migrate_templates.py b/script/migrate_templates.py new file mode 100755 index 0000000..004b691 --- /dev/null +++ b/script/migrate_templates.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Migrate all pipeline templates from one OpenHEXA server to another. + +Templates are globally visible on a server, so this script runs once per +target server (independently of any workspace migration). For each +source template it: + + - finds or creates a host pipeline on target (the source pipeline in + its original workspace if already migrated, otherwise inside a + "Template pipelines" workspace this script creates as needed), + - uploads any missing pipeline versions, + - calls ``createPipelineTemplateVersion`` per missing template version + (matched against target by name + ``versionNumber``), + - applies ``description`` / ``functionalType`` / ``tags`` via + ``updatePipelineTemplate``. + +Re-runnable. Auth: Django superuser email/password on both ends, same +shape as ``migrate_workspace.py``. + +Out of scope: ``validatedAt`` (not settable via the API; warned about); +relinking pipelines that were created from templates on source back to +those templates on target. +""" + +import argparse +import sys + +try: + from openhexa.graphql.graphql_client.exceptions import ( + GraphQLClientGraphQLMultiError, + GraphQLClientHttpError, + ) +except ImportError: + sys.stderr.write( + "error: this script requires the 'openhexa.sdk' package " + "(install with: pip install openhexa.sdk)\n" + ) + sys.exit(1) + +from migrate_lib import templates, transport +from migrate_lib.templates import TemplatesResult +from migrate_lib.transport import GraphQLError, build_client + + +DEFAULT_SOURCE_URL = "https://api.openhexa.org/graphql/" +DEFAULT_TARGET_URL = "http://localhost:8000/graphql/" + + +def _print_summary(result: TemplatesResult) -> None: + print("\n=== Template migration summary ===") + print(f"Templates created on target: {len(result.created)}") + for name in result.created: + print(f" * {name}") + if result.versions_added: + print( + f"Template versions added: {sum(len(v) for _, v in result.versions_added)}" + ) + for name, nums in result.versions_added: + nums_str = ", ".join(f"v{n}" for n in nums) + print(f" * {name}: {nums_str}") + if result.skipped_unchanged: + print( + f"Templates already up to date (skipped): {len(result.skipped_unchanged)}" + ) + for name in result.skipped_unchanged: + print(f" * {name}") + if result.warnings: + print("Warnings:") + for w in result.warnings: + print(f" - {w}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Migrate every pipeline template from a source " + "OpenHEXA server to a target server.", + ) + parser.add_argument( + "--source-url", + default=DEFAULT_SOURCE_URL, + help=f"Source GraphQL endpoint (default: {DEFAULT_SOURCE_URL}).", + ) + parser.add_argument( + "--source-email", + required=True, + help="Email of the Django superuser on the source server.", + ) + parser.add_argument( + "--source-password", + required=True, + help="Password of the Django superuser on the source server.", + ) + parser.add_argument( + "--target-url", + default=DEFAULT_TARGET_URL, + help=f"Target GraphQL endpoint (default: {DEFAULT_TARGET_URL}).", + ) + parser.add_argument( + "--target-email", + required=True, + help="Email of the Django superuser on the target server.", + ) + parser.add_argument( + "--target-password", + required=True, + help="Password of the Django superuser on the target server.", + ) + parser.add_argument( + "--debug", + "-v", + action="store_true", + help="Print each GraphQL request (operation + variables) and response status.", + ) + args = parser.parse_args() + + transport.DEBUG = args.debug + + print(f"Source: {args.source_url}") + print(f"Target: {args.target_url}") + + try: + source = build_client( + args.source_url, args.source_email, args.source_password, label="source" + ) + target = build_client( + args.target_url, args.target_email, args.target_password, label="target" + ) + result = templates.migrate_all(source, target) + _print_summary(result) + except GraphQLClientHttpError as exc: + sys.stderr.write( + f"\nerror: HTTP {exc.status_code} from server:\n{exc.response.text[:4000]}\n" + ) + return 1 + except GraphQLClientGraphQLMultiError as exc: + sys.stderr.write("\nerror: GraphQL errors from server:\n") + for e in exc.errors: + sys.stderr.write(f" - {e.message}") + if e.path: + sys.stderr.write(f" (at path: {'.'.join(map(str, e.path))})") + sys.stderr.write("\n") + return 1 + except GraphQLError as exc: + sys.stderr.write(f"\nerror: {exc}\n") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 58622b76598486d3a22823848d5c3ba5064c118c Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Wed, 27 May 2026 12:05:31 +0200 Subject: [PATCH 14/14] Improve BLSQ official templates wording --- script/migrate_lib/templates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/migrate_lib/templates.py b/script/migrate_lib/templates.py index c94630a..ab2cae2 100644 --- a/script/migrate_lib/templates.py +++ b/script/migrate_lib/templates.py @@ -323,7 +323,8 @@ def _migrate_one( # 5. Warn on validated-on-source. if src_template.get("validatedAt"): result.warnings.append( - f"template '{name}' is validated on source; target stays community " + f"template '{name}' is an official Bluesquare template on source; " + "on target it will appear as a community template " "(validatedAt is not settable via the API)." )