diff --git a/CHANGES.md b/CHANGES.md index 4740280..2d79474 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,12 @@ ## 2.4.1 (unreleased) -- No changes yet. +- Feature: Generate generic `convert-import.conf` and `convert-export.conf` + for all non-filestorage backends (RelStorage, PGJsonb, ZEO) using the + storage-agnostic [zodb-convert](https://pypi.org/project/zodb-convert/) + tool. Deprecate RelStorage-specific `zodbconvert` configuration in favor + of `zodb-convert`. Add migration how-to guide. + [@jensens] ## 2.4.0 (2026-02-25) diff --git a/cookiecutter.json b/cookiecutter.json index dbdb6ca..3aee3f1 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -154,6 +154,11 @@ "db_pgjsonb_blob_cache_dir": "", "db_pgjsonb_blob_cache_size": "", + "db_convert_import_filestorage_location": "", + "db_convert_import_blobs_location": "", + "db_convert_export_filestorage_location": "", + "db_convert_export_blobs_location": "", + "db_z3blobs_enabled": false, "db_z3blobs_bucket_name": "", "db_z3blobs_s3_prefix": "", diff --git a/docs/sources/explanation/storage-backends.md b/docs/sources/explanation/storage-backends.md index ce935b6..88c6590 100644 --- a/docs/sources/explanation/storage-backends.md +++ b/docs/sources/explanation/storage-backends.md @@ -161,5 +161,12 @@ For most teams, the decision comes down to: 5. **Need S3 blob storage with direct, relstorage, or zeo?** Enable `db_z3blobs_enabled`. All four backends support the same ZODB API, so switching between them is a -configuration change (plus data migration). Your application code does not -need to change. +configuration change plus a one-time data migration. The +[zodb-convert](https://pypi.org/project/zodb-convert/) tool handles this +migration generically -- it can copy data between any two ZODB-compatible +storages. Because `cookiecutter-zope-instance` generates a complete +`zope.conf` for each backend, `zodb-convert` can read the storage +configuration directly from these files. See {doc}`/how-to/migrate-storage` +for a step-by-step guide. + +Your application code does not need to change. diff --git a/docs/sources/how-to/configure-pgjsonb.md b/docs/sources/how-to/configure-pgjsonb.md index 86f319c..88ae530 100644 --- a/docs/sources/how-to/configure-pgjsonb.md +++ b/docs/sources/how-to/configure-pgjsonb.md @@ -98,6 +98,8 @@ used** with PGJsonb. PGJsonb manages blob storage through its own settings ## Next steps +- {doc}`migrate-storage` -- Migrate data from an existing storage backend + into PGJsonb using `zodb-convert`. - {doc}`/reference/database-pgjsonb` -- Full reference for all PGJsonb options including connection pool, cache, and S3 settings. - {doc}`/explanation/storage-backends` -- Comparison of all storage backends. diff --git a/docs/sources/how-to/configure-relstorage.md b/docs/sources/how-to/configure-relstorage.md index fe3b035..b067f42 100644 --- a/docs/sources/how-to/configure-relstorage.md +++ b/docs/sources/how-to/configure-relstorage.md @@ -104,8 +104,18 @@ for RelStorage command-line utilities: settings are provided. Used with `zodbconvert` to import data from a filestorage. +:::{tip} +For migrating data between storage backends, the generic +[zodb-convert](https://pypi.org/project/zodb-convert/) tool is now the +recommended approach. It works with all storage backends and can read storage +configuration directly from `zope.conf` files. +See {doc}`migrate-storage` for a step-by-step guide. +::: + ## Next steps +- {doc}`migrate-storage` -- Migrate data between any storage backends using + `zodb-convert`. - {doc}`/reference/database-relstorage` -- Full reference for all RelStorage options including caching, replication, MySQL, Oracle, and SQLite settings. - {doc}`/explanation/storage-backends` -- Comparison of all storage backends. diff --git a/docs/sources/how-to/index.md b/docs/sources/how-to/index.md index d1c84c5..1b3f58f 100644 --- a/docs/sources/how-to/index.md +++ b/docs/sources/how-to/index.md @@ -11,6 +11,10 @@ Goal-oriented guides for common tasks. - {doc}`configure-zeo` - {doc}`configure-z3blobs` +## Storage Migration + +- {doc}`migrate-storage` + ## Web Layer - {doc}`configure-cors` @@ -31,6 +35,7 @@ configure-relstorage configure-pgjsonb configure-zeo configure-z3blobs +migrate-storage configure-cors use-environment-variables upgrade-v1-to-v2 diff --git a/docs/sources/how-to/migrate-storage.md b/docs/sources/how-to/migrate-storage.md new file mode 100644 index 0000000..c455ca6 --- /dev/null +++ b/docs/sources/how-to/migrate-storage.md @@ -0,0 +1,140 @@ +# Migrate Data Between Storage Backends + + + +This guide shows how to copy ZODB data from one storage backend to another +using the generic [zodb-convert](https://pypi.org/project/zodb-convert/) tool. + +## Prerequisites + +- The `zodb-convert` package installed in your virtual environment +- The storage-specific packages for both source and destination installed + (e.g. `ZODB` for filestorage, `RelStorage` for relstorage, `zodb-pgjsonb` + for pgjsonb) + +## Approach 1: Generated configuration files + +When your instance uses a non-filestorage backend (RelStorage, PGJsonb, or +ZEO), the cookiecutter generates `convert-import.conf` and +`convert-export.conf` files in the `etc/` directory. These are ready-made +configuration files for `zodb-convert` that convert between your configured +backend and a FileStorage. + +### Step 1: Set the conversion paths + +In your `instance.yaml`, provide the FileStorage paths for the "other side" +of the conversion: + +```yaml +default_context: + db_storage: pgjsonb + db_pgjsonb_dsn: "dbname='zodb' user='zodb' host='localhost' port='5433'" + + # Conversion settings (FileStorage side) + db_convert_import_filestorage_location: var/import/Data.fs + db_convert_import_blobs_location: var/import/blobs + db_convert_export_filestorage_location: var/export/Data.fs + db_convert_export_blobs_location: var/export/blobs +``` + +These settings work with all non-filestorage backends. + +### Step 2: Import data from FileStorage + +Place your `Data.fs` and blobs at the import locations, then run: + +```bash +zodb-convert etc/convert-import.conf +``` + +### Step 3: Export data to FileStorage + +To create a portable FileStorage backup: + +```bash +zodb-convert etc/convert-export.conf +``` + +## Approach 2: Using existing zope.conf files + +If you have two Zope instances with different storage backends, `zodb-convert` +can read storage configuration directly from their `zope.conf` files. No +extra configuration files are needed. + +### Step 1: Generate the destination instance + +Create a second instance configured for the new storage backend: + +```yaml +# instance-new.yaml +default_context: + db_storage: pgjsonb + db_pgjsonb_dsn: "dbname='zodb' user='zodb' host='localhost' port='5433'" +``` + +Run cookiecutter to generate: + +```bash +cookiecutter -f --no-input --config-file instance-new.yaml \ + gh:plone/cookiecutter-zope-instance +``` + +### Step 2: Run the conversion + +```bash +zodb-convert \ + --source-zope-conf /old-instance/etc/zope.conf \ + --dest-zope-conf /new-instance/etc/zope.conf +``` + +### Specifying a named database + +If your `zope.conf` defines multiple `` sections: + +```bash +zodb-convert \ + --source-zope-conf old/etc/zope.conf --source-db main \ + --dest-zope-conf new/etc/zope.conf --dest-db main +``` + +The default database name is `main`. + +## Useful options + +- `--dry-run` -- preview what would be copied without writing data +- `--incremental` -- resume from the last transaction in the destination + (useful for large databases or after interruptions) +- `-v` -- show INFO-level progress; `-vv` for DEBUG-level detail + +## Common migration scenarios + +| From | To | Notes | +|---|---|---| +| direct (filestorage) | relstorage | Classic scale-out migration | +| direct (filestorage) | pgjsonb | Move to SQL-queryable storage | +| relstorage | pgjsonb | Switch to JSONB representation | +| zeo | relstorage | Remove ZEO server dependency | +| any | direct (filestorage) | Create a portable backup | + +All combinations work -- `zodb-convert` is storage-agnostic. + +## Configuration reference + +| Setting | Default | Description | +|---|---|---| +| `db_convert_import_filestorage_location` | *(unset)* | Path to the source FileStorage for import | +| `db_convert_import_blobs_location` | *(unset)* | Path to the source blob directory for import | +| `db_convert_export_filestorage_location` | *(unset)* | Path for the destination FileStorage on export | +| `db_convert_export_blobs_location` | *(unset)* | Path for the destination blob directory on export | + +The `convert-import.conf` file is generated when both `db_convert_import_*` +settings are provided. The `convert-export.conf` file is generated when both +`db_convert_export_*` settings are provided. Neither file is generated for +the `direct` (filestorage) backend since it *is* the portable format. + +## Next steps + +- [zodb-convert on PyPI](https://pypi.org/project/zodb-convert/) -- Full + documentation and source code +- {doc}`/explanation/storage-backends` -- Understanding the trade-offs + between backends diff --git a/docs/sources/reference/database-relstorage.md b/docs/sources/reference/database-relstorage.md index e37aa4c..16fea5f 100644 --- a/docs/sources/reference/database-relstorage.md +++ b/docs/sources/reference/database-relstorage.md @@ -83,6 +83,20 @@ replications. For details read RelStorage provides helper scripts for packing (`zodbpack`) and import/export from filestorage (`zodbconvert`). +:::{deprecated} 2.5 +The RelStorage-specific `zodbconvert` import/export configuration files +(`relstorage-import.conf`, `relstorage-export.conf`) and their corresponding +`db_relstorage_import_*` / `db_relstorage_export_*` settings are deprecated +in favor of the generic +[zodb-convert](https://pypi.org/project/zodb-convert/) tool, which works +with any ZODB storage backend. Use the new `db_convert_*` settings and the +generated `convert-import.conf` / `convert-export.conf` files instead. +See {doc}`/how-to/migrate-storage` for details. + +The existing settings and generated files continue to work but may be +removed in a future major version. +::: + The file `relstorage-pack.conf` for the command line utility `zodbpack` is always generated for all RelStorage configurations. For usage information read [Packing Or Reference Checking A ZODB Storage: zodbpack](https://relstorage.readthedocs.io/en/latest/zodbpack.html). diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index e373c4c..b4ffa3f 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -74,6 +74,29 @@ if db_relstorage_export_filestorage_location: filepath = Path(db_relstorage_export_filestorage_location) filepath.parent.mkdir(parents=True, exist_ok=True) + if "{{ cookiecutter.db_storage }}" != "direct": + # generic convert import directories + db_convert_import_blobs = ( + "{{ cookiecutter.db_convert_import_blobs_location | abspath }}" + ) + if db_convert_import_blobs: + Path(db_convert_import_blobs).mkdir(parents=True, exist_ok=True) + db_convert_import_fs = ( + "{{ cookiecutter.db_convert_import_filestorage_location | abspath }}" + ) + if db_convert_import_fs: + Path(db_convert_import_fs).parent.mkdir(parents=True, exist_ok=True) + # generic convert export directories + db_convert_export_blobs = ( + "{{ cookiecutter.db_convert_export_blobs_location | abspath }}" + ) + if db_convert_export_blobs: + Path(db_convert_export_blobs).mkdir(parents=True, exist_ok=True) + db_convert_export_fs = ( + "{{ cookiecutter.db_convert_export_filestorage_location | abspath }}" + ) + if db_convert_export_fs: + Path(db_convert_export_fs).parent.mkdir(parents=True, exist_ok=True) if "{{ cookiecutter.db_storage }}" == "pgjsonb": blob_temp_dir = "{{ cookiecutter.db_pgjsonb_blob_temp_dir }}" if blob_temp_dir: @@ -85,6 +108,10 @@ z3blobs_cache_dir = "{{ cookiecutter.db_z3blobs_cache_dir }}" if z3blobs_cache_dir: Path(z3blobs_cache_dir).mkdir(parents=True, exist_ok=True) + if "{{ cookiecutter.db_storage }}" == "direct": + # cleanup convert files — filestorage doesn't need conversion configs + (etc / "convert-import.conf").unlink() + (etc / "convert-export.conf").unlink() if "{{ cookiecutter.db_storage }}" != "relstorage": # cleanup relstorage files if no relstorage is configured (etc / "relstorage-export.conf").unlink() diff --git a/templates/db.conf b/templates/db.conf index 0d300a9..5be349e 100644 --- a/templates/db.conf +++ b/templates/db.conf @@ -192,9 +192,10 @@ drop_cache_rather_verify, client_label, storage, - wait + wait, + tagmarker="" )-%} - + {%- if not z3blobs_active %} # blobs shared-blob-dir {{ "true" if blob_mode == "shared" else "false" }} @@ -248,7 +249,7 @@ {%- if wait %} wait {{ wait }} {%- endif %} - + {%- endmacro %} # ============================================================================= @@ -271,10 +272,11 @@ s3_secret_key, s3_use_ssl, blob_cache_dir, - blob_cache_size + blob_cache_size, + tagmarker="" )-%} %import zodb_pgjsonb - + dsn {{ dsn }} {%- if name and name != "pgjsonb" %} name {{ name }} @@ -326,7 +328,7 @@ blob-cache-size {{ blob_cache_size }} {%- endif %} {%- endif %} - + {%- endmacro %} # ============================================================================= diff --git a/tests/test_bake_defaults.py b/tests/test_bake_defaults.py index cd6a3a0..d536e1f 100644 --- a/tests/test_bake_defaults.py +++ b/tests/test_bake_defaults.py @@ -557,6 +557,113 @@ def test_bake_with_zeo_wait(cookies): assert "wait false" in zope_conf +# ============================================================================= +# Generic zodb-convert config tests +# ============================================================================= + + +def test_bake_default_no_convert_files(cookies): + """Default bake (filestorage) should NOT have convert-*.conf files.""" + with bake_in_temp_dir(cookies) as result: + assert result.exit_code == 0 + assert not (result.project_path / "etc" / "convert-import.conf").exists() + assert not (result.project_path / "etc" / "convert-export.conf").exists() + + +def test_bake_pgjsonb_has_convert_files(cookies): + """PGJsonb with db_convert_* settings should generate convert configs.""" + with bake_in_temp_dir( + cookies, + extra_context={ + "db_storage": "pgjsonb", + "db_pgjsonb_dsn": "dbname=zodb host=localhost port=5433", + "db_convert_import_filestorage_location": "var/import/Data.fs", + "db_convert_import_blobs_location": "var/import/blobs", + "db_convert_export_filestorage_location": "var/export/Data.fs", + "db_convert_export_blobs_location": "var/export/blobs", + }, + ) as result: + assert result.exit_code == 0 + + import_conf = (result.project_path / "etc" / "convert-import.conf").read_text() + assert "" in import_conf + assert "" in import_conf + assert "" in import_conf + assert "dsn dbname=zodb host=localhost port=5433" in import_conf + + export_conf = (result.project_path / "etc" / "convert-export.conf").read_text() + assert "" in export_conf + assert "" in export_conf + assert "" in export_conf + + +def test_bake_relstorage_has_both_convert_and_legacy(cookies): + """RelStorage should have BOTH old relstorage-*.conf AND new convert-*.conf.""" + with bake_in_temp_dir( + cookies, + extra_context={ + "db_storage": "relstorage", + "db_relstorage_postgresql_dsn": "dbname=plone", + "db_relstorage_import_filestorage_location": "var/rs-import/Data.fs", + "db_relstorage_import_blobs_location": "var/rs-import/blobs", + "db_convert_import_filestorage_location": "var/import/Data.fs", + "db_convert_import_blobs_location": "var/import/blobs", + "db_convert_export_filestorage_location": "var/export/Data.fs", + "db_convert_export_blobs_location": "var/export/blobs", + }, + ) as result: + assert result.exit_code == 0 + # Legacy relstorage files should exist + assert (result.project_path / "etc" / "relstorage-import.conf").exists() + assert (result.project_path / "etc" / "relstorage-pack.conf").exists() + # New generic convert files should also exist + import_conf = (result.project_path / "etc" / "convert-import.conf").read_text() + assert "" in import_conf + export_conf = (result.project_path / "etc" / "convert-export.conf").read_text() + assert "" in export_conf + + +def test_bake_zeo_has_convert_files(cookies): + """ZEO with db_convert_* settings should generate convert configs.""" + with bake_in_temp_dir( + cookies, + extra_context={ + "db_storage": "zeo", + "db_convert_import_filestorage_location": "var/import/Data.fs", + "db_convert_import_blobs_location": "var/import/blobs", + "db_convert_export_filestorage_location": "var/export/Data.fs", + "db_convert_export_blobs_location": "var/export/blobs", + }, + ) as result: + assert result.exit_code == 0 + + import_conf = (result.project_path / "etc" / "convert-import.conf").read_text() + assert "" in import_conf + assert "" in import_conf + assert "" in import_conf + + export_conf = (result.project_path / "etc" / "convert-export.conf").read_text() + assert "" in export_conf + assert "" in export_conf + assert "" in export_conf + + +def test_bake_pgjsonb_no_convert_settings(cookies): + """PGJsonb without db_convert_* settings should have fallback comment.""" + with bake_in_temp_dir( + cookies, + extra_context={ + "db_storage": "pgjsonb", + "db_pgjsonb_dsn": "dbname=zodb", + }, + ) as result: + assert result.exit_code == 0 + # Files should exist but with the "missing settings" fallback + import_conf = (result.project_path / "etc" / "convert-import.conf").read_text() + assert "Missing db_convert_import_*" in import_conf + assert " + path {{ cookiecutter.db_convert_export_filestorage_location | abspath }} + blob-dir {{ cookiecutter.db_convert_export_blobs_location | abspath }} + + +{% else %} +# !!! Missing db_convert_export_* settings or storage is filestorage! +{%- endif %} + +# This file was generated by cookiecutter-zope-instance {{ cookiecutter._version }}. +# for details follow https://github.com/plone/cookiecutter-zope-instance diff --git a/{{ cookiecutter.target }}/etc/convert-import.conf b/{{ cookiecutter.target }}/etc/convert-import.conf new file mode 100644 index 0000000..c085898 --- /dev/null +++ b/{{ cookiecutter.target }}/etc/convert-import.conf @@ -0,0 +1,112 @@ +{%- set z3blobs_active = false -%} +{% import 'db.conf' as db with context +%}{%- set db_blob_mode = cookiecutter.db_blobs_mode or cookiecutter.db_blob_mode%} +{%- set db_blob_location = cookiecutter.db_blobs_location or cookiecutter.db_blob_location +%}## Configuration file to import a FileStorage database into the configured storage. +# Use with: zodb-convert convert-import.conf +# For more information see https://pypi.org/project/zodb-convert/ + +{%- if cookiecutter.db_storage != "direct" and cookiecutter.db_convert_import_filestorage_location and cookiecutter.db_convert_import_blobs_location %} + + + path {{ cookiecutter.db_convert_import_filestorage_location | abspath }} + blob-dir {{ cookiecutter.db_convert_import_blobs_location | abspath }} + + +{%- if cookiecutter.db_storage == "relstorage" %} + +{{ db.relstorage( + db_blob_mode, + db_blob_location, + cookiecutter.db_blob_cache_size, + cookiecutter.db_blob_cache_size_check, + cookiecutter.db_relstorage_keep_history, + cookiecutter.db_relstorage_read_only, + cookiecutter.db_relstorage_create_schema, + cookiecutter.db_relstorage_commit_lock_timeout, + cookiecutter.db_relstorage_blob_cache_size_check_external, + cookiecutter.db_relstorage_blob_chunk_size, + cookiecutter.db_relstorage_cache_local_mb, + cookiecutter.db_relstorage_cache_local_object_max, + cookiecutter.db_relstorage_cache_local_compression, + cookiecutter.db_relstorage_cache_local_dir, + cookiecutter.db_relstorage_cache_prefix, + cookiecutter.db_relstorage_replica_conf, + cookiecutter.db_relstorage_ro_replica_conf, + cookiecutter.db_relstorage_replica_timeout, + cookiecutter.db_relstorage_replica_revert_when_stale, + cookiecutter.db_relstorage, + cookiecutter.db_relstorage_postgresql_driver, + cookiecutter.db_relstorage_postgresql_dsn, + cookiecutter.db_relstorage_mysql_driver, + cookiecutter.db_relstorage_mysql_parameters, + cookiecutter.db_relstorage_oracle_driver, + cookiecutter.db_relstorage_commit_lock_id, + cookiecutter.db_relstorage_oracle_user, + cookiecutter.db_relstorage_oracle_password, + cookiecutter.db_relstorage_oracle_dsn, + cookiecutter.db_relstorage_sqlite3_driver, + cookiecutter.db_relstorage_sqlite3_data_dir, + cookiecutter.db_relstorage_sqlite3_gevent_yield_interval, + cookiecutter.db_relstorage_sqlite3_pragma, + " destination", + cookiecutter.db_relstorage_name, + cookiecutter.db_relstorage_pack_gc, +) }} + +{%- elif cookiecutter.db_storage == "pgjsonb" %} + +{{ db.pgjsonb( + cookiecutter.db_pgjsonb_dsn, + cookiecutter.db_pgjsonb_name, + cookiecutter.db_pgjsonb_history_preserving, + cookiecutter.db_pgjsonb_blob_temp_dir, + cookiecutter.db_pgjsonb_cache_local_mb, + cookiecutter.db_pgjsonb_pool_size, + cookiecutter.db_pgjsonb_pool_max_size, + cookiecutter.db_pgjsonb_pool_timeout, + cookiecutter.db_pgjsonb_blob_threshold, + cookiecutter.db_pgjsonb_s3_bucket_name, + cookiecutter.db_pgjsonb_s3_prefix, + cookiecutter.db_pgjsonb_s3_endpoint_url, + cookiecutter.db_pgjsonb_s3_region, + cookiecutter.db_pgjsonb_s3_access_key, + cookiecutter.db_pgjsonb_s3_secret_key, + cookiecutter.db_pgjsonb_s3_use_ssl, + cookiecutter.db_pgjsonb_blob_cache_dir, + cookiecutter.db_pgjsonb_blob_cache_size, + tagmarker=" destination", +) }} + +{%- elif cookiecutter.db_storage == "zeo" %} + +{{ db.zeo( + db_blob_mode, + db_blob_location, + cookiecutter.db_blob_cache_size, + cookiecutter.db_blob_cache_size_check, + cookiecutter.db_zeo_server, + cookiecutter.db_zeo_name, + cookiecutter.db_zeo_client, + cookiecutter.db_zeo_var, + cookiecutter.db_zeo_cache_size, + cookiecutter.db_zeo_username, + cookiecutter.db_zeo_password, + cookiecutter.db_zeo_realm, + cookiecutter.db_zeo_read_only_fallback, + cookiecutter.db_zeo_read_only, + cookiecutter.db_zeo_drop_cache_rather_verify, + cookiecutter.db_zeo_client_label, + cookiecutter.db_zeo_storage, + cookiecutter.db_zeo_wait, + tagmarker=" destination", +) }} + +{%- endif %} + +{% else %} +# !!! Missing db_convert_import_* settings or storage is filestorage! +{%- endif %} + +# This file was generated by cookiecutter-zope-instance {{ cookiecutter._version }}. +# for details follow https://github.com/plone/cookiecutter-zope-instance