From 4bd04707ee454671ad15cbcedf99890acc91b127 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 20 Jul 2026 20:59:36 +0800 Subject: [PATCH 1/3] build: package complete Rust release crates --- crates/integrations/datafusion/Cargo.toml | 1 - crates/integrations/datafusion/LICENSE | 201 ++++++++++++++++++ crates/integrations/datafusion/NOTICE | 5 + .../datafusion/src/physical_plan/scan.rs | 2 +- .../datafusion}/test_utils.rs | 0 crates/paimon/Cargo.toml | 1 - crates/paimon/LICENSE | 201 ++++++++++++++++++ crates/paimon/NOTICE | 5 + crates/{ => paimon}/blob_test_utils.rs | 0 crates/paimon/src/arrow/format/blob.rs | 5 +- .../paimon/src/table/data_evolution_reader.rs | 7 +- .../data_evolution_reader/blob_fallback.rs | 5 +- crates/paimon/src/table/read_builder.rs | 2 +- crates/paimon/test_utils.rs | 101 +++++++++ 14 files changed, 519 insertions(+), 17 deletions(-) create mode 100644 crates/integrations/datafusion/LICENSE create mode 100644 crates/integrations/datafusion/NOTICE rename crates/{ => integrations/datafusion}/test_utils.rs (100%) create mode 100644 crates/paimon/LICENSE create mode 100644 crates/paimon/NOTICE rename crates/{ => paimon}/blob_test_utils.rs (100%) create mode 100644 crates/paimon/test_utils.rs diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/integrations/datafusion/Cargo.toml index 9be9a350..37fb1947 100644 --- a/crates/integrations/datafusion/Cargo.toml +++ b/crates/integrations/datafusion/Cargo.toml @@ -20,7 +20,6 @@ name = "paimon-datafusion" edition.workspace = true version.workspace = true publish = ["crates-io"] -exclude = ["testdata/"] license.workspace = true rust-version.workspace = true homepage = "https://paimon.apache.org/docs/rust/datafusion/" diff --git a/crates/integrations/datafusion/LICENSE b/crates/integrations/datafusion/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/crates/integrations/datafusion/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/integrations/datafusion/NOTICE b/crates/integrations/datafusion/NOTICE new file mode 100644 index 00000000..79131e55 --- /dev/null +++ b/crates/integrations/datafusion/NOTICE @@ -0,0 +1,5 @@ +Apache Paimon Rust +Copyright 2024-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index eb510df5..bf8373dd 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -1156,7 +1156,7 @@ impl DisplayAs for PaimonTableScan { mod tests { use super::*; mod test_utils { - include!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../test_utils.rs")); + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/test_utils.rs")); } use datafusion::arrow::array::Int32Array; diff --git a/crates/test_utils.rs b/crates/integrations/datafusion/test_utils.rs similarity index 100% rename from crates/test_utils.rs rename to crates/integrations/datafusion/test_utils.rs diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index 5349896c..f8bb188b 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -21,7 +21,6 @@ description = "The rust implementation of Apache Paimon" documentation = "https://docs.rs/paimon" name = "paimon" publish = ["crates-io"] -exclude = ["testdata/"] homepage.workspace = true repository.workspace = true diff --git a/crates/paimon/LICENSE b/crates/paimon/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/crates/paimon/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/paimon/NOTICE b/crates/paimon/NOTICE new file mode 100644 index 00000000..79131e55 --- /dev/null +++ b/crates/paimon/NOTICE @@ -0,0 +1,5 @@ +Apache Paimon Rust +Copyright 2024-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/crates/blob_test_utils.rs b/crates/paimon/blob_test_utils.rs similarity index 100% rename from crates/blob_test_utils.rs rename to crates/paimon/blob_test_utils.rs diff --git a/crates/paimon/src/arrow/format/blob.rs b/crates/paimon/src/arrow/format/blob.rs index 51ea8a97..080c5056 100644 --- a/crates/paimon/src/arrow/format/blob.rs +++ b/crates/paimon/src/arrow/format/blob.rs @@ -1374,10 +1374,7 @@ mod tests { #[allow(dead_code)] mod blob_test_utils { - include!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../blob_test_utils.rs" - )); + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/blob_test_utils.rs")); } #[tokio::test] diff --git a/crates/paimon/src/table/data_evolution_reader.rs b/crates/paimon/src/table/data_evolution_reader.rs index bc307879..e9cee401 100644 --- a/crates/paimon/src/table/data_evolution_reader.rs +++ b/crates/paimon/src/table/data_evolution_reader.rs @@ -2268,15 +2268,12 @@ mod tests { use tempfile::tempdir; mod blob_test_utils { - include!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../blob_test_utils.rs" - )); + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/blob_test_utils.rs")); } #[allow(dead_code)] mod test_utils { - include!(concat!(env!("CARGO_MANIFEST_DIR"), "/../test_utils.rs")); + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/test_utils.rs")); } use blob_test_utils::{write_blob_file, write_blob_file_with_values, BlobFixtureValue}; diff --git a/crates/paimon/src/table/data_evolution_reader/blob_fallback.rs b/crates/paimon/src/table/data_evolution_reader/blob_fallback.rs index 4f778b24..4331b4d6 100644 --- a/crates/paimon/src/table/data_evolution_reader/blob_fallback.rs +++ b/crates/paimon/src/table/data_evolution_reader/blob_fallback.rs @@ -314,10 +314,7 @@ mod tests { #[allow(dead_code)] mod blob_test_utils { - include!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../blob_test_utils.rs" - )); + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/blob_test_utils.rs")); } use blob_test_utils::{build_blob_file_bytes_with_values, BlobFixtureValue}; diff --git a/crates/paimon/src/table/read_builder.rs b/crates/paimon/src/table/read_builder.rs index c75ee528..4870b8f8 100644 --- a/crates/paimon/src/table/read_builder.rs +++ b/crates/paimon/src/table/read_builder.rs @@ -635,7 +635,7 @@ mod tests { use super::{PaimonReadBuilder, ReadBuilder, ReadBuilderKind}; use crate::table::TableRead; mod test_utils { - include!(concat!(env!("CARGO_MANIFEST_DIR"), "/../test_utils.rs")); + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/test_utils.rs")); } use crate::catalog::Identifier; diff --git a/crates/paimon/test_utils.rs b/crates/paimon/test_utils.rs new file mode 100644 index 00000000..4813cc53 --- /dev/null +++ b/crates/paimon/test_utils.rs @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fs::File; +use std::path::Path; +use std::sync::Arc; + +use arrow_array::{Array, Int32Array, RecordBatch}; +use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; +use chrono::Utc; +use parquet::arrow::ArrowWriter; +use parquet::file::properties::WriterProperties; +use serde::de::DeserializeOwned; + +pub(crate) fn write_int_parquet_file( + path: &Path, + columns: Vec<(&str, Vec)>, + max_row_group_size: Option, +) { + let schema = Arc::new(ArrowSchema::new( + columns + .iter() + .map(|(name, _)| ArrowField::new(*name, ArrowDataType::Int32, false)) + .collect::>(), + )); + let arrays: Vec> = columns + .iter() + .map(|(_, values)| Arc::new(Int32Array::from(values.clone())) as Arc) + .collect(); + let batch = RecordBatch::try_new(schema.clone(), arrays).unwrap(); + + let props = max_row_group_size.map(|size| { + WriterProperties::builder() + .set_max_row_group_row_count(Some(size)) + .build() + }); + let file = File::create(path).unwrap(); + let mut writer = ArrowWriter::try_new(file, schema, props).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); +} + +pub(crate) fn local_file_path(path: &Path) -> String { + let normalized = path.to_string_lossy().replace('\\', "/"); + if normalized.starts_with('/') { + format!("file:{normalized}") + } else { + format!("file:/{normalized}") + } +} + +pub(crate) fn test_data_file(file_name: &str, row_count: i64, file_size: i64) -> T +where + T: DeserializeOwned, +{ + serde_json::from_value(serde_json::json!({ + "_FILE_NAME": file_name, + "_FILE_SIZE": file_size, + "_ROW_COUNT": row_count, + "_MIN_KEY": [], + "_MAX_KEY": [], + "_KEY_STATS": { + "_MIN_VALUES": [], + "_MAX_VALUES": [], + "_NULL_COUNTS": [] + }, + "_VALUE_STATS": { + "_MIN_VALUES": [], + "_MAX_VALUES": [], + "_NULL_COUNTS": [] + }, + "_MIN_SEQUENCE_NUMBER": 0, + "_MAX_SEQUENCE_NUMBER": 0, + "_SCHEMA_ID": 0, + "_LEVEL": 1, + "_EXTRA_FILES": [], + "_CREATION_TIME": Utc::now().timestamp_millis(), + "_DELETE_ROW_COUNT": null, + "_EMBEDDED_FILE_INDEX": null, + "_FILE_SOURCE": null, + "_VALUE_STATS_COLS": null, + "_FIRST_ROW_ID": null, + "_WRITE_COLS": null, + "_EXTERNAL_PATH": null + })) + .unwrap() +} From eec0f4766d4d6cf457e4fbafb2e12402b9e7885c Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 20 Jul 2026 20:59:50 +0800 Subject: [PATCH 2/3] ci: publish Rust crates in dependency order --- .github/workflows/release-rust.yml | 215 +++++++++++++++++++++++++---- 1 file changed, 186 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release-rust.yml b/.github/workflows/release-rust.yml index a5cef30e..1d72ebde 100644 --- a/.github/workflows/release-rust.yml +++ b/.github/workflows/release-rust.yml @@ -15,12 +15,8 @@ # specific language governing permissions and limitations # under the License. -# Publish paimon and paimon-datafusion crates to crates.io. -# -# Trigger: push a version tag (e.g. v0.1.0, v0.1.0-rc1). -# Pre-release tags (containing '-') only run dry-run checks without publishing. -# -# Token auth: add secret CARGO_REGISTRY_TOKEN for crates.io publishing. +# GA tags publish; RC tags validate only. +# Secret: CARGO_REGISTRY_TOKEN. name: Release Rust @@ -31,19 +27,15 @@ on: - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + jobs: - publish: + validate: runs-on: ubuntu-latest permissions: contents: read - strategy: - # Publish crates sequentially to respect dependency order - max-parallel: 1 - matrix: - # Order matters: paimon must be published before paimon-datafusion - package: - - paimon - - paimon-datafusion steps: - uses: actions/checkout@v7 @@ -52,24 +44,189 @@ jobs: rustup update stable rustup default stable - - name: Dry run - # Only dry-run paimon since paimon-datafusion depends on it - if: matrix.package == 'paimon' - run: cargo publish -p ${{ matrix.package }} --all-features --dry-run + - name: Validate release tag + if: github.ref_type == 'tag' + shell: bash + run: | + set -euo pipefail + if [[ ! "${GITHUB_REF_NAME}" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)(-rc[0-9]+)?$ ]]; then + echo "::error::invalid release tag: ${GITHUB_REF_NAME}" + exit 1 + fi + TAG_VERSION="${BASH_REMATCH[1]}" + WORKSPACE_VERSION=$( + cargo metadata --locked --format-version 1 --no-deps | + jq -er ' + [.packages[] | + select(.name == "paimon" or .name == "paimon-datafusion") | + .version] as $versions | + if ($versions | length) == 2 and + ($versions | unique | length) == 1 + then $versions[0] + else error("release crate versions differ") + end + ' + ) + if [ "${TAG_VERSION}" != "${WORKSPACE_VERSION}" ]; then + echo "::error::tag ${TAG_VERSION} does not match workspace ${WORKSPACE_VERSION}" + exit 1 + fi + + - name: Check package inventory + shell: bash + run: | + set -euo pipefail + for crate_dir in crates/paimon crates/integrations/datafusion; do + for legal_file in LICENSE NOTICE; do + cmp --silent "${legal_file}" "${crate_dir}/${legal_file}" || { + echo "::error::${crate_dir}/${legal_file} differs from ${legal_file}" + exit 1 + } + done + done + + for package in paimon paimon-datafusion; do + files="/tmp/${package}-package-files.txt" + cargo package --locked -p "${package}" --list > "${files}" + for required in LICENSE NOTICE DEPENDENCIES.rust.tsv test_utils.rs; do + grep --fixed-strings --line-regexp --quiet "${required}" "${files}" || { + echo "::error::${package} package is missing ${required}" + exit 1 + } + done + grep --quiet '^testdata/' "${files}" || { + echo "::error::${package} package is missing testdata" + exit 1 + } + done + + grep --fixed-strings --line-regexp --quiet \ + blob_test_utils.rs /tmp/paimon-package-files.txt || { + echo "::error::paimon package is missing blob_test_utils.rs" + exit 1 + } + + - name: Package release crates + run: > + cargo package --locked + -p paimon + -p paimon-datafusion + --all-features + + - name: Dry run paimon + run: cargo publish --locked -p paimon --all-features --dry-run + + publish-paimon: + needs: validate + if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + - name: Setup Rust toolchain + run: | + rustup update stable + rustup default stable - - name: Publish ${{ matrix.package }} to crates.io - if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') - run: cargo publish -p ${{ matrix.package }} --all-features + - name: Publish paimon if absent + shell: bash env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: | + set -euo pipefail + VERSION=$( + cargo metadata --locked --format-version 1 --no-deps | + jq -er '.packages[] | select(.name == "paimon") | .version' + ) + USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})" + STATUS=$( + curl --silent --show-error --retry 3 \ + --user-agent "${USER_AGENT}" \ + --output /tmp/paimon-version.json \ + --write-out '%{http_code}' \ + "https://crates.io/api/v1/crates/paimon/${VERSION}" + ) + if [ "${STATUS}" = 200 ]; then + jq -e --arg version "${VERSION}" '.version.num == $version' \ + /tmp/paimon-version.json >/dev/null + echo "paimon ${VERSION} already exists; skipping" + elif [ "${STATUS}" = 404 ]; then + cargo publish --locked -p paimon --all-features + else + echo "::error::crates.io returned HTTP ${STATUS} for paimon ${VERSION}" + exit 1 + fi - - name: Wait for crates.io to update - if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') && matrix.package == 'paimon' + - name: Wait for paimon on crates.io + shell: bash run: | - VERSION=${GITHUB_REF_NAME#v} - for i in {1..10}; do - CURRENT=$(curl -s https://crates.io/api/v1/crates/paimon | jq -r '.crate.max_version') - [ "$CURRENT" = "$VERSION" ] && echo "✅ paimon $VERSION available" && exit 0 - echo "Waiting for $VERSION (currently $CURRENT)... $i/10" + set -euo pipefail + VERSION=$( + cargo metadata --locked --format-version 1 --no-deps | + jq -er '.packages[] | select(.name == "paimon") | .version' + ) + USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})" + for attempt in {1..20}; do + CURRENT=$( + curl --fail-with-body --silent --show-error --retry 3 \ + --user-agent "${USER_AGENT}" \ + "https://crates.io/api/v1/crates/paimon/${VERSION}" | + jq -r '.version.num // empty' || true + ) + if [ "${CURRENT}" = "${VERSION}" ]; then + echo "paimon ${VERSION} is available" + exit 0 + fi + echo "Waiting for paimon ${VERSION}... ${attempt}/20" sleep 30 done + echo "::error::paimon ${VERSION} did not appear before timeout" + exit 1 + + publish-paimon-datafusion: + needs: publish-paimon + if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + - name: Setup Rust toolchain + run: | + rustup update stable + rustup default stable + + - name: Dry run paimon-datafusion + run: cargo publish --locked -p paimon-datafusion --all-features --dry-run + + - name: Publish paimon-datafusion if absent + shell: bash + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: | + set -euo pipefail + VERSION=$( + cargo metadata --locked --format-version 1 --no-deps | + jq -er '.packages[] | select(.name == "paimon-datafusion") | .version' + ) + USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})" + STATUS=$( + curl --silent --show-error --retry 3 \ + --user-agent "${USER_AGENT}" \ + --output /tmp/paimon-datafusion-version.json \ + --write-out '%{http_code}' \ + "https://crates.io/api/v1/crates/paimon-datafusion/${VERSION}" + ) + if [ "${STATUS}" = 200 ]; then + jq -e --arg version "${VERSION}" '.version.num == $version' \ + /tmp/paimon-datafusion-version.json >/dev/null + echo "paimon-datafusion ${VERSION} already exists; skipping" + elif [ "${STATUS}" = 404 ]; then + cargo publish --locked -p paimon-datafusion --all-features + else + echo "::error::crates.io returned HTTP ${STATUS} for paimon-datafusion ${VERSION}" + exit 1 + fi From eb96683326b1a1dbcea61f40f30f687ac97f675d Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 21 Jul 2026 11:31:27 +0800 Subject: [PATCH 3/3] ci: clarify joint crate packaging --- .github/workflows/release-rust.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-rust.yml b/.github/workflows/release-rust.yml index 1d72ebde..a71f0f3a 100644 --- a/.github/workflows/release-rust.yml +++ b/.github/workflows/release-rust.yml @@ -107,6 +107,7 @@ jobs: } - name: Package release crates + # Cargo verifies workspace dependencies through its temporary registry. run: > cargo package --locked -p paimon