Skip to content

Commit 1281018

Browse files
zvonandzvonand
authored
Merge branch 'releases/25.3.14' into backports/releases/25.3.14/89367
2 parents 345136d + 3fb1674 commit 1281018

19 files changed

Lines changed: 80 additions & 104 deletions

File tree

‎docker/keeper/Dockerfile‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,8 @@ RUN arch=${TARGETARCH:-amd64} \
3535
arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
3636
esac
3737

38-
# lts / testing / prestable / etc
39-
ARG REPO_CHANNEL="stable"
40-
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
41-
ARG VERSION="25.2.2.39"
42-
ARG PACKAGES="clickhouse-keeper"
38+
# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile
39+
4340
ARG DIRECT_DOWNLOAD_URLS=""
4441

4542
# user/group precreated explicitly with fixed uid/gid on purpose.
@@ -63,12 +60,7 @@ RUN arch=${TARGETARCH:-amd64} \
6360
&& wget -c -q "$url" \
6461
; done \
6562
else \
66-
for package in ${PACKAGES}; do \
67-
cd /tmp \
68-
&& echo "Get ${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
69-
&& wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
70-
&& wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz.sha512" \
71-
; done \
63+
exit 1; \
7264
fi \
7365
&& cat *.tgz.sha512 | sha512sum -c \
7466
&& for file in *.tgz; do \

‎docker/server/Dockerfile.alpine‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,10 @@ RUN arch=${TARGETARCH:-amd64} \
3232
arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
3333
esac
3434

35-
# lts / testing / prestable / etc
36-
ARG REPO_CHANNEL="stable"
37-
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
38-
ARG VERSION="25.2.2.39"
39-
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
35+
36+
37+
# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile
38+
4039
ARG DIRECT_DOWNLOAD_URLS=""
4140

4241
# user/group precreated explicitly with fixed uid/gid on purpose.
@@ -59,11 +58,7 @@ RUN arch=${TARGETARCH:-amd64} \
5958
&& wget -c -q "$url" \
6059
; done \
6160
else \
62-
for package in ${PACKAGES}; do \
63-
echo "Get ${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
64-
&& wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
65-
&& wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz.sha512" \
66-
; done \
61+
exit 1; \
6762
fi \
6863
&& cat *.tgz.sha512 | sed 's:/output/:/tmp/:' | sha512sum -c \
6964
&& for file in *.tgz; do \

‎docker/server/Dockerfile.ubuntu‎

Lines changed: 3 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -26,23 +26,12 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
2626
wget \
2727
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
2828

29-
ARG REPO_CHANNEL="stable"
30-
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
31-
ARG VERSION="25.2.2.39"
32-
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
33-
3429
#docker-official-library:off
3530
# The part between `docker-official-library` tags is related to our builds
3631

37-
# set non-empty deb_location_url url to create a docker image
38-
# from debs created by CI build, for example:
39-
# docker build . --network host --build-arg version="21.4.1.6282" --build-arg deb_location_url="https://..." -t ...
40-
ARG deb_location_url=""
41-
ARG DIRECT_DOWNLOAD_URLS=""
32+
# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile
4233

43-
# set non-empty single_binary_location_url to create docker image
44-
# from a single binary url (useful for non-standard builds - with sanitizers, for arm64).
45-
ARG single_binary_location_url=""
34+
ARG DIRECT_DOWNLOAD_URLS=""
4635

4736
ARG TARGETARCH
4837

@@ -58,64 +47,7 @@ RUN if [ -n "${DIRECT_DOWNLOAD_URLS}" ]; then \
5847
&& rm -rf /tmp/* ; \
5948
fi
6049

61-
# install from a web location with deb packages
62-
RUN arch="${TARGETARCH:-amd64}" \
63-
&& if [ -n "${deb_location_url}" ]; then \
64-
echo "installing from custom url with deb packages: ${deb_location_url}" \
65-
&& rm -rf /tmp/clickhouse_debs \
66-
&& mkdir -p /tmp/clickhouse_debs \
67-
&& for package in ${PACKAGES}; do \
68-
{ wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_${arch}.deb" -P /tmp/clickhouse_debs || \
69-
wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_all.deb" -P /tmp/clickhouse_debs ; } \
70-
|| exit 1 \
71-
; done \
72-
&& dpkg -i /tmp/clickhouse_debs/*.deb \
73-
&& rm -rf /tmp/* ; \
74-
fi
75-
76-
# install from a single binary
77-
RUN if [ -n "${single_binary_location_url}" ]; then \
78-
echo "installing from single binary url: ${single_binary_location_url}" \
79-
&& rm -rf /tmp/clickhouse_binary \
80-
&& mkdir -p /tmp/clickhouse_binary \
81-
&& wget --progress=bar:force:noscroll "${single_binary_location_url}" -O /tmp/clickhouse_binary/clickhouse \
82-
&& chmod +x /tmp/clickhouse_binary/clickhouse \
83-
&& /tmp/clickhouse_binary/clickhouse install --user "clickhouse" --group "clickhouse" \
84-
&& rm -rf /tmp/* ; \
85-
fi
86-
87-
# The rest is the same in the official docker and in our build system
88-
#docker-official-library:on
89-
90-
# A fallback to installation from ClickHouse repository
91-
# It works unless the clickhouse binary already exists
92-
RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \
93-
; apt-get update \
94-
&& apt-get install --yes --no-install-recommends \
95-
dirmngr \
96-
gnupg2 \
97-
&& mkdir -p /etc/apt/sources.list.d \
98-
&& GNUPGHOME=$(mktemp -d) \
99-
&& GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
100-
--keyring /usr/share/keyrings/clickhouse-keyring.gpg \
101-
--keyserver hkp://keyserver.ubuntu.com:80 \
102-
--recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
103-
&& rm -rf "$GNUPGHOME" \
104-
&& chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
105-
&& echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
106-
&& echo "installing from repository: ${REPOSITORY}" \
107-
&& apt-get update \
108-
&& for package in ${PACKAGES}; do \
109-
packages="${packages} ${package}=${VERSION}" \
110-
; done \
111-
&& apt-get install --yes --no-install-recommends ${packages} || exit 1 \
112-
&& rm -rf \
113-
/var/lib/apt/lists/* \
114-
/var/cache/debconf \
115-
/tmp/* \
116-
&& apt-get autoremove --purge -yq dirmngr gnupg2 \
117-
&& chmod ugo+Xrw -R /etc/clickhouse-server /etc/clickhouse-client
118-
# The last chmod is here to make the next one is No-op in docker official library Dockerfile
50+
# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile
11951

12052
# post install
12153
# we need to allow "others" access to clickhouse folder, because docker container

‎programs/server/config.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1687,7 +1687,7 @@
16871687
<anonymize>false</anonymize>
16881688
<!-- Default endpoint should be changed to different Sentry DSN only if you have -->
16891689
<!-- some in-house engineers or hired consultants who're going to debug ClickHouse issues for you -->
1690-
<endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
1690+
<endpoint></endpoint>
16911691
<!-- Send LOGICAL_ERRORs as well (default: false) -->
16921692
<send_logical_errors>false</send_logical_errors>
16931693
</send_crash_reports>

‎programs/server/config.yaml.example‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -939,6 +939,6 @@ send_crash_reports:
939939
anonymize: false
940940
# Default endpoint should be changed to different Sentry DSN only if you have
941941
# some in-house engineers or hired consultants who're going to debug ClickHouse issues for you
942-
endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277'
942+
endpoint: ''
943943
# Uncomment to disable ClickHouse internal DNS caching.
944944
# disable_internal_dns_cache: 1

‎src/Core/Settings.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5975,6 +5975,9 @@ This only affects operations performed on the client side, in particular parsing
59755975
Normally this setting should be set in user profile (users.xml or queries like `ALTER USER`), not through the client (client command line arguments, `SET` query, or `SETTINGS` section of `SELECT` query). Through the client it can be changed to false, but can't be changed to true (because the server won't send the settings if user profile has `apply_settings_from_server = false`).
59765976
59775977
Note that initially (24.12) there was a server setting (`send_settings_to_client`), but latter it got replaced with this client setting, for better usability.
5978+
)", 0) \
5979+
DECLARE(Bool, allow_local_data_lakes, false, R"(
5980+
Allow using local data lake engines and table functions (IcebergLocal, DeltaLakeLocal, etc.).
59785981
)", 0) \
59795982
\
59805983
/* ####################################################### */ \

‎src/Core/SettingsChangesHistory.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,10 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
6969
addSettingsChanges(settings_changes_history, "25.4",
7070
{
7171
});
72+
addSettingsChanges(settings_changes_history, "25.3.8",
73+
{
74+
{"allow_local_data_lakes", false, false, "New setting to guard local data lake engines and table functions"},
75+
});
7276
addSettingsChanges(settings_changes_history, "25.3",
7377
{
7478
/// Release closed. Please use 25.4

‎src/Storages/ObjectStorage/registerStorageObjectStorage.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ namespace DB
1818
namespace ErrorCodes
1919
{
2020
extern const int BAD_ARGUMENTS;
21+
extern const int SUPPORT_IS_DISABLED;
22+
}
23+
24+
namespace Setting
25+
{
26+
extern const SettingsBool allow_local_data_lakes;
2127
}
2228

2329
namespace
@@ -235,6 +241,11 @@ void registerStorageIceberg(StorageFactory & factory)
235241
"IcebergLocal",
236242
[&](const StorageFactory::Arguments & args)
237243
{
244+
if (!args.getLocalContext()->getSettingsRef()[Setting::allow_local_data_lakes])
245+
throw Exception(
246+
ErrorCodes::SUPPORT_IS_DISABLED,
247+
"IcebergLocal is disabled. Set `allow_local_data_lakes` to enable it");
248+
238249
auto configuration = std::make_shared<StorageLocalIcebergConfiguration>();
239250
return createStorageObjectStorage(args, configuration);
240251
},

‎src/TableFunctions/TableFunctionObjectStorage.cpp‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ namespace DB
3131

3232
namespace Setting
3333
{
34+
extern const SettingsBool allow_local_data_lakes;
3435
extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas;
3536
extern const SettingsBool parallel_replicas_for_cluster_engines;
3637
extern const SettingsString cluster_for_parallel_replicas;
@@ -40,6 +41,7 @@ namespace Setting
4041
namespace ErrorCodes
4142
{
4243
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
44+
extern const int SUPPORT_IS_DISABLED;
4345
}
4446

4547
template <typename Definition, typename Configuration>
@@ -79,6 +81,15 @@ std::vector<size_t> TableFunctionObjectStorage<Definition, Configuration>::skipA
7981
template <typename Definition, typename Configuration>
8082
void TableFunctionObjectStorage<Definition, Configuration>::parseArguments(const ASTPtr & ast_function, ContextPtr context)
8183
{
84+
if constexpr (std::is_same_v<Definition, IcebergLocalDefinition>)
85+
{
86+
if (!context->getSettingsRef()[Setting::allow_local_data_lakes])
87+
throw Exception(
88+
ErrorCodes::SUPPORT_IS_DISABLED,
89+
"Table function '{}' is disabled. Set `allow_local_data_lakes` to enable it",
90+
getName());
91+
}
92+
8293
/// Clone ast function, because we can modify its arguments like removing headers.
8394
auto ast_copy = ast_function->clone();
8495
ASTs & args_func = ast_copy->children;
@@ -109,6 +120,15 @@ template <typename Definition, typename Configuration>
109120
ColumnsDescription TableFunctionObjectStorage<
110121
Definition, Configuration>::getActualTableStructure(ContextPtr context, bool is_insert_query) const
111122
{
123+
if constexpr (std::is_same_v<Definition, IcebergLocalDefinition>)
124+
{
125+
if (!context->getSettingsRef()[Setting::allow_local_data_lakes])
126+
throw Exception(
127+
ErrorCodes::SUPPORT_IS_DISABLED,
128+
"Table function '{}' is disabled. Set `allow_local_data_lakes` to enable it",
129+
getName());
130+
}
131+
112132
if (configuration->structure == "auto")
113133
{
114134
context->checkAccess(getSourceAccessType());
@@ -129,6 +149,15 @@ StoragePtr TableFunctionObjectStorage<Definition, Configuration>::executeImpl(
129149
ColumnsDescription cached_columns,
130150
bool is_insert_query) const
131151
{
152+
if constexpr (std::is_same_v<Definition, IcebergLocalDefinition>)
153+
{
154+
if (!context->getSettingsRef()[Setting::allow_local_data_lakes])
155+
throw Exception(
156+
ErrorCodes::SUPPORT_IS_DISABLED,
157+
"Table function '{}' is disabled. Set `allow_local_data_lakes` to enable it",
158+
getName());
159+
}
160+
132161
chassert(configuration);
133162
ColumnsDescription columns;
134163

@@ -287,6 +316,10 @@ template class TableFunctionObjectStorage<HDFSClusterDefinition, StorageHDFSConf
287316
#endif
288317
template class TableFunctionObjectStorage<LocalDefinition, StorageLocalConfiguration>;
289318

319+
#if USE_AVRO
320+
template class TableFunctionObjectStorage<IcebergLocalDefinition, StorageLocalIcebergConfiguration>;
321+
#endif
322+
290323
#if USE_AVRO && USE_AWS_S3
291324
template class TableFunctionObjectStorage<IcebergS3ClusterDefinition, StorageS3IcebergConfiguration>;
292325
#endif

‎tests/ci/docker_server.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,8 @@ def buildx_args(
184184
args = [
185185
f"--platform=linux/{arch}",
186186
f"--label=build-url={GITHUB_RUN_URL}",
187-
f"--label=com.clickhouse.build.githash={git.sha}",
188-
f"--label=com.clickhouse.build.version={version}",
187+
f"--label=com.altinity.build.githash={git.sha}",
188+
f"--label=com.altinity.build.version={version}",
189189
]
190190
if direct_urls:
191191
args.append(f"--build-arg=DIRECT_DOWNLOAD_URLS='{' '.join(direct_urls)}'")

0 commit comments

Comments
 (0)