Skip to content

Commit 762f056

Browse files
JosepSampeJosepSampe
committed
Update
1 parent 0081fc8 commit 762f056

3 files changed

Lines changed: 25 additions & 58 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
## [v3.6.5.dev0]
44

55
### Added
6+
- [GCP Compute Engine] Added new GCP Compute Engine standalone backend
67
- [Core] Added support for variable-length parameters in functions passed to the executor.
78

89
### Changed
@@ -15,6 +16,7 @@
1516
### Fixed
1617
- [K8s] Fixed default runtime builds impacted by Debian Buster end-of-life.
1718
- [GCP Cloud Run] Added Artifact Registry (`pkg.dev`) runtime deployment support
19+
- [K8s] Run default runtime image as non-root user (uid 1000) (#1469)
1820

1921

2022
## [v3.6.4]

‎lithops/standalone/backends/gcp_compute_engine/gcp_compute_engine.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -966,9 +966,13 @@ def __str__(self):
966966

967967
def get_ssh_client(self):
968968
self.get_instance_data()
969+
if not self.instance_data:
970+
raise Exception(f'VM instance {self.name} does not exist')
971+
969972
if self.public:
970973
if not self.public_ip:
971-
if self.get_status() == 'TERMINATED':
974+
status = self.get_status()
975+
if status == 'TERMINATED':
972976
self.start()
973977
else:
974978
self._wait_public_ip(timeout=60)
@@ -1080,6 +1084,8 @@ def _wait_public_ip(self, timeout=INSTANCE_START_TIMEOUT):
10801084
start = time.time()
10811085
while time.time() - start < timeout:
10821086
self.get_instance_data()
1087+
if not self.instance_data:
1088+
raise Exception(f'VM instance {self.name} does not exist')
10831089
if self.public_ip:
10841090
return self.public_ip
10851091
time.sleep(2)
@@ -1121,7 +1127,9 @@ def create(self, public=False, ssh_public_key=None, user_data=None,
11211127
}
11221128
# Master: external IP for SSH from the Lithops client.
11231129
# Workers: no external IP; outbound internet uses Cloud NAT on the subnet.
1124-
if public or self.config.get('worker_public_ip', False):
1130+
# Use self.public (set in __init__) when create() is called without public=...
1131+
use_public_ip = public or self.public or self.config.get('worker_public_ip', False)
1132+
if use_public_ip:
11251133
network_iface['accessConfigs'] = [{'name': 'External NAT', 'type': 'ONE_TO_ONE_NAT'}]
11261134

11271135
body = {
@@ -1165,7 +1173,7 @@ def create(self, public=False, ssh_public_key=None, user_data=None,
11651173
f'GCS access from the VM will fail'
11661174
)
11671175

1168-
if self.config.get('request_spot_instances', False) and not public:
1176+
if self.config.get('request_spot_instances', False) and not use_public_ip:
11691177
body['scheduling'] = {
11701178
'provisioningModel': 'SPOT',
11711179
'instanceTerminationAction': 'STOP'

‎lithops/standalone/utils.py‎

Lines changed: 12 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -213,10 +213,9 @@ def get_host_setup_script(
213213
configure_redis_for_standalone(){{
214214
# Workers connect to the master private IP; Redis must not listen on loopback only.
215215
if [ ! -f /etc/redis/redis.conf ]; then
216-
echo "--> Redis: /etc/redis/redis.conf not found, skipping Redis configuration"
217216
return 0
218217
fi
219-
echo "--> Redis: configuring for standalone workers (bind 0.0.0.0)"
218+
echo "--> Configuring Redis for standalone workers (bind 0.0.0.0)"
220219
sed -i -E 's/^bind .*/bind 0.0.0.0 -::1/' /etc/redis/redis.conf
221220
if grep -q '^protected-mode yes' /etc/redis/redis.conf; then
222221
sed -i 's/^protected-mode yes/protected-mode no/' /etc/redis/redis.conf
@@ -227,55 +226,26 @@ def get_host_setup_script(
227226
228227
install_packages(){{
229228
set -e
230-
if [ -f {SA_SETUP_DONE_FILE} ]; then
231-
echo "--> Lithops host setup: already completed ({SA_SETUP_DONE_FILE}), skipping install_packages"
232-
return 0
233-
fi
234-
echo "--> Lithops host setup: running install_packages"
235229
export DEBIAN_FRONTEND=noninteractive
236230
export DOCKER_REQUIRED={str(docker).lower()};
237-
INSTALL_DOCKER=false
238-
INSTALL_LITHOPS_DEPS=false
239-
if command -v docker >/dev/null 2>&1; then
240-
echo "--> docker: already installed ($(docker --version 2>/dev/null | head -1))"
241-
else
242-
INSTALL_DOCKER=true
243-
INSTALL_LITHOPS_DEPS=true
244-
echo "--> docker: not found, will install"
245-
fi
246-
if command -v unzip >/dev/null 2>&1; then
247-
echo "--> unzip: already installed"
248-
else
249-
INSTALL_LITHOPS_DEPS=true
250-
echo "--> unzip: not found, will install Lithops system dependencies"
251-
fi
252-
if command -v pip3 >/dev/null 2>&1; then
253-
echo "--> pip3: already installed ($(pip3 --version 2>/dev/null | head -1))"
254-
else
255-
INSTALL_LITHOPS_DEPS=true
256-
echo "--> pip3: not found, will install Lithops system dependencies"
257-
fi
231+
command -v docker >/dev/null 2>&1 || {{ export INSTALL_DOCKER=true; export INSTALL_LITHOPS_DEPS=true;}};
232+
command -v unzip >/dev/null 2>&1 || {{ export INSTALL_LITHOPS_DEPS=true; }};
233+
command -v pip3 >/dev/null 2>&1 || {{ export INSTALL_LITHOPS_DEPS=true; }};
258234
259235
if [ "$INSTALL_DOCKER" = true ] && [ "$DOCKER_REQUIRED" = true ]; then
260236
wait_internet_connection;
261-
echo "--> Installing Docker apt repository"
237+
echo "--> Installing Docker repository"
262238
apt_install update
263239
apt_install install -y apt-transport-https ca-certificates curl gnupg software-properties-common
264240
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
265241
DOCKER_APT="deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg]"
266242
DOCKER_APT="$DOCKER_APT https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
267243
echo "$DOCKER_APT" > /etc/apt/sources.list.d/docker.list
268-
else
269-
if [ "$DOCKER_REQUIRED" != true ]; then
270-
echo "--> Docker: not required for this runtime, skipping Docker repository setup"
271-
else
272-
echo "--> Docker: repository setup skipped (Docker already installed)"
273-
fi
274244
fi;
275245
276246
if [ "$INSTALL_LITHOPS_DEPS" = true ]; then
277247
wait_internet_connection;
278-
echo "--> Installing Lithops system dependencies (apt)"
248+
echo "--> Installing Lithops system dependencies"
279249
apt_install update
280250
281251
if [ "$INSTALL_DOCKER" = true ] && [ "$DOCKER_REQUIRED" = true ]; then
@@ -284,9 +254,7 @@ def get_host_setup_script(
284254
apt_install install -y unzip redis-server python3-pip
285255
fi;
286256
configure_redis_for_standalone
287-
else
288-
echo "--> Lithops system dependencies (apt): already present, skipping apt install"
289-
configure_redis_for_standalone
257+
290258
fi;
291259
292260
EXTRA_APT="{extra_apt_packages}"
@@ -295,15 +263,11 @@ def get_host_setup_script(
295263
apt_install update
296264
echo "--> Installing extra apt packages: $EXTRA_APT"
297265
apt_install install -y $EXTRA_APT
298-
else
299-
echo "--> Extra apt packages: none configured, skipping"
300266
fi;
301267
302-
if pip3 list 2>/dev/null | grep -q lithops; then
303-
echo "--> Lithops python package: already installed ($(pip3 show lithops 2>/dev/null | awk '/^Version:/ {{print $2}}' | head -1)), skipping pip install"
304-
else
268+
if ! pip3 list 2>/dev/null | grep -q lithops; then
305269
wait_internet_connection;
306-
echo "--> Lithops python package: not found, installing ({lithops_pip_spec})"
270+
echo "--> Installing Lithops python dependencies ({lithops_pip_spec})"
307271
export PIP_BREAK_SYSTEM_PACKAGES=1
308272
# --ignore-installed: do not uninstall Debian python packages (avoids RECORD errors)
309273
pip3 install --ignore-installed -U pip
@@ -315,18 +279,11 @@ def get_host_setup_script(
315279
echo "--> Installing extra python packages: $EXTRA_PY"
316280
export PIP_BREAK_SYSTEM_PACKAGES=1
317281
pip3 install --ignore-installed $EXTRA_PY
318-
else
319-
echo "--> Extra python packages: none configured, skipping"
320282
fi;
321-
echo "--> Lithops host setup: install_packages finished"
322283
}}
323284
"""
324285
if run_install:
325-
script += f"""echo "--> Lithops host setup: starting ({SA_SETUP_LOG_FILE})"
326-
install_packages 2>&1 | tee -a {SA_SETUP_LOG_FILE}
327-
touch {SA_SETUP_DONE_FILE}
328-
echo "--> Lithops host setup: marked complete ({SA_SETUP_DONE_FILE})"
329-
"""
286+
script += f"install_packages >> {SA_SETUP_LOG_FILE} 2>&1 && touch {SA_SETUP_DONE_FILE};\n"
330287
return script
331288

332289

@@ -374,7 +331,7 @@ def get_master_setup_script(config, vm_data):
374331
echo '127.0.0.1 lithops-master' >> /etc/hosts;
375332
cat $USER_HOME/.ssh/id_rsa.pub >> $USER_HOME/.ssh/authorized_keys;
376333
}}
377-
install_packages 2>&1 | tee -a {SA_SETUP_LOG_FILE}; test ${{PIPESTATUS[0]}} -eq 0 && touch {SA_SETUP_DONE_FILE} && \\
334+
install_packages >> {SA_SETUP_LOG_FILE} 2>&1 && touch {SA_SETUP_DONE_FILE} && \\
378335
setup_host >> {SA_SETUP_LOG_FILE} 2>&1 && \\
379336
setup_service >> {SA_SETUP_LOG_FILE} 2>&1 && \\
380337
(test -f $USER_HOME/.ssh/lithops_id_rsa || generate_ssh_key >> {SA_SETUP_LOG_FILE} 2>&1)
@@ -419,7 +376,7 @@ def get_worker_setup_script(config, vm_data):
419376
systemctl enable {WORKER_SERVICE_NAME};
420377
systemctl start {WORKER_SERVICE_NAME};
421378
}}
422-
install_packages 2>&1 | tee -a {SA_SETUP_LOG_FILE}; test ${{PIPESTATUS[0]}} -eq 0 && touch {SA_SETUP_DONE_FILE} && \\
379+
install_packages >> {SA_SETUP_LOG_FILE} 2>&1 && touch {SA_SETUP_DONE_FILE} && \\
423380
setup_host >> {SA_SETUP_LOG_FILE} 2>&1 && \\
424381
setup_service >> {SA_SETUP_LOG_FILE} 2>&1
425382
echo '{vm_data['master_ip']} lithops-master' >> /etc/hosts

0 commit comments

Comments
 (0)