Skip to content

Commit af774eb

Browse files
authored
Merge pull request #676 from hudeng-go/upstream/master
feat: Add ISO file upload progress support.
2 parents c0f3190 + 326579a commit af774eb

4 files changed

Lines changed: 210 additions & 49 deletions

File tree

conf/requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ libvirt-python==11.4.0
1515
lxml==6.0.0
1616
ldap3==2.9.1
1717
markdown==3.8.2
18+
paramiko==3.4.0
1819
#psycopg2-binary
1920
python-engineio==4.12.0
2021
python-socketio==5.13.0

storages/templates/create_stg_vol_block.html

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,27 @@
1616
<h5 class="modal-title">{% trans "Upload ISO Image" %}</h5>
1717
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
1818
</div>
19-
<div class="modal-body">
20-
<form enctype="multipart/form-data" method="post" role="form" aria-label="Upload iso form">{% csrf_token %}
21-
<div class="row">
22-
<label class="col-sm-3 col-form-label">{% trans "Name" %}</label>
23-
<div class="col-sm-6">
24-
<input type="file" name="file" id="id_file">
19+
<form id="isoUploadForm" enctype="multipart/form-data" method="post" role="form" aria-label="Upload iso form">{% csrf_token %}
20+
<div class="modal-body">
21+
<div class="row mb-3">
22+
<label for="id_file" class="col-sm-3 col-form-label">{% trans "Name" %}</label>
23+
<div class="col-sm-9">
24+
<input type="file" name="file" id="id_file" class="form-control" required>
2525
</div>
2626
</div>
27-
</div>
28-
<div class="modal-footer">
29-
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">{% trans "Close" %}</button>
30-
<button type="submit" class="btn btn-primary" name="iso_upload">{% trans "Upload" %}</button>
31-
</div>
32-
</form>
27+
<div id="upload-progress-container" class="mt-3" style="display: none;">
28+
<div class="progress">
29+
<div id="upload-progress-bar" class="progress-bar" role="progressbar" style="width: 0%;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">0%</div>
30+
</div>
31+
</div>
32+
<div id="upload-error-message" class="alert alert-danger mt-3" style="display: none;"></div>
33+
</div>
34+
<div class="modal-footer">
35+
<input type="hidden" name="iso_upload" value="true">
36+
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">{% trans "Close" %}</button>
37+
<button type="submit" class="btn btn-primary">{% trans "Upload" %}</button>
38+
</div>
39+
</form>
3340

3441
</div> <!-- /.modal-content -->
3542
</div> <!-- /.modal-dialog -->
@@ -62,4 +69,4 @@ <h5 class="modal-title">{% trans "Add New Volume" %}</h5>
6269
</div> <!-- /.modal -->
6370
{% endif %}
6471
{% endif %}
65-
{% endif %}
72+
{% endif %}

storages/templates/storage.html

Lines changed: 83 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,32 +58,31 @@
5858
<dd class="col-6">{{ used|filesizeformat }}</dd>
5959
<dt class="col-6">{% trans "State" %}</dt>
6060
<dd class="col-6">
61-
<form action="" method="post" role="form" aria-label="Storage start/stop form">{% csrf_token %}
61+
<form action="" method="post" role="form" aria-label="Storage start/stop form" class="confirm-form">{% csrf_token %}
6262
{% if state == 0 %}
6363
<input type="submit" class="btn btn-sm btn-secondary" name="start" value="{% trans "Start" %}">
64-
<input type="submit" class="btn btn-sm btn-danger" name="delete" value="{% trans "Delete" %}"
65-
onclick="return confirm('{% trans "Are you sure?" %}')">
64+
<input type="submit" class="btn btn-sm btn-danger" name="delete" value="{% trans "Delete" %}">
6665
{% else %}
67-
<input type="submit" class="btn btn-sm btn-secondary" name="stop" value="{% trans "Stop" %}"
68-
onclick="return confirm('{% trans "Are you sure?" %}')">
66+
<input type="submit" class="btn btn-sm btn-secondary" name="stop" value="{% trans "Stop" %}">
6967
{% endif %}
7068
</form>
7169
</dd>
7270
<dt class="col-6">{% trans "Autostart" %}</dt>
7371
<dd class="col-6">
74-
<form action="" method="post" role="form" aria-label="Storage disable/enable autostart form">{% csrf_token %}
72+
<form action="" method="post" role="form" aria-label="Storage disable/enable autostart form" class="confirm-form">{% csrf_token %}
7573
{% if autostart == 0 %}
7674
<input type="submit" class="btn btn-sm btn-secondary" name="set_autostart"
7775
value="{% trans "Enable" %}">
7876
{% else %}
7977
<input type="submit" class="btn btn-sm btn-secondary" name="unset_autostart"
80-
onclick="return confirm('{% trans "Are you sure?" %}')" value="{% trans "Disable" %}">
78+
value="{% trans "Disable" %}">
8179
{% endif %}
8280
</form>
8381
</dd>
8482
</dl>
8583
</div>
8684
</div>
85+
8786

8887
{% if state %}
8988
<p>
@@ -171,9 +170,9 @@ <h5 class="modal-title">{% trans "Clone image" %} <span class="text-danger">{{ v
171170
{% endif %}
172171
</td>
173172
<td>
174-
<form action="" method="post" role="form" aria-label="Delete volume form">{% csrf_token %}
173+
<form action="" method="post" role="form" aria-label="Delete volume form" class="confirm-form">{% csrf_token %}
175174
<input type="hidden" name="volname" value="{{ volume.name }}">
176-
<button type="submit" class="btn btn-sm btn-secondary" name="del_volume" title="{% trans "Delete" %}" onclick="return confirm('{% trans "Are you sure?" %}')">
175+
<button type="submit" class="btn btn-sm btn-secondary" name="del_volume" title="{% trans "Delete" %}">
177176
{% bs_icon 'trash' %}
178177
</button>
179178
</form>
@@ -216,5 +215,80 @@ <h5 class="modal-title">{% trans "Clone image" %} <span class="text-danger">{{ v
216215
$('.meta-prealloc').hide();
217216
}
218217
});
218+
219+
$(document).ready(function() {
220+
$('.confirm-form').on('submit', function(e) {
221+
if (!confirm('{% trans "Are you sure?"|escapejs %}')) {
222+
e.preventDefault();
223+
}
224+
});
225+
226+
$('#isoUploadForm').on('submit', function(e) {
227+
e.preventDefault();
228+
229+
var fileInput = $('#id_file')[0];
230+
if (fileInput.files.length === 0) {
231+
alert("{% trans 'Please select a file to upload.'|escapejs %}");
232+
return;
233+
}
234+
235+
var file = fileInput.files[0];
236+
var fileName = file.name;
237+
var chunkSize = 10 * 1024 * 1024; // 10MB
238+
var totalChunks = Math.ceil(file.size / chunkSize);
239+
var chunkIndex = 0;
240+
241+
var progressBar = $('#upload-progress-bar');
242+
var progressContainer = $('#upload-progress-container');
243+
var errorMessage = $('#upload-error-message');
244+
245+
progressContainer.show();
246+
progressBar.width('0%').attr('aria-valuenow', 0).text('0%');
247+
errorMessage.hide();
248+
249+
function uploadChunk() {
250+
if (chunkIndex >= totalChunks) {
251+
return;
252+
}
253+
254+
var start = chunkIndex * chunkSize;
255+
var end = Math.min(start + chunkSize, file.size);
256+
var chunk = file.slice(start, end);
257+
258+
var formData = new FormData();
259+
formData.append('file', chunk, fileName);
260+
formData.append('file_name', fileName);
261+
formData.append('chunk_index', chunkIndex);
262+
formData.append('total_chunks', totalChunks);
263+
formData.append('iso_upload', 'true');
264+
formData.append('csrfmiddlewaretoken', '{{ csrf_token }}');
265+
266+
$.ajax({
267+
url: "{% url 'storage' compute.id pool %}",
268+
type: 'POST',
269+
data: formData,
270+
processData: false,
271+
contentType: false,
272+
success: function(data) {
273+
chunkIndex++;
274+
var percentComplete = Math.round((chunkIndex / totalChunks) * 100);
275+
progressBar.width(percentComplete + '%').attr('aria-valuenow', percentComplete).text(percentComplete + '%');
276+
277+
if (data.reload) {
278+
location.reload();
279+
} else if (chunkIndex < totalChunks) {
280+
uploadChunk();
281+
}
282+
},
283+
error: function(jqXHR, textStatus, errorThrown) {
284+
var errorText = jqXHR.responseJSON && jqXHR.responseJSON.error ? jqXHR.responseJSON.error : "{% trans 'An error occurred during upload.'|escapejs %}";
285+
errorMessage.text(errorText).show();
286+
progressContainer.hide();
287+
}
288+
});
289+
}
290+
uploadChunk();
291+
});
292+
});
219293
</script>
220294
{% endblock %}

storages/views.py

Lines changed: 106 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,14 @@
55
from appsettings.settings import app_settings
66
from computes.models import Compute
77
from django.contrib import messages
8-
from django.http import HttpResponse, HttpResponseRedirect
8+
from django.http import HttpResponse, HttpResponseRedirect, JsonResponse
99
from django.shortcuts import get_object_or_404, redirect, render
1010
from django.urls import reverse
1111
from django.utils.translation import gettext_lazy as _
1212
from libvirt import libvirtError
13+
import paramiko
14+
15+
from vrtManager.connection import CONN_SSH, CONN_SOCKET
1316
from vrtManager.storage import wvmStorage, wvmStorages
1417

1518
from storages.forms import AddStgPool, CloneImage, CreateVolumeForm
@@ -102,20 +105,52 @@ def storage(request, compute_id, pool):
102105
:param pool:
103106
:return:
104107
"""
108+
def handle_uploaded_file(conn, path, file_name, file_chunk, is_last_chunk):
109+
temp_name = f"{file_name}.part"
110+
target_temp = os.path.normpath(os.path.join(path, temp_name))
111+
target_final = os.path.normpath(os.path.join(path, file_name))
105112

106-
def handle_uploaded_file(path, f_name):
107-
target = os.path.normpath(os.path.join(path, str(f_name)))
108-
if not target.startswith(path):
113+
if not target_temp.startswith(path) or not target_final.startswith(path):
109114
raise Exception(_("Security Issues with file uploading"))
110115

111-
try:
112-
with open(target, "wb+") as f:
113-
for chunk in f_name.chunks():
114-
f.write(chunk)
115-
except FileNotFoundError:
116-
messages.error(
117-
request, _("File not found. Check the path variable and filename")
118-
)
116+
if conn.conn == CONN_SSH:
117+
try:
118+
hostname, port = conn.host, 22
119+
if ":" in hostname:
120+
hostname, port_str = hostname.split(":")
121+
port = int(port_str)
122+
123+
ssh = paramiko.SSHClient()
124+
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
125+
ssh.connect(hostname=hostname, port=port, username=conn.login, password=conn.passwd)
126+
sftp = ssh.open_sftp()
127+
128+
remote_file = sftp.open(target_temp, 'ab')
129+
remote_file.set_pipelined(True)
130+
for chunk_data in file_chunk.chunks():
131+
remote_file.write(chunk_data)
132+
remote_file.close()
133+
134+
if is_last_chunk:
135+
sftp.rename(target_temp, target_final)
136+
137+
sftp.close()
138+
ssh.close()
139+
except Exception as e:
140+
raise Exception(_("SSH upload failed: {}").format(e))
141+
elif conn.conn == CONN_SOCKET:
142+
try:
143+
with open(target_temp, "ab") as f:
144+
for chunk_data in file_chunk.chunks():
145+
f.write(chunk_data)
146+
if is_last_chunk:
147+
if os.path.exists(target_final):
148+
os.remove(target_final)
149+
os.rename(target_temp, target_final)
150+
except FileNotFoundError:
151+
raise Exception(_("File not found. Check the path variable and filename"))
152+
else:
153+
raise Exception(_("Unsupported connection type for file upload."))
119154

120155
compute = get_object_or_404(Compute, pk=compute_id)
121156
meta_prealloc = False
@@ -127,12 +162,16 @@ def handle_uploaded_file(path, f_name):
127162

128163
storages = conn.get_storages()
129164
state = conn.is_active()
130-
size, free = conn.get_size()
131-
used = size - free
132-
if state:
133-
percent = (used * 100) // size
134-
else:
135-
percent = 0
165+
try:
166+
size, free = conn.get_size()
167+
used = size - free
168+
if state:
169+
percent = (used * 100) // size
170+
else:
171+
percent = 0
172+
except libvirtError:
173+
size, free, used, percent = 0, 0, 0, 0
174+
136175
status = conn.get_status()
137176
path = conn.get_target_path()
138177
type = conn.get_type()
@@ -170,16 +209,56 @@ def handle_uploaded_file(path, f_name):
170209
return redirect(reverse("storage", args=[compute.id, pool]))
171210
# return HttpResponseRedirect(request.get_full_path())
172211
if "iso_upload" in request.POST:
173-
if str(request.FILES["file"]) in conn.update_volumes():
174-
error_msg = _("ISO image already exist")
212+
file_chunk = request.FILES.get("file")
213+
if not file_chunk:
214+
return JsonResponse({"error": _("No file chunk was submitted.")}, status=400)
215+
216+
file_name = request.POST.get("file_name")
217+
chunk_index = int(request.POST.get("chunk_index", 0))
218+
total_chunks = int(request.POST.get("total_chunks", 1))
219+
is_last_chunk = chunk_index == total_chunks - 1
220+
221+
# On first chunk, check if file already exists
222+
if chunk_index == 0:
223+
if file_name in conn.get_volumes():
224+
return JsonResponse({"error": _("ISO image already exists")}, status=400)
225+
# Clean up any partial files from previous failed uploads
226+
temp_part_file = os.path.normpath(os.path.join(path, f"{file_name}.part"))
227+
if conn.conn == CONN_SOCKET and os.path.exists(temp_part_file):
228+
os.remove(temp_part_file)
229+
elif conn.conn == CONN_SSH:
230+
try:
231+
hostname, port = conn.host, 22
232+
if ":" in hostname:
233+
hostname, port_str = hostname.split(":")
234+
port = int(port_str)
235+
ssh = paramiko.SSHClient()
236+
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
237+
ssh.connect(hostname=hostname, port=port, username=conn.login, password=conn.passwd)
238+
sftp = ssh.open_sftp()
239+
try:
240+
sftp.remove(temp_part_file)
241+
except FileNotFoundError:
242+
pass # File doesn't exist, which is fine
243+
sftp.close()
244+
ssh.close()
245+
except Exception:
246+
# Best effort to clean up, if it fails, let it be.
247+
pass
248+
249+
try:
250+
handle_uploaded_file(conn, path, file_name, file_chunk, is_last_chunk)
251+
252+
if is_last_chunk:
253+
success_msg = _("ISO: %(file)s has been uploaded successfully.") % {"file": file_name}
254+
messages.success(request, success_msg)
255+
return JsonResponse({"success": True, "message": success_msg, "reload": True})
256+
else:
257+
return JsonResponse({"success": True, "message": "Chunk received."})
258+
except Exception as e:
259+
error_msg = str(e)
175260
messages.error(request, error_msg)
176-
else:
177-
handle_uploaded_file(path, request.FILES["file"])
178-
messages.success(
179-
request,
180-
_("ISO: %(file)s is uploaded.") % {"file": request.FILES["file"]},
181-
)
182-
return HttpResponseRedirect(request.get_full_path())
261+
return JsonResponse({"error": error_msg}, status=500)
183262
if "cln_volume" in request.POST:
184263
form = CloneImage(request.POST)
185264
if form.is_valid():

0 commit comments

Comments
 (0)