Replace slurm_jobscripts.py by AccountingStoreFlags=job_script

docs/jobstats.md

@@ -16,5 +16,4 @@ Optional:
 * node\_exporter (show node information)
 * redfish\_exporter (show power information)
 * lustre\_exporter and lustre\_exporter\_slurm (show Lustre information)
-* slurm_jobscripts.py (show the submitted jobscript)
 * pcm-sensor-server from Intel PCM (show CPU information like memory bandwidth, cache misses, etc.)

jobstats/admin.py

@@ -1,9 +0,0 @@
-from django.contrib import admin
-from jobstats.models import JobScript
-
-
-class JobScriptAdmin(admin.ModelAdmin):
-    pass
-
-
-admin.site.register(JobScript, JobScriptAdmin)

jobstats/migrations/0001_initial.py

@@ -1,6 +1,6 @@
 # Generated by Django 3.1.7 on 2021-05-27 19:57
 
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
@@ -11,12 +11,4 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.CreateModel(
-            name='JobScript',
-            fields=[
-                ('id_job', models.PositiveIntegerField(primary_key=True, serialize=False)),
-                ('last_modified', models.DateTimeField(auto_now=True)),
-                ('submit_script', models.TextField()),
-            ],
-        ),
     ]

jobstats/migrations/0002_utf8.py

@@ -10,6 +10,4 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunSQL('ALTER TABLE jobstats_jobscript CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'),
-        migrations.RunSQL('ALTER TABLE jobstats_jobscript MODIFY submit_script LONGTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'),
     ]

jobstats/models.py

@@ -1,7 +0,0 @@
-from django.db import models
-
-
-class JobScript(models.Model):
-    id_job = models.PositiveIntegerField(primary_key=True)
-    last_modified = models.DateTimeField(auto_now=True)
-    submit_script = models.TextField()

jobstats/serializers.py

@@ -1,16 +1,9 @@
-from jobstats.models import JobScript
 from slurm.models import JobTable
 from rest_framework import serializers
 import time
 import datetime
 
 
-class JobScriptSerializer(serializers.HyperlinkedModelSerializer):
-    class Meta:
-        model = JobScript
-        fields = ['id_job', 'submit_script']
-
-
 class UnixEpochDateField(serializers.DateTimeField):
     def to_internal_value(self, value):
         """ Return epoch time for a datetime object or ``None``"""

jobstats/templates/jobstats/job.html

@@ -103,7 +103,7 @@ <h3>{% translate "Job analysis" %}</h3>
     </p>
     <div class="collapse" id="collapse_job_script">
       <div class="card card-body">
-        <pre><code class="language-bash">{{job_script.submit_script}}</code></pre>
+        <pre><code class="language-bash">{{job_script.batch_script}}</code></pre>
       </div>
     </div>
   {% else %}

jobstats/tests.py

@@ -1,6 +1,5 @@
 from django.conf import settings
 from tests.tests import CustomTestCase
-from jobstats.models import JobScript
 
 
 class JobstatsTestCase(CustomTestCase):
@@ -93,22 +92,3 @@ def test_user_jobstats_job(self):
             self.assertContains(response, 'Details on job')
             self.assertContains(response, '({jobid})'.format(jobid=job[1]))
             self.assertContains(response, 'Submitted job script is not available')
-
-    def test_user_jobstats_jobscript(self):
-        job = settings.TESTS_JOBSTATS[0]
-        jobscript = JobScript(
-            id_job=job[1],
-            submit_script="""#!/bin/bash
-echo 'Hello World!'
-sleep 60
-echo 'Bye World!'"""
-        )
-        jobscript.save()
-
-        response = self.user_client.get('/secure/jobstats/{user}/{jobid}/'.format(
-            user=job[0],
-            jobid=job[1]))
-        self.assertEqual(response.status_code, 200)
-        self.assertNotContains(response, 'Submitted job script is not available')
-        self.assertContains(response, 'Hello World!')
-        self.assertContains(response, 'Line 3: sleep command is used')

jobstats/views.py

@@ -1,15 +1,14 @@
 from django.shortcuts import render, redirect
 from django.http import HttpResponseNotFound, JsonResponse
-from slurm.models import JobTable, AssocTable, EventTable
+from slurm.models import JobTable, AssocTable, EventTable, JobScriptTable
 from userportal.common import user_or_staff, username_to_uid, Prometheus, request_to_username, compute_allocations_by_user, get_step, parse_start_end, fixed_zoom_config
 from django.conf import settings
 from datetime import datetime, timedelta
 from django.contrib.auth.decorators import login_required
 from django.utils.translation import gettext as _
 from rest_framework import viewsets
 from rest_framework import permissions
-from jobstats.models import JobScript
-from jobstats.serializers import JobSerializer, JobScriptSerializer
+from jobstats.serializers import JobSerializer
 from notes.models import Note
 import statistics
 from jobstats.analyze_job import find_loaded_modules, analyze_jobscript
@@ -423,15 +422,17 @@ def job(request, username, job_id):
             context['gpu_power'] = None
 
     try:
-        context['job_script'] = JobScript.objects.get(id_job=job_id)
+        context['job_script'] = JobScriptTable.objects.get(hash_inx=job.script_hash_inx)
+    except JobScriptTable.DoesNotExist:
+        context['job_script'] = None
+        context['loaded_modules'] = None
+    else:
         try:
-            modules = find_loaded_modules(context['job_script'].submit_script)
+            modules = find_loaded_modules(context['job_script'].batch_script)
             context['loaded_modules'] = modules
-            comments += analyze_jobscript(context['job_script'].submit_script, context['loaded_modules'], job)
+            comments += analyze_jobscript(context['job_script'].batch_script, context['loaded_modules'], job)
         except ValueError:
             context['loaded_modules'] = None  # Could not parse jobscript to find loaded modules
-    except JobScript.DoesNotExist:
-        context['job_script'] = None
 
     if context['cpu_used'] is not None:
         if context['cpu_used'] < 1 and context['tres_req']['total_cores'] > 1:
@@ -1984,12 +1985,6 @@ def value_cost(request, username, job_id):
     return JsonResponse(response)
 
 
-class JobScriptViewSet(viewsets.ModelViewSet):
-    queryset = JobScript.objects.all().order_by('-last_modified')
-    serializer_class = JobScriptSerializer
-    permission_classes = [permissions.IsAdminUser]
-
-
 class JobsViewSet(viewsets.ReadOnlyModelViewSet):
     serializer_class = JobSerializer
     permission_classes = [permissions.IsAuthenticated]

slurm/models.py

@@ -247,6 +247,7 @@ class StatesJob(models.IntegerChoices):
     tres_alloc = models.TextField()
     tres_req = models.TextField()
     submit_line = models.TextField(blank=True, null=True)
+    script_hash_inx = models.PositiveBigIntegerField()
 
     class Meta:
         managed = False
@@ -406,6 +407,19 @@ def depends_on_this(self):
         return deps
 
 
+class JobScriptTable(models.Model):
+    deleted = models.IntegerField()
+    hash_inx = models.BigAutoField(primary_key=True)
+    last_used = models.DateTimeField()
+    script_hash = models.TextField()
+    batch_script = models.TextField(null=True)
+
+    class Meta:
+        managed = False
+        db_table = settings.CLUSTER_NAME + '_job_script_table'
+        unique_together = (('hash_inx', 'script_hash'),)
+
+
 class LastRanTable(models.Model):
     hourly_rollup = models.PositiveBigIntegerField(primary_key=True)
     daily_rollup = models.PositiveBigIntegerField()

userportal/urls.py

@@ -29,8 +29,7 @@
 last_modified_date = timezone.now()
 
 if 'jobstats' in settings.INSTALLED_APPS:
-    from jobstats.views import JobScriptViewSet, JobsViewSet
-    router.register(r'jobscripts', JobScriptViewSet)
+    from jobstats.views import JobsViewSet
     router.register(r'jobs', JobsViewSet, basename='jobs')
 
 urlpatterns = [