From 1034ee7380647011304bfb53afbd9f27802f4a7e Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 5 Feb 2024 13:41:23 -0800 Subject: [PATCH 01/51] 14438 script model --- .../migrations/0107_alter_script_options.py | 35 +++++++++++++++++++ netbox/extras/models/scripts.py | 17 ++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 netbox/extras/migrations/0107_alter_script_options.py diff --git a/netbox/extras/migrations/0107_alter_script_options.py b/netbox/extras/migrations/0107_alter_script_options.py new file mode 100644 index 0000000000..6829761f3e --- /dev/null +++ b/netbox/extras/migrations/0107_alter_script_options.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.9 on 2024-02-05 21:37 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('extras', '0106_bookmark_user_cascade_deletion'), + ] + + operations = [ + migrations.AddField( + model_name='script', + name='name', + field=models.CharField(default=None, max_length=79), + preserve_default=False, + ), + migrations.AddField( + model_name='script', + name='script_module', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.PROTECT, related_name='scripts', to='extras.scriptmodule'), + preserve_default=False, + ), + migrations.AlterField( + model_name='script', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False), + ), + migrations.AlterModelOptions( + name='script', + options={'ordering': ('name', 'pk')}, + ), + ] diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index 93275acdab..2f1454e2ad 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -22,11 +22,18 @@ class Script(EventRulesMixin, models.Model): - """ - Dummy model used to generate permissions for custom scripts. Does not exist in the database. - """ + name = models.CharField( + verbose_name=_('name'), + max_length=79, + ) + script_module = models.ForeignKey( + to='extras.ScriptModule', + on_delete=models.PROTECT, + related_name='scripts' + ) + class Meta: - managed = False + ordering = ('name', 'pk') class ScriptModuleManager(models.Manager.from_queryset(RestrictedQuerySet)): @@ -52,6 +59,7 @@ def get_absolute_url(self): def __str__(self): return self.python_name + ''' @cached_property def scripts(self): @@ -75,6 +83,7 @@ def _get_name(cls): scripts[_get_name(cls)] = cls return scripts + ''' def save(self, *args, **kwargs): self.file_root = ManagedFileRootPathChoices.SCRIPTS From 4cb3b63f79ea9814bc4627bcad4f8be683591cca Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 5 Feb 2024 14:16:24 -0800 Subject: [PATCH 02/51] 14438 script model --- .../migrations/0107_alter_script_options.py | 18 +++++++++++++++++- netbox/extras/models/scripts.py | 6 ++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/netbox/extras/migrations/0107_alter_script_options.py b/netbox/extras/migrations/0107_alter_script_options.py index 6829761f3e..47612d8f86 100644 --- a/netbox/extras/migrations/0107_alter_script_options.py +++ b/netbox/extras/migrations/0107_alter_script_options.py @@ -4,6 +4,18 @@ import django.db.models.deletion +def update_scripts(apps, schema_editor): + ScriptModule = apps.get_model('extras', 'ScriptModule') + Script = apps.get_model('extras', 'Script') + + for module in ScriptModule.objects.all(): + for script, cls in module.get_module_scripts: + Script.objects.create( + name=script, + script_module=module, + ) + + class Migration(migrations.Migration): dependencies = [ @@ -19,7 +31,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='script', - name='script_module', + name='module', field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.PROTECT, related_name='scripts', to='extras.scriptmodule'), preserve_default=False, ), @@ -32,4 +44,8 @@ class Migration(migrations.Migration): name='script', options={'ordering': ('name', 'pk')}, ), + migrations.RunPython( + code=update_scripts, + reverse_code=migrations.RunPython.noop + ), ] diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index 2f1454e2ad..baaa9b9731 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -26,7 +26,7 @@ class Script(EventRulesMixin, models.Model): verbose_name=_('name'), max_length=79, ) - script_module = models.ForeignKey( + module = models.ForeignKey( to='extras.ScriptModule', on_delete=models.PROTECT, related_name='scripts' @@ -59,9 +59,8 @@ def get_absolute_url(self): def __str__(self): return self.python_name - ''' @cached_property - def scripts(self): + def get_module_scripts(self): def _get_name(cls): # For child objects in submodules use the full import path w/o the root module as the name @@ -83,7 +82,6 @@ def _get_name(cls): scripts[_get_name(cls)] = cls return scripts - ''' def save(self, *args, **kwargs): self.file_root = ManagedFileRootPathChoices.SCRIPTS From fc890e36afab0354b9918829bd323cfeddf8a0ce Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 5 Feb 2024 15:15:01 -0800 Subject: [PATCH 03/51] 14438 script model --- .../migrations/0107_alter_script_options.py | 37 +++++++------------ netbox/extras/models/scripts.py | 7 ++++ netbox/templates/extras/script_list.html | 8 ++-- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/netbox/extras/migrations/0107_alter_script_options.py b/netbox/extras/migrations/0107_alter_script_options.py index 47612d8f86..b49b9ba5e7 100644 --- a/netbox/extras/migrations/0107_alter_script_options.py +++ b/netbox/extras/migrations/0107_alter_script_options.py @@ -5,14 +5,15 @@ def update_scripts(apps, schema_editor): - ScriptModule = apps.get_model('extras', 'ScriptModule') + from extras.models import ScriptModule + ScriptModuleNew = apps.get_model('extras', 'ScriptModule') Script = apps.get_model('extras', 'Script') for module in ScriptModule.objects.all(): - for script, cls in module.get_module_scripts: + for script in module.get_module_scripts.keys(): Script.objects.create( name=script, - script_module=module, + module=ScriptModuleNew.objects.get(file_root=module.file_root, file_path=module.file_path), ) @@ -23,26 +24,16 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AddField( - model_name='script', - name='name', - field=models.CharField(default=None, max_length=79), - preserve_default=False, - ), - migrations.AddField( - model_name='script', - name='module', - field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.PROTECT, related_name='scripts', to='extras.scriptmodule'), - preserve_default=False, - ), - migrations.AlterField( - model_name='script', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False), - ), - migrations.AlterModelOptions( - name='script', - options={'ordering': ('name', 'pk')}, + migrations.CreateModel( + name='Script', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=79)), + ('module', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='scripts', to='extras.scriptmodule')), + ], + options={ + 'ordering': ('name', 'pk'), + }, ), migrations.RunPython( code=update_scripts, diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index baaa9b9731..fb0be49c68 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -32,9 +32,16 @@ class Script(EventRulesMixin, models.Model): related_name='scripts' ) + def __str__(self): + return self.name + class Meta: ordering = ('name', 'pk') + @cached_property + def python_class(self): + return self.module.get_module_scripts.get(self.name) + class ScriptModuleManager(models.Manager.from_queryset(RestrictedQuerySet)): diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html index bb91f75222..d5c8711e51 100644 --- a/netbox/templates/extras/script_list.html +++ b/netbox/templates/extras/script_list.html @@ -53,15 +53,15 @@
{% with jobs=module.get_latest_jobs %} - {% for script_name, script_class in module.scripts.items %} + {% for script in module.scripts.all %} - {{ script_class.name }} + {{ script.python_class.name }} - {{ script_class.Meta.description|markdown|placeholder }} + {{ script.python_class.Meta.description|markdown|placeholder }} - {% with last_result=jobs|get_key:script_class.class_name %} + {% with last_result=jobs|get_key:script.python_class.class_name %} {% if last_result %} {{ last_result.created|annotated_date }} From 6930e441ae7a0be01f1d7367e048c6393d461ebd Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 6 Feb 2024 09:32:19 -0800 Subject: [PATCH 04/51] 14438 script job mapping --- .../migrations/0107_alter_script_options.py | 11 +++- netbox/extras/models/scripts.py | 15 +++++- netbox/extras/urls.py | 6 +-- netbox/extras/views.py | 51 ++++++++++--------- netbox/templates/extras/script.html | 2 +- netbox/templates/extras/script/base.html | 6 +-- netbox/templates/extras/script/source.html | 4 +- netbox/templates/extras/script_list.html | 2 +- 8 files changed, 61 insertions(+), 36 deletions(-) diff --git a/netbox/extras/migrations/0107_alter_script_options.py b/netbox/extras/migrations/0107_alter_script_options.py index b49b9ba5e7..25988d7e40 100644 --- a/netbox/extras/migrations/0107_alter_script_options.py +++ b/netbox/extras/migrations/0107_alter_script_options.py @@ -8,14 +8,19 @@ def update_scripts(apps, schema_editor): from extras.models import ScriptModule ScriptModuleNew = apps.get_model('extras', 'ScriptModule') Script = apps.get_model('extras', 'Script') + ContentType = apps.get_model('contenttypes', 'ContentType') + ct = ContentType.objects.get(app_label='extras', model='script') for module in ScriptModule.objects.all(): for script in module.get_module_scripts.keys(): - Script.objects.create( + obj = Script.objects.create( name=script, module=ScriptModuleNew.objects.get(file_root=module.file_root, file_path=module.file_path), ) + # update all jobs associated with this module/name to point to the new script obj + module.jobs.filter(name=name).update(object_type=ct, object_id=obj.id) + class Migration(migrations.Migration): @@ -35,6 +40,10 @@ class Migration(migrations.Migration): 'ordering': ('name', 'pk'), }, ), + migrations.AddConstraint( + model_name='script', + constraint=models.UniqueConstraint(fields=('name', 'module'), name='extras_script_unique_name_module'), + ), migrations.RunPython( code=update_scripts, reverse_code=migrations.RunPython.noop diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index fb0be49c68..f39ef2e441 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -21,7 +21,7 @@ logger = logging.getLogger('netbox.data_backends') -class Script(EventRulesMixin, models.Model): +class Script(EventRulesMixin, JobsMixin, models.Model): name = models.CharField( verbose_name=_('name'), max_length=79, @@ -37,11 +37,24 @@ def __str__(self): class Meta: ordering = ('name', 'pk') + constraints = ( + models.UniqueConstraint( + fields=('name', 'module'), + name='%(app_label)s_%(class)s_unique_name_module' + ), + ) + verbose_name = _('script') + verbose_name_plural = _('scripts') @cached_property def python_class(self): return self.module.get_module_scripts.get(self.name) + def get_jobs(self): + return self.module.jobs.filter( + name=self.name + ) + class ScriptModuleManager(models.Manager.from_queryset(RestrictedQuerySet)): diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index 0a1786f1f3..919c54c6fd 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -130,9 +130,9 @@ path('scripts/add/', views.ScriptModuleCreateView.as_view(), name='scriptmodule_add'), path('scripts/results//', views.ScriptResultView.as_view(), name='script_result'), path('scripts//', include(get_model_urls('extras', 'scriptmodule'))), - path('scripts///', views.ScriptView.as_view(), name='script'), - path('scripts///source/', views.ScriptSourceView.as_view(), name='script_source'), - path('scripts///jobs/', views.ScriptJobsView.as_view(), name='script_jobs'), + path('scripts/class//', views.ScriptView.as_view(), name='script'), + path('scripts/class//source/', views.ScriptSourceView.as_view(), name='script_source'), + path('scripts/class//jobs/', views.ScriptJobsView.as_view(), name='script_jobs'), # Markdown path('render/markdown/', views.RenderMarkdownView.as_view(), name="render_markdown"), diff --git a/netbox/extras/views.py b/netbox/extras/views.py index ad00d74123..c81c03a5e4 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -1226,27 +1226,28 @@ class ScriptView(ContentTypePermissionRequiredMixin, View): def get_required_permission(self): return 'extras.view_script' - def get(self, request, module, name): - module = get_script_module(module, request) - script = module.scripts[name]() - jobs = module.get_jobs(script.class_name) - form = script.as_form(initial=normalize_querydict(request.GET)) + def get(self, request, pk): + script = Script.objects.get(pk=pk) + script_class = script.python_class() + jobs = script.get_jobs() + form = script_class.as_form(initial=normalize_querydict(request.GET)) return render(request, 'extras/script.html', { 'job_count': jobs.count(), - 'module': module, + 'module': script.module, 'script': script, + 'script_class': script_class, 'form': form, }) - def post(self, request, module, name): + def post(self, request, pk): if not request.user.has_perm('extras.run_script'): return HttpResponseForbidden() - module = get_script_module(module, request) - script = module.scripts[name]() - jobs = module.get_jobs(script.class_name) - form = script.as_form(request.POST, request.FILES) + script = Script.objects.get(pk=pk) + script_class = script.python_class() + jobs = script.get_jobs() + form = script_class.as_form(request.POST, request.FILES) # Allow execution only if RQ worker process is running if not get_workers_for_queue('default'): @@ -1255,8 +1256,8 @@ def post(self, request, module, name): elif form.is_valid(): job = Job.enqueue( run_script, - instance=module, - name=script.class_name, + instance=script.module, + name=script_class.class_name, user=request.user, schedule_at=form.cleaned_data.pop('_schedule_at'), interval=form.cleaned_data.pop('_interval'), @@ -1270,8 +1271,9 @@ def post(self, request, module, name): return render(request, 'extras/script.html', { 'job_count': jobs.count(), - 'module': module, + 'module': script.module, 'script': script, + 'script_class': script_class, 'form': form, }) @@ -1281,15 +1283,16 @@ class ScriptSourceView(ContentTypePermissionRequiredMixin, View): def get_required_permission(self): return 'extras.view_script' - def get(self, request, module, name): - module = get_script_module(module, request) - script = module.scripts[name]() - jobs = module.get_jobs(script.class_name) + def get(self, request, pk): + script = Script.objects.get(pk=pk) + script_class = script.python_class() + jobs = script.get_jobs() return render(request, 'extras/script/source.html', { 'job_count': jobs.count(), - 'module': module, + 'module': script.module, 'script': script, + 'script_class': script_class, 'tab': 'source', }) @@ -1299,10 +1302,10 @@ class ScriptJobsView(ContentTypePermissionRequiredMixin, View): def get_required_permission(self): return 'extras.view_script' - def get(self, request, module, name): - module = get_script_module(module, request) - script = module.scripts[name]() - jobs = module.get_jobs(script.class_name) + def get(self, request, pk): + script = Script.objects.get(pk=pk) + script_class = script.python_class() + jobs = script.get_jobs() jobs_table = JobTable( data=jobs, @@ -1313,7 +1316,7 @@ def get(self, request, module, name): return render(request, 'extras/script/jobs.html', { 'job_count': jobs.count(), - 'module': module, + 'module': script.module, 'script': script, 'table': jobs_table, 'tab': 'jobs', diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html index dbed132be8..074d06fe5d 100644 --- a/netbox/templates/extras/script.html +++ b/netbox/templates/extras/script.html @@ -17,7 +17,7 @@ {% csrf_token %}
{# Render grouped fields according to declared fieldsets #} - {% for group, fields in script.get_fieldsets %} + {% for group, fields in script_class.get_fieldsets %} {% if fields %}
diff --git a/netbox/templates/extras/script/base.html b/netbox/templates/extras/script/base.html index 194f097e7f..429db590e3 100644 --- a/netbox/templates/extras/script/base.html +++ b/netbox/templates/extras/script/base.html @@ -26,13 +26,13 @@ {% block tabs %}
    diff --git a/netbox/templates/extras/script/source.html b/netbox/templates/extras/script/source.html index 733f6a1f3a..96362c7ef0 100644 --- a/netbox/templates/extras/script/source.html +++ b/netbox/templates/extras/script/source.html @@ -1,6 +1,6 @@ {% extends 'extras/script/base.html' %} {% block content %} - {{ script.filename }} - 
    {{ script.source }}
    + {{ script_class.filename }} + 
    {{ script_class.source }}
    {% endblock %} diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html index d5c8711e51..7abf814516 100644 --- a/netbox/templates/extras/script_list.html +++ b/netbox/templates/extras/script_list.html @@ -56,7 +56,7 @@
    {% for script in module.scripts.all %} - {{ script.python_class.name }} + {{ script.python_class.name }} {{ script.python_class.Meta.description|markdown|placeholder }} From 5981fc958dac30381c526240c6bdfb2320feb719 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 6 Feb 2024 09:58:07 -0800 Subject: [PATCH 05/51] 14438 script job mapping --- .../migrations/0107_alter_script_options.py | 2 +- netbox/extras/models/scripts.py | 5 -- netbox/extras/views.py | 12 ++--- netbox/templates/extras/script_list.html | 46 +++++++++---------- 4 files changed, 29 insertions(+), 36 deletions(-) diff --git a/netbox/extras/migrations/0107_alter_script_options.py b/netbox/extras/migrations/0107_alter_script_options.py index 25988d7e40..88a24e4656 100644 --- a/netbox/extras/migrations/0107_alter_script_options.py +++ b/netbox/extras/migrations/0107_alter_script_options.py @@ -19,7 +19,7 @@ def update_scripts(apps, schema_editor): ) # update all jobs associated with this module/name to point to the new script obj - module.jobs.filter(name=name).update(object_type=ct, object_id=obj.id) + module.jobs.filter(name=script).update(object_type=ct, object_id=obj.id) class Migration(migrations.Migration): diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index f39ef2e441..147cb64067 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -50,11 +50,6 @@ class Meta: def python_class(self): return self.module.get_module_scripts.get(self.name) - def get_jobs(self): - return self.module.jobs.filter( - name=self.name - ) - class ScriptModuleManager(models.Manager.from_queryset(RestrictedQuerySet)): diff --git a/netbox/extras/views.py b/netbox/extras/views.py index c81c03a5e4..70a662c063 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -1209,7 +1209,7 @@ def get_required_permission(self): return 'extras.view_script' def get(self, request): - script_modules = ScriptModule.objects.restrict(request.user) + script_modules = ScriptModule.objects.restrict(request.user).prefetch_related('jobs') return render(request, 'extras/script_list.html', { 'model': ScriptModule, @@ -1229,7 +1229,7 @@ def get_required_permission(self): def get(self, request, pk): script = Script.objects.get(pk=pk) script_class = script.python_class() - jobs = script.get_jobs() + jobs = script.jobs.all() form = script_class.as_form(initial=normalize_querydict(request.GET)) return render(request, 'extras/script.html', { @@ -1246,7 +1246,7 @@ def post(self, request, pk): script = Script.objects.get(pk=pk) script_class = script.python_class() - jobs = script.get_jobs() + jobs = script.jobs.all() form = script_class.as_form(request.POST, request.FILES) # Allow execution only if RQ worker process is running @@ -1256,7 +1256,7 @@ def post(self, request, pk): elif form.is_valid(): job = Job.enqueue( run_script, - instance=script.module, + instance=script, name=script_class.class_name, user=request.user, schedule_at=form.cleaned_data.pop('_schedule_at'), @@ -1286,7 +1286,7 @@ def get_required_permission(self): def get(self, request, pk): script = Script.objects.get(pk=pk) script_class = script.python_class() - jobs = script.get_jobs() + jobs = script.jobs.all() return render(request, 'extras/script/source.html', { 'job_count': jobs.count(), @@ -1305,7 +1305,7 @@ def get_required_permission(self): def get(self, request, pk): script = Script.objects.get(pk=pk) script_class = script.python_class() - jobs = script.get_jobs() + jobs = script.jobs.all() jobs_table = JobTable( data=jobs, diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html index 7abf814516..74f7be8336 100644 --- a/netbox/templates/extras/script_list.html +++ b/netbox/templates/extras/script_list.html @@ -52,31 +52,29 @@
    - {% with jobs=module.get_latest_jobs %} - {% for script in module.scripts.all %} - - - {{ script.python_class.name }} - - - {{ script.python_class.Meta.description|markdown|placeholder }} - - {% with last_result=jobs|get_key:script.python_class.class_name %} - {% if last_result %} - - {{ last_result.created|annotated_date }} - - - {% badge last_result.get_status_display last_result.get_status_color %} - - {% else %} - {% trans "Never" %} - {{ ''|placeholder }} - {% endif %} - {% endwith %} + {% for script in module.scripts.all %} + + + {{ script.python_class.name }} + + + {{ script.python_class.Meta.description|markdown|placeholder }} + + {% with last_result=script.get_latest_jobs|get_key:script.name %} + {% if last_result %} + + {{ last_result.created|annotated_date }} + + + {% badge last_result.get_status_display last_result.get_status_color %} + + {% else %} + {% trans "Never" %} + {{ ''|placeholder }} + {% endif %} - {% endfor %} - {% endwith %} + {% endwith %} + {% endfor %} {% endif %} From 95431690253ee7cd98b0982aab3716c826c7ebb2 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 6 Feb 2024 10:23:51 -0800 Subject: [PATCH 06/51] 14438 script job run --- netbox/extras/scripts.py | 5 ++--- netbox/extras/views.py | 8 +++----- netbox/templates/extras/script_result.html | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index f28465547b..403aaec29f 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -16,7 +16,7 @@ from core.models import Job from extras.api.serializers import ScriptOutputSerializer from extras.choices import LogLevelChoices -from extras.models import ScriptModule +from extras.models import ScriptModule, Script as ScriptModel from extras.signals import clear_events from ipam.formfields import IPAddressFormField, IPNetworkFormField from ipam.validators import MaxPrefixLengthValidator, MinPrefixLengthValidator, prefix_validator @@ -485,8 +485,7 @@ def run_script(data, job, request=None, commit=True, **kwargs): """ job.start() - module = ScriptModule.objects.get(pk=job.object_id) - script = module.scripts.get(job.name)() + script = ScriptModel.objects.get(pk=job.object_id).python_class() logger = logging.getLogger(f"netbox.scripts.{script.full_name}") logger.info(f"Running script (commit={commit})") diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 70a662c063..a0cc9a8792 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -1252,7 +1252,6 @@ def post(self, request, pk): # Allow execution only if RQ worker process is running if not get_workers_for_queue('default'): messages.error(request, "Unable to run script: RQ worker process not running.") - elif form.is_valid(): job = Job.enqueue( run_script, @@ -1263,7 +1262,7 @@ def post(self, request, pk): interval=form.cleaned_data.pop('_interval'), data=form.cleaned_data, request=copy_safe_request(request), - job_timeout=script.job_timeout, + job_timeout=script.python_class.job_timeout, commit=form.cleaned_data.pop('_commit') ) @@ -1329,11 +1328,10 @@ def get_required_permission(self): return 'extras.view_script' def get(self, request, job_pk): - object_type = ContentType.objects.get_by_natural_key(app_label='extras', model='scriptmodule') + object_type = ContentType.objects.get_by_natural_key(app_label='extras', model='script') job = get_object_or_404(Job.objects.all(), pk=job_pk, object_type=object_type) - module = job.object - script = module.scripts[job.name]() + script = job.object # If this is an HTMX request, return only the result HTML if request.htmx: diff --git a/netbox/templates/extras/script_result.html b/netbox/templates/extras/script_result.html index 985c58d634..53606ed816 100644 --- a/netbox/templates/extras/script_result.html +++ b/netbox/templates/extras/script_result.html @@ -16,7 +16,7 @@
    1. {% trans "Scripts" %}
    2. {{ script.module|bettertitle }}
      3. -
    3. {{ script }}
      4. +
    4. {{ script }}
    5. {{ job.created|annotated_date }}
    From d162d440b045b446545236ebd57de68ab0b6920b Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 6 Feb 2024 10:43:13 -0800 Subject: [PATCH 07/51] 14438 fix migration --- netbox/extras/migrations/0107_alter_script_options.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/netbox/extras/migrations/0107_alter_script_options.py b/netbox/extras/migrations/0107_alter_script_options.py index 88a24e4656..74716df673 100644 --- a/netbox/extras/migrations/0107_alter_script_options.py +++ b/netbox/extras/migrations/0107_alter_script_options.py @@ -9,7 +9,7 @@ def update_scripts(apps, schema_editor): ScriptModuleNew = apps.get_model('extras', 'ScriptModule') Script = apps.get_model('extras', 'Script') ContentType = apps.get_model('contenttypes', 'ContentType') - ct = ContentType.objects.get(app_label='extras', model='script') + ct = ContentType.objects.filter(app_label='extras', model='script').first() for module in ScriptModule.objects.all(): for script in module.get_module_scripts.keys(): @@ -19,7 +19,8 @@ def update_scripts(apps, schema_editor): ) # update all jobs associated with this module/name to point to the new script obj - module.jobs.filter(name=script).update(object_type=ct, object_id=obj.id) + if ct: + module.jobs.filter(name=script).update(object_type=ct, object_id=obj.id) class Migration(migrations.Migration): From 0c214d37b31118a042f96fe1671f26220f379efb Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 8 Feb 2024 10:32:39 -0800 Subject: [PATCH 08/51] 14438 fix merge --- netbox/templates/extras/script_list.html | 124 +++++++++++------------ 1 file changed, 61 insertions(+), 63 deletions(-) diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html index b4c6a05198..251d4d43ca 100644 --- a/netbox/templates/extras/script_list.html +++ b/netbox/templates/extras/script_list.html @@ -44,75 +44,73 @@
    - {% with jobs=module.get_latest_jobs %} - {% for script_name, script in module.scripts.items %} - {% with last_job=jobs|get_key:script.class_name %} - + {% for script in module.scripts.all %} + {% with last_job=script.get_latest_jobs|get_key:script.name %} + + + {{ script.name }} + + {{ script.description|markdown|placeholder }} + {% if last_job %} - {{ script.name }} + {{ last_job.created|annotated_date }} - {{ script.description|markdown|placeholder }} - {% if last_job %} - - {{ last_job.created|annotated_date }} - - - {% badge last_job.get_status_display last_job.get_status_color %} - - {% else %} - {% trans "Never" %} - {{ ''|placeholder }} - {% endif %} - {% if perms.extras.run_script %} -
    -
    - {% csrf_token %} - -
    -
    - {% endif %} + {% badge last_job.get_status_display last_job.get_status_color %} - - {% if last_job %} - {% for test_name, data in last_job.data.tests.items %} - - - {{ test_name }} - - - {{ data.success }} - {{ data.info }} - {{ data.warning }} - {{ data.failure }} - - - {% endfor %} - {% elif not last_job.data.log %} - {# legacy #} - {% for method, stats in last_job.data.items %} - - - {{ method }} - - - {{ stats.success }} - {{ stats.info }} - {{ stats.warning }} - {{ stats.failure }} - - - {% endfor %} + {% else %} + {% trans "Never" %} + {{ ''|placeholder }} {% endif %} - {% endwith %} + + {% if perms.extras.run_script %} +
    +
    + {% csrf_token %} + +
    +
    + {% endif %} + + + {% if last_job %} + {% for test_name, data in last_job.data.tests.items %} + + + {{ test_name }} + + + {{ data.success }} + {{ data.info }} + {{ data.warning }} + {{ data.failure }} + + + {% endfor %} + {% elif not last_job.data.log %} + {# legacy #} + {% for method, stats in last_job.data.items %} + + + {{ method }} + + + {{ stats.success }} + {{ stats.info }} + {{ stats.warning }} + {{ stats.failure }} + + + {% endfor %} + {% endif %} + {% endwith %} {% endfor %} - {% endwith %} {% else %} From 41c792a3e53bb172ad73d63ad14a7efc1a391657 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 8 Feb 2024 10:33:20 -0800 Subject: [PATCH 09/51] 14438 fix merge --- netbox/templates/extras/script_list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html index 251d4d43ca..719ff60cbc 100644 --- a/netbox/templates/extras/script_list.html +++ b/netbox/templates/extras/script_list.html @@ -110,7 +110,7 @@
    {% endfor %} {% endif %} {% endwith %} - {% endfor %} + {% endfor %} {% else %} From 172d1b00cc7c00b1d34ef13a3ff501adb7d16661 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 8 Feb 2024 13:49:29 -0800 Subject: [PATCH 10/51] 14438 check valid script for views --- .../extras/migrations/0108_script_models.py | 7 +- netbox/extras/models/scripts.py | 44 ++++++- netbox/extras/views.py | 112 ++++++++++++------ 3 files changed, 121 insertions(+), 42 deletions(-) diff --git a/netbox/extras/migrations/0108_script_models.py b/netbox/extras/migrations/0108_script_models.py index e4385347ed..0098c17bf5 100644 --- a/netbox/extras/migrations/0108_script_models.py +++ b/netbox/extras/migrations/0108_script_models.py @@ -35,12 +35,17 @@ class Migration(migrations.Migration): fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), ('name', models.CharField(max_length=79)), - ('module', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='scripts', to='extras.scriptmodule')), + ('module', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, related_name='scripts', to='extras.scriptmodule')), ], options={ 'ordering': ('name', 'pk'), }, ), + migrations.AddField( + model_name='script', + name='is_valid', + field=models.BooleanField(default=True), + ), migrations.AddConstraint( model_name='script', constraint=models.UniqueConstraint(fields=('name', 'module'), name='extras_script_unique_name_module'), diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index 3be7d155e1..b4e7ac4fc9 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -29,9 +29,12 @@ class Script(EventRulesMixin, JobsMixin, models.Model): ) module = models.ForeignKey( to='extras.ScriptModule', - on_delete=models.PROTECT, + on_delete=models.RESTRICT, related_name='scripts' ) + is_valid = models.BooleanField( + default=True + ) def __str__(self): return self.name @@ -51,6 +54,14 @@ class Meta: def python_class(self): return self.module.get_module_scripts.get(self.name) + def delete_if_no_jobs(self): + if self.jobs.all(): + self.is_valid = False + self.save() + else: + self.delete() + self.id = None + class ScriptModuleManager(models.Manager.from_queryset(RestrictedQuerySet)): @@ -100,6 +111,35 @@ def _get_name(cls): return scripts + def sync_classes(self): + db_classes = {} + for obj in self.scripts.filter(module=self): + db_classes[obj.name] = obj + + db_classes_set = {k for k in db_classes.keys()} + + module_scripts = self.get_module_scripts + + module_classes_set = {k for k in module_scripts.keys()} + + # remove any existing db classes if they are no longer in the file + removed = db_classes_set - module_classes_set + for name in removed: + db_classes[name].delete_if_no_jobs() + + added = module_classes_set - db_classes_set + for name in added: + Script.objects.create( + module=self, + name=name, + is_valid=True, + ) + + def sync_data(self): + super().sync_data() + self.sync_classes() + def save(self, *args, **kwargs): self.file_root = ManagedFileRootPathChoices.SCRIPTS - return super().save(*args, **kwargs) + super().save(*args, **kwargs) + self.sync_classes() diff --git a/netbox/extras/views.py b/netbox/extras/views.py index fb057b8528..d00656c5e8 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -920,7 +920,7 @@ def post(self, request): widget = widget_class(**data) request.user.dashboard.add_widget(widget) request.user.dashboard.save() - messages.success(request, f'Added widget {widget.id}') + messages.success(request, _('Added widget: ') + str(widget.id)) return HttpResponse(headers={ 'HX-Redirect': reverse('home'), @@ -961,7 +961,7 @@ def post(self, request, id): data['config'] = config_form.cleaned_data request.user.dashboard.config[str(id)].update(data) request.user.dashboard.save() - messages.success(request, f'Updated widget {widget.id}') + messages.success(request, _('Updated widget: ') + str(widget.id)) return HttpResponse(headers={ 'HX-Redirect': reverse('home'), @@ -997,9 +997,9 @@ def post(self, request, id): if form.is_valid(): request.user.dashboard.delete_widget(id) request.user.dashboard.save() - messages.success(request, f'Deleted widget {id}') + messages.success(request, _('Deleted widget: ') + str(id)) else: - messages.error(request, f'Error deleting widget: {form.errors[0]}') + messages.error(request, _('Error deleting widget: ') + str(form.errors[0])) return redirect(reverse('home')) @@ -1042,19 +1042,45 @@ def get_script_module(module, request): return get_object_or_404(ScriptModule.objects.restrict(request.user), file_path__regex=f"^{module}\\.") -class ScriptView(ContentTypePermissionRequiredMixin, View): +class BaseScriptView(ContentTypePermissionRequiredMixin, View): + script = None + script_class = None + jobs = None + + def get_required_permission(self): + return 'extras.view_script' + + def get_script(self, request, pk): + self.script = Script.objects.get(pk=pk) + if self.script.python_class: + self.script_class = script.python_class() + else: + self.script.delete_if_no_jobs() + messages.error(request, _("Script class has been deleted.")) + if not self.script.id: + return redirect('extras:script_list') + else: + return redirect('extras:script_jobs', pk=self.script.id) + + self.jobs = self.script.jobs.all() + return None + + +class ScriptView(BaseScriptView): def get_required_permission(self): return 'extras.view_script' def get(self, request, pk): - script = Script.objects.get(pk=pk) - script_class = script.python_class() - jobs = script.jobs.all() - form = script_class.as_form(initial=normalize_querydict(request.GET)) + if ret := self.get_script(request, pk): + return ret + + form = None + if self.script_class: + form = script_class.as_form(initial=normalize_querydict(request.GET)) return render(request, 'extras/script.html', { - 'job_count': jobs.count(), + 'job_count': self.jobs.count(), 'module': script.module, 'script': script, 'script_class': script_class, @@ -1065,85 +1091,93 @@ def post(self, request, pk): if not request.user.has_perm('extras.run_script'): return HttpResponseForbidden() - script = Script.objects.get(pk=pk) - script_class = script.python_class() - jobs = script.jobs.all() - form = script_class.as_form(request.POST, request.FILES) + if ret := self.get_script(request, pk): + return ret + + form = None + if self.script_class: + form = script_class.as_form(request.POST, request.FILES) # Allow execution only if RQ worker process is running if not get_workers_for_queue('default'): - messages.error(request, "Unable to run script: RQ worker process not running.") + messages.error(request, _("Unable to run script: RQ worker process not running.")) elif form.is_valid(): job = Job.enqueue( run_script, - instance=script, - name=script_class.class_name, + instance=self.script, + name=self.script_class.class_name, user=request.user, schedule_at=form.cleaned_data.pop('_schedule_at'), interval=form.cleaned_data.pop('_interval'), data=form.cleaned_data, request=copy_safe_request(request), - job_timeout=script.python_class.job_timeout, + job_timeout=self.script.python_class.job_timeout, commit=form.cleaned_data.pop('_commit') ) return redirect('extras:script_result', job_pk=job.pk) return render(request, 'extras/script.html', { - 'job_count': jobs.count(), - 'module': script.module, - 'script': script, - 'script_class': script_class, + 'job_count': self.jobs.count(), + 'module': self.script.module, + 'script': self.script, + 'script_class': self.script_class, 'form': form, }) -class ScriptSourceView(ContentTypePermissionRequiredMixin, View): +class ScriptSourceView(BaseScriptView): def get_required_permission(self): return 'extras.view_script' def get(self, request, pk): - script = Script.objects.get(pk=pk) - script_class = script.python_class() - jobs = script.jobs.all() + if ret := self.get_script(request, pk): + return ret return render(request, 'extras/script/source.html', { - 'job_count': jobs.count(), - 'module': script.module, - 'script': script, - 'script_class': script_class, + 'job_count': self.jobs.count(), + 'module': self.script.module, + 'script': self.script, + 'script_class': self.script_class, 'tab': 'source', }) -class ScriptJobsView(ContentTypePermissionRequiredMixin, View): +class ScriptJobsView(BaseScriptView): def get_required_permission(self): return 'extras.view_script' def get(self, request, pk): - script = Script.objects.get(pk=pk) - script_class = script.python_class() - jobs = script.jobs.all() + self.script = Script.objects.get(pk=pk) + if self.script.python_class: + self.script_class = script.python_class() + else: + self.script.delete_if_no_jobs() + if not self.script.id: + messages.error(request, _("Script class has been deleted.")) + return redirect('extras:script_list') + + self.jobs = self.script.jobs.all() jobs_table = JobTable( - data=jobs, + data=self.jobs, orderable=False, user=request.user ) jobs_table.configure(request) return render(request, 'extras/script/jobs.html', { - 'job_count': jobs.count(), - 'module': script.module, - 'script': script, + 'job_count': self.jobs.count(), + 'module': self.script.module, + 'script': self.script, 'table': jobs_table, 'tab': 'jobs', }) -class ScriptResultView(ContentTypePermissionRequiredMixin, View): +class ScriptResultView(BaseScriptView): def get_required_permission(self): return 'extras.view_script' From c0a5de09a7511de85ece2ab53f258daf46c7ffcc Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 8 Feb 2024 14:06:32 -0800 Subject: [PATCH 11/51] 14438 check valid script for views --- netbox/templates/extras/script/base.html | 4 ++-- netbox/templates/extras/script/jobs.html | 13 +++++++++++++ netbox/templates/extras/script_list.html | 7 ++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/netbox/templates/extras/script/base.html b/netbox/templates/extras/script/base.html index 429db590e3..501daa752d 100644 --- a/netbox/templates/extras/script/base.html +++ b/netbox/templates/extras/script/base.html @@ -26,10 +26,10 @@ {% block tabs %}