| OS Releases Affected |
@@ -78,5 +79,6 @@
+{% endwith %}
{% endblock %}
diff --git a/errata/views.py b/errata/views.py
index 285f7483..21382eff 100644
--- a/errata/views.py
+++ b/errata/views.py
@@ -29,7 +29,7 @@
@login_required
def erratum_list(request):
- errata = Erratum.objects.select_related()
+ errata = Erratum.objects.all()
if 'e_type' in request.GET:
errata = errata.filter(e_type=request.GET['e_type']).distinct()
diff --git a/hosts/apps.py b/hosts/apps.py
index 84efc94f..a805f23e 100644
--- a/hosts/apps.py
+++ b/hosts/apps.py
@@ -19,3 +19,6 @@
class HostsConfig(AppConfig):
name = 'hosts'
+
+ def ready(self):
+ import hosts.signals # noqa
diff --git a/hosts/managers.py b/hosts/managers.py
index 85a489d4..73301b58 100644
--- a/hosts/managers.py
+++ b/hosts/managers.py
@@ -20,4 +20,4 @@
class HostManager(models.Manager):
def get_queryset(self):
- return super().get_queryset().select_related()
+ return super().get_queryset().select_related('osvariant', 'arch', 'domain')
diff --git a/hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py b/hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py
new file mode 100644
index 00000000..58e363d0
--- /dev/null
+++ b/hosts/migrations/0011_host_bug_updates_count_host_errata_count_and_more.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.28 on 2026-02-11 04:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('hosts', '0010_alter_hostrepo_options'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='host',
+ name='bug_updates_count',
+ field=models.PositiveIntegerField(db_index=True, default=0),
+ ),
+ migrations.AddField(
+ model_name='host',
+ name='errata_count',
+ field=models.PositiveIntegerField(db_index=True, default=0),
+ ),
+ migrations.AddField(
+ model_name='host',
+ name='packages_count',
+ field=models.PositiveIntegerField(db_index=True, default=0),
+ ),
+ migrations.AddField(
+ model_name='host',
+ name='sec_updates_count',
+ field=models.PositiveIntegerField(db_index=True, default=0),
+ ),
+ ]
diff --git a/hosts/migrations/0012_backfill_cached_counts.py b/hosts/migrations/0012_backfill_cached_counts.py
new file mode 100644
index 00000000..4e7b515d
--- /dev/null
+++ b/hosts/migrations/0012_backfill_cached_counts.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.28 on 2026-02-11
+
+from django.db import migrations
+
+
+def backfill_host_counts(apps, schema_editor):
+ """Backfill cached count fields for existing hosts."""
+ Host = apps.get_model('hosts', 'Host')
+ for host in Host.objects.all():
+ host.sec_updates_count = host.updates.filter(security=True).count()
+ host.bug_updates_count = host.updates.filter(security=False).count()
+ host.packages_count = host.packages.count()
+ host.errata_count = host.errata.count()
+ host.save(update_fields=[
+ 'sec_updates_count', 'bug_updates_count',
+ 'packages_count', 'errata_count'
+ ])
+
+
+def reverse_backfill(apps, schema_editor):
+ """No-op reverse - counts will be recalculated on next report."""
+ pass
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('hosts', '0011_host_bug_updates_count_host_errata_count_and_more'),
+ ]
+
+ operations = [
+ migrations.RunPython(backfill_host_counts, reverse_backfill),
+ ]
diff --git a/hosts/models.py b/hosts/models.py
index 7c646b98..96f163bf 100644
--- a/hosts/models.py
+++ b/hosts/models.py
@@ -60,6 +60,11 @@ class Host(models.Model):
tags = TaggableManager(blank=True)
updated_at = models.DateTimeField(default=timezone.now)
errata = models.ManyToManyField(Erratum, blank=True)
+ # Cached count fields for query optimization
+ sec_updates_count = models.PositiveIntegerField(default=0, db_index=True)
+ bug_updates_count = models.PositiveIntegerField(default=0, db_index=True)
+ packages_count = models.PositiveIntegerField(default=0, db_index=True)
+ errata_count = models.PositiveIntegerField(default=0, db_index=True)
from hosts.managers import HostManager
objects = HostManager()
@@ -97,20 +102,23 @@ def get_absolute_url(self):
return reverse('hosts:host_detail', args=[self.hostname])
def get_num_security_updates(self):
- return self.updates.filter(security=True).count()
+ return self.sec_updates_count
def get_num_bugfix_updates(self):
- return self.updates.filter(security=False).count()
+ return self.bug_updates_count
def get_num_updates(self):
- return self.updates.count()
+ return self.sec_updates_count + self.bug_updates_count
def get_num_packages(self):
- return self.packages.count()
+ return self.packages_count
def get_num_repos(self):
return self.repos.count()
+ def get_num_errata(self):
+ return self.errata_count
+
def check_rdns(self):
if self.check_dns:
update_rdns(self)
@@ -127,9 +135,9 @@ def clean_reports(self):
""" Remove all but the last 3 reports for a host
"""
from reports.models import Report
- reports = Report.objects.filter(host=self).order_by('-created')[3:]
- rlen = reports.count()
- for report in Report.objects.filter(host=self).order_by('-created')[3:]:
+ reports = list(Report.objects.filter(host=self).order_by('-created')[3:])
+ rlen = len(reports)
+ for report in reports:
report.delete()
if rlen > 0:
info_message(text=f'{self.hostname}: removed {rlen} old reports')
@@ -149,14 +157,14 @@ def get_host_repo_packages(self):
Q(mirror__repo__in=self.repos.all(),
mirror__enabled=True,
mirror__repo__enabled=True)
- return Package.objects.select_related().filter(hostrepos_q).distinct()
+ return Package.objects.select_related('name', 'arch').filter(hostrepos_q).distinct()
def process_update(self, package, highest_package):
if self.host_repos_only:
host_repos = Q(repo__host=self)
else:
host_repos = Q(repo__osrelease__osvariant__host=self, repo__arch=self.arch) | Q(repo__host=self)
- mirrors = highest_package.mirror_set.filter(host_repos)
+ mirrors = highest_package.mirror_set.filter(host_repos).select_related('repo')
security = False
# if any of the containing repos are security, mark the update as a security update
for mirror in mirrors:
@@ -215,7 +223,7 @@ def find_host_repo_updates(self, host_packages, repo_packages):
repo__mirror__refresh=True,
repo__mirror__repo__enabled=True,
host=self)
- hostrepos = HostRepo.objects.select_related().filter(hostrepos_q)
+ hostrepos = HostRepo.objects.select_related('host', 'repo').filter(hostrepos_q)
for package in host_packages:
highest_package = package
diff --git a/hosts/serializers.py b/hosts/serializers.py
index c0313e47..9565bae6 100644
--- a/hosts/serializers.py
+++ b/hosts/serializers.py
@@ -33,10 +33,10 @@ class Meta:
'updated_at', 'bugfix_update_count', 'security_update_count')
def get_bugfix_update_count(self, obj):
- return obj.updates.filter(security=False).count()
+ return obj.bug_updates_count
def get_security_update_count(self, obj):
- return obj.updates.filter(security=True).count()
+ return obj.sec_updates_count
class HostRepoSerializer(serializers.HyperlinkedModelSerializer):
diff --git a/hosts/signals.py b/hosts/signals.py
new file mode 100644
index 00000000..ca5de922
--- /dev/null
+++ b/hosts/signals.py
@@ -0,0 +1,45 @@
+# Copyright 2026 Marcus Furlong
+#
+# This file is part of Patchman.
+#
+# Patchman is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 only.
+#
+# Patchman is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchman. If not, see
+
+from django.db.models.signals import m2m_changed
+from django.dispatch import receiver
+
+from hosts.models import Host
+
+
+@receiver(m2m_changed, sender=Host.packages.through)
+def update_host_packages_count(sender, instance, action, **kwargs):
+ """Update packages_count when Host.packages M2M changes."""
+ if action in ('post_add', 'post_remove', 'post_clear'):
+ instance.packages_count = instance.packages.count()
+ instance.save(update_fields=['packages_count'])
+
+
+@receiver(m2m_changed, sender=Host.updates.through)
+def update_host_updates_count(sender, instance, action, **kwargs):
+ """Update sec_updates_count and bug_updates_count when Host.updates M2M changes."""
+ if action in ('post_add', 'post_remove', 'post_clear'):
+ instance.sec_updates_count = instance.updates.filter(security=True).count()
+ instance.bug_updates_count = instance.updates.filter(security=False).count()
+ instance.save(update_fields=['sec_updates_count', 'bug_updates_count'])
+
+
+@receiver(m2m_changed, sender=Host.errata.through)
+def update_host_errata_count(sender, instance, action, **kwargs):
+ """Update errata_count when Host.errata M2M changes."""
+ if action in ('post_add', 'post_remove', 'post_clear'):
+ instance.errata_count = instance.errata.count()
+ instance.save(update_fields=['errata_count'])
diff --git a/hosts/tables.py b/hosts/tables.py
index 715a29ae..b34b7d61 100644
--- a/hosts/tables.py
+++ b/hosts/tables.py
@@ -31,7 +31,7 @@
'{% endwith %}'
)
AFFECTED_ERRATA_TEMPLATE = (
- '{% with count=record.errata.count %}'
+ '{% with count=record.errata_count %}'
'{% if count != 0 %}'
'{{ count }}'
'{% else %}{% endif %}{% endwith %}'
diff --git a/hosts/tasks.py b/hosts/tasks.py
index f186760f..71ec203a 100755
--- a/hosts/tasks.py
+++ b/hosts/tasks.py
@@ -15,7 +15,6 @@
# along with Patchman. If not, see
from celery import shared_task
-from django.db.models import Count
from hosts.models import Host
from util import get_datetime_now
@@ -34,7 +33,7 @@ def find_host_updates(host_id):
def find_all_host_updates():
""" Task to find updates for all hosts
"""
- for host in Host.objects.all():
+ for host in Host.objects.all().iterator():
find_host_updates.delay(host.id)
@@ -42,39 +41,35 @@ def find_all_host_updates():
def find_all_host_updates_homogenous():
""" Task to find updates for all hosts where hosts are expected to be homogenous
"""
- updated_hosts = []
+ updated_host_ids = set()
ts = get_datetime_now()
- for host in Host.objects.all():
- if host not in updated_hosts:
+ for host in Host.objects.all().iterator():
+ if host.id not in updated_host_ids:
host.find_updates()
host.updated_at = ts
host.save()
# only include hosts with the exact same number of packages
- filtered_hosts = Host.objects.annotate(
- packages_count=Count('packages')).filter(
- packages_count=host.packages.count()
- )
+ filtered_hosts = Host.objects.filter(
+ packages_count=host.packages_count
+ )
# and exclude hosts with the current timestamp
filtered_hosts = filtered_hosts.exclude(updated_at=ts)
- packages = set(host.packages.all())
- repos = set(host.repos.all())
- updates = host.updates.all()
+ package_ids = frozenset(host.packages.values_list('id', flat=True))
+ repo_ids = frozenset(host.repos.values_list('id', flat=True))
+ updates = list(host.updates.all())
- phosts = []
- for fhost in filtered_hosts:
- frepos = set(fhost.repos.all())
- if repos != frepos:
+ for fhost in filtered_hosts.iterator():
+ frepo_ids = frozenset(fhost.repos.values_list('id', flat=True))
+ if repo_ids != frepo_ids:
continue
- fpackages = set(fhost.packages.all())
- if packages != fpackages:
+ fpackage_ids = frozenset(fhost.packages.values_list('id', flat=True))
+ if package_ids != fpackage_ids:
continue
- phosts.append(fhost)
- for phost in phosts:
- phost.updates.set(updates)
- phost.updated_at = ts
- phost.save()
- updated_hosts.append(phost)
- info_message(text=f'Added the same updates to {phost}')
+ fhost.updates.set(updates)
+ fhost.updated_at = ts
+ fhost.save()
+ updated_host_ids.add(fhost.id)
+ info_message(text=f'Added the same updates to {fhost}')
diff --git a/hosts/views.py b/hosts/views.py
index 259cef57..f940c3a4 100644
--- a/hosts/views.py
+++ b/hosts/views.py
@@ -15,9 +15,11 @@
# You should have received a copy of the GNU General Public License
# along with Patchman. If not, see
+from urllib.parse import parse_qs
+
from django.contrib import messages
from django.contrib.auth.decorators import login_required
-from django.db.models import Count, Q
+from django.db.models import Q
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django_filters import rest_framework as filters
@@ -31,6 +33,7 @@
from hosts.models import Host, HostRepo
from hosts.serializers import HostRepoSerializer, HostSerializer
from hosts.tables import HostTable
+from hosts.tasks import find_host_updates
from operatingsystems.models import OSRelease, OSVariant
from reports.models import Report
from util import sanitize_filter_params
@@ -39,10 +42,9 @@
def _get_filtered_hosts(filter_params):
"""Helper to reconstruct filtered queryset from filter params."""
- from urllib.parse import parse_qs
params = parse_qs(filter_params)
- hosts = Host.objects.select_related()
+ hosts = Host.objects.select_related('osvariant', 'arch', 'domain')
if 'domain_id' in params:
hosts = hosts.filter(domain=params['domain_id'][0])
@@ -76,12 +78,8 @@ def _get_filtered_hosts(filter_params):
@login_required
def host_list(request):
- hosts = Host.objects.select_related().annotate(
- sec_updates_count=Count('updates', filter=Q(updates__security=True), distinct=True),
- bug_updates_count=Count('updates', filter=Q(updates__security=False), distinct=True),
- errata_count=Count('errata', distinct=True),
- packages_count=Count('packages', distinct=True),
- )
+ # Use cached count fields instead of expensive annotations
+ hosts = Host.objects.select_related('osvariant', 'arch', 'domain')
if 'domain_id' in request.GET:
hosts = hosts.filter(domain=request.GET['domain_id'])
@@ -160,7 +158,7 @@ def host_detail(request, hostname):
hostrepos = HostRepo.objects.filter(host=host)
# Build packages list with update info
- updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related()}
+ updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related('oldpackage', 'newpackage')}
packages_with_updates = []
for package in host.packages.select_related('name', 'arch').order_by('name__name'):
package.update = updates_by_package.get(package.id)
@@ -226,7 +224,6 @@ def host_delete(request, hostname):
def host_find_updates(request, hostname):
""" Find updates using a celery task
"""
- from hosts.tasks import find_host_updates
host = get_object_or_404(Host, hostname=hostname)
find_host_updates.delay(host.id)
text = f'Finding updates for Host {host}'
@@ -265,7 +262,6 @@ def host_bulk_action(request):
name = Host._meta.verbose_name if count == 1 else Host._meta.verbose_name_plural
if action == 'find_updates':
- from hosts.tasks import find_host_updates
for host in hosts:
find_host_updates.delay(host.id)
messages.success(request, f'Queued {count} {name} for update check')
@@ -298,7 +294,7 @@ class HostViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows hosts to be viewed or edited.
"""
- queryset = Host.objects.all()
+ queryset = Host.objects.select_related('osvariant', 'arch', 'domain').all()
serializer_class = HostSerializer
filterset_class = HostFilter
@@ -307,5 +303,5 @@ class HostRepoViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows host repos to be viewed or edited.
"""
- queryset = HostRepo.objects.all()
+ queryset = HostRepo.objects.select_related('host', 'repo').all()
serializer_class = HostRepoSerializer
diff --git a/modules/managers.py b/modules/managers.py
index ebf7fe31..ca0a6dee 100644
--- a/modules/managers.py
+++ b/modules/managers.py
@@ -19,4 +19,4 @@
class ModuleManager(models.Manager):
def get_queryset(self):
- return super().get_queryset().select_related()
+ return super().get_queryset().select_related('arch', 'repo')
diff --git a/modules/views.py b/modules/views.py
index 45703f29..bde0b741 100644
--- a/modules/views.py
+++ b/modules/views.py
@@ -28,7 +28,7 @@
@login_required
def module_list(request):
- modules = Module.objects.select_related()
+ modules = Module.objects.select_related('arch', 'repo')
if 'search' in request.GET:
terms = request.GET['search'].lower()
@@ -62,6 +62,6 @@ class ModuleViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows modules to be viewed or edited.
"""
- queryset = Module.objects.all()
+ queryset = Module.objects.select_related('arch', 'repo').all()
serializer_class = ModuleSerializer
permission_classes = (permissions.IsAuthenticatedOrReadOnly,)
diff --git a/operatingsystems/apps.py b/operatingsystems/apps.py
index 4a7f65f7..b0467e42 100644
--- a/operatingsystems/apps.py
+++ b/operatingsystems/apps.py
@@ -19,3 +19,6 @@
class OperatingsystemsConfig(AppConfig):
name = 'operatingsystems'
+
+ def ready(self):
+ import operatingsystems.signals # noqa
diff --git a/operatingsystems/forms.py b/operatingsystems/forms.py
index fa319182..5d063b42 100644
--- a/operatingsystems/forms.py
+++ b/operatingsystems/forms.py
@@ -47,7 +47,7 @@ class Meta:
class AddReposToOSReleaseForm(ModelForm):
repos = ModelMultipleChoiceField(
- queryset=Repository.objects.select_related(),
+ queryset=Repository.objects.select_related('arch'),
required=False,
label=None,
widget=FilteredSelectMultiple('Repos', False, attrs={'size': '30'}))
diff --git a/operatingsystems/migrations/0009_osvariant_hosts_count.py b/operatingsystems/migrations/0009_osvariant_hosts_count.py
new file mode 100644
index 00000000..e388f1ad
--- /dev/null
+++ b/operatingsystems/migrations/0009_osvariant_hosts_count.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.28 on 2026-02-11 04:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('operatingsystems', '0008_alter_osrelease_options_alter_osvariant_options'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='osvariant',
+ name='hosts_count',
+ field=models.PositiveIntegerField(db_index=True, default=0),
+ ),
+ ]
diff --git a/operatingsystems/migrations/0010_backfill_osvariant_counts.py b/operatingsystems/migrations/0010_backfill_osvariant_counts.py
new file mode 100644
index 00000000..a14fc90d
--- /dev/null
+++ b/operatingsystems/migrations/0010_backfill_osvariant_counts.py
@@ -0,0 +1,29 @@
+# Generated by Django 4.2.28 on 2026-02-11
+
+from django.db import migrations
+
+
+def backfill_osvariant_counts(apps, schema_editor):
+ """Backfill hosts_count for existing osvariants."""
+ OSVariant = apps.get_model('operatingsystems', 'OSVariant')
+ Host = apps.get_model('hosts', 'Host')
+ for osvariant in OSVariant.objects.all():
+ osvariant.hosts_count = Host.objects.filter(osvariant=osvariant).count()
+ osvariant.save(update_fields=['hosts_count'])
+
+
+def reverse_backfill(apps, schema_editor):
+ """No-op reverse."""
+ pass
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('operatingsystems', '0009_osvariant_hosts_count'),
+ ('hosts', '0012_backfill_cached_counts'),
+ ]
+
+ operations = [
+ migrations.RunPython(backfill_osvariant_counts, reverse_backfill),
+ ]
diff --git a/operatingsystems/models.py b/operatingsystems/models.py
index 234b8ab3..911332a9 100644
--- a/operatingsystems/models.py
+++ b/operatingsystems/models.py
@@ -57,6 +57,8 @@ class OSVariant(models.Model):
arch = models.ForeignKey(MachineArchitecture, blank=True, null=True, on_delete=models.CASCADE)
osrelease = models.ForeignKey(OSRelease, blank=True, null=True, on_delete=models.SET_NULL)
codename = models.CharField(max_length=255, blank=True)
+ # Cached count field for query optimization
+ hosts_count = models.PositiveIntegerField(default=0, db_index=True)
class Meta:
verbose_name = 'Operating System Variant'
diff --git a/operatingsystems/signals.py b/operatingsystems/signals.py
new file mode 100644
index 00000000..5f1d0507
--- /dev/null
+++ b/operatingsystems/signals.py
@@ -0,0 +1,56 @@
+# Copyright 2026 Marcus Furlong
+#
+# This file is part of Patchman.
+#
+# Patchman is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 only.
+#
+# Patchman is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchman. If not, see
+
+from django.db.models.signals import post_delete, post_save, pre_save
+from django.dispatch import receiver
+
+from hosts.models import Host
+
+
+@receiver(pre_save, sender=Host)
+def track_osvariant_change(sender, instance, **kwargs):
+ """Track old osvariant before save to update its count."""
+ if instance.pk:
+ try:
+ old_instance = Host.objects.get(pk=instance.pk)
+ instance._old_osvariant = old_instance.osvariant
+ except Host.DoesNotExist:
+ instance._old_osvariant = None
+ else:
+ instance._old_osvariant = None
+
+
+@receiver(post_save, sender=Host)
+def update_osvariant_count_on_save(sender, instance, created, **kwargs):
+ """Update OSVariant.hosts_count when Host is created or osvariant changes."""
+ # Update new osvariant count
+ if instance.osvariant:
+ instance.osvariant.hosts_count = Host.objects.filter(osvariant=instance.osvariant).count()
+ instance.osvariant.save(update_fields=['hosts_count'])
+
+ # Update old osvariant count if it changed
+ old_osvariant = getattr(instance, '_old_osvariant', None)
+ if old_osvariant and old_osvariant != instance.osvariant:
+ old_osvariant.hosts_count = Host.objects.filter(osvariant=old_osvariant).count()
+ old_osvariant.save(update_fields=['hosts_count'])
+
+
+@receiver(post_delete, sender=Host)
+def update_osvariant_count_on_delete(sender, instance, **kwargs):
+ """Update OSVariant.hosts_count when Host is deleted."""
+ if instance.osvariant:
+ instance.osvariant.hosts_count = Host.objects.filter(osvariant=instance.osvariant).count()
+ instance.osvariant.save(update_fields=['hosts_count'])
diff --git a/operatingsystems/templates/operatingsystems/osrelease_detail.html b/operatingsystems/templates/operatingsystems/osrelease_detail.html
index 740b9c4b..be94f43d 100644
--- a/operatingsystems/templates/operatingsystems/osrelease_detail.html
+++ b/operatingsystems/templates/operatingsystems/osrelease_detail.html
@@ -12,6 +12,7 @@
{% block content %}
+{% with osvariant_count=osrelease.osvariant_set.count repos_count=osrelease.repos.count %}
- Details
- Variants
@@ -26,8 +27,8 @@
| Name | {{ osrelease.name }} |
|---|
| CPE Name | {% if osrelease.cpe_name %}{{ osrelease.cpe_name }}{% endif %} |
|---|
| Codename | {% if osrelease.codename %}{{ osrelease.codename }}{% endif %} |
|---|
- | OS Variants | {{ osrelease.osvariant_set.count }} |
|---|
- | Repositories | {{ osrelease.repos.count }} |
|---|
+ | OS Variants | {{ osvariant_count }} |
|---|
+ | Repositories | {{ repos_count }} |
|---|
| Hosts | {{ host_count }} |
|---|
| Errata | {{ osrelease.erratum_set.count }} |
|---|
@@ -40,7 +41,7 @@
- {% if osrelease.osvariant_set.count == 0 %}
+ {% if osvariant_count == 0 %}
{{ osrelease }} has no Variants
{% else %}
{% gen_table osrelease.osvariant_set.select_related %}
@@ -50,7 +51,7 @@
- {% if osrelease.repos.count == 0 %}
+ {% if repos_count == 0 %}
{{ osrelease }} has no Repositories
{% else %}
{% gen_table osrelease.repos.select_related %}
@@ -68,5 +69,6 @@
+{% endwith %}
{% endblock %}
diff --git a/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html b/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html
index f6d0373d..6887e0a7 100644
--- a/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html
+++ b/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html
@@ -24,7 +24,7 @@
{% for osvariant in osvariants %}
| {{ osvariant }} |
- {% if osvariant.host_set.count != None %} {{ osvariant.host_set.count }} {% else %} 0 {% endif %} |
+ {{ osvariant.host_set.count|default:0 }} |
{% if osvariant.osrelease != None %} {{ osvariant.osrelease }} {% else %}No OS Release{% endif %} |
{% endfor %}
diff --git a/operatingsystems/templates/operatingsystems/osvariant_detail.html b/operatingsystems/templates/operatingsystems/osvariant_detail.html
index 0c1d306b..d4f011c6 100644
--- a/operatingsystems/templates/operatingsystems/osvariant_detail.html
+++ b/operatingsystems/templates/operatingsystems/osvariant_detail.html
@@ -24,7 +24,7 @@
| Name | {{ osvariant.name }} |
|---|
| Architecture | {{ osvariant.arch }} |
|---|
| Codename | {{ osvariant.codename }} |
|---|
- | Hosts | {{ osvariant.host_set.count }} |
|---|
+ | Hosts | {{ osvariant.hosts_count }} |
|---|
| OS Release | {% if osvariant.osrelease != None %} {{ osvariant.osrelease }} {% else %}No OS Release{% endif %} |
|---|
{% if user.is_authenticated and perms.is_admin %}
diff --git a/operatingsystems/views.py b/operatingsystems/views.py
index 7dd4fcb5..1a6b3fda 100644
--- a/operatingsystems/views.py
+++ b/operatingsystems/views.py
@@ -15,6 +15,8 @@
# You should have received a copy of the GNU General Public License
# along with Patchman. If not, see
+from urllib.parse import parse_qs
+
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.db.models import Count, Q
@@ -37,10 +39,9 @@
def _get_filtered_osvariants(filter_params):
"""Helper to reconstruct filtered queryset from filter params."""
- from urllib.parse import parse_qs
params = parse_qs(filter_params)
- osvariants = OSVariant.objects.select_related()
+ osvariants = OSVariant.objects.select_related('osrelease', 'arch')
if 'osrelease_id' in params:
osvariants = osvariants.filter(osrelease=params['osrelease_id'][0])
@@ -57,10 +58,9 @@ def _get_filtered_osvariants(filter_params):
def _get_filtered_osreleases(filter_params):
"""Helper to reconstruct filtered queryset from filter params."""
- from urllib.parse import parse_qs
params = parse_qs(filter_params)
- osreleases = OSRelease.objects.select_related()
+ osreleases = OSRelease.objects.all()
if 'erratum_id' in params:
osreleases = osreleases.filter(erratum=params['erratum_id'][0])
@@ -77,8 +77,8 @@ def _get_filtered_osreleases(filter_params):
@login_required
def osvariant_list(request):
- osvariants = OSVariant.objects.select_related().annotate(
- hosts_count=Count('host'),
+ # Use cached hosts_count instead of expensive annotation
+ osvariants = OSVariant.objects.select_related('osrelease', 'arch').annotate(
repos_count=Count('osrelease__repos'),
)
@@ -182,7 +182,7 @@ def delete_nohost_osvariants(request):
@login_required
def osrelease_list(request):
- osreleases = OSRelease.objects.select_related()
+ osreleases = OSRelease.objects.all()
if 'erratum_id' in request.GET:
osreleases = osreleases.filter(erratum=request.GET['erratum_id'])
@@ -347,7 +347,7 @@ class OSVariantViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows operating system variants to be viewed or edited.
"""
- queryset = OSVariant.objects.all()
+ queryset = OSVariant.objects.select_related('osrelease', 'arch').all()
serializer_class = OSVariantSerializer
filterset_fields = ['name']
diff --git a/packages/managers.py b/packages/managers.py
index c268f5cf..3a64e7ab 100644
--- a/packages/managers.py
+++ b/packages/managers.py
@@ -20,4 +20,4 @@
class PackageManager(models.Manager):
def get_queryset(self):
- return super().get_queryset().select_related()
+ return super().get_queryset().select_related('name', 'arch')
diff --git a/packages/utils.py b/packages/utils.py
index 87395ff6..c7f86b4c 100644
--- a/packages/utils.py
+++ b/packages/utils.py
@@ -19,6 +19,7 @@
from django.core.exceptions import MultipleObjectsReturned
from django.db import IntegrityError, transaction
+from django.db.models import Count, Min
from arch.models import PackageArchitecture
from packages.models import (
@@ -279,21 +280,30 @@ def get_matching_packages(name, epoch, version, release, p_type, arch=None):
def clean_packageupdates():
""" Removes PackageUpdate objects that are no longer linked to any hosts
"""
- package_updates = list(PackageUpdate.objects.all())
- for update in package_updates:
- if update.host_set.count() == 0:
- text = f'Removing unused PackageUpdate {update}'
+ orphaned = PackageUpdate.objects.filter(host__isnull=True)
+ for update in orphaned:
+ text = f'Removing unused PackageUpdate {update}'
+ info_message(text=text)
+ update.delete()
+
+ duplicate_updates = PackageUpdate.objects.values(
+ 'oldpackage', 'newpackage', 'security'
+ ).annotate(count=Count('id'), keep_id=Min('id')).filter(count__gt=1)
+
+ for update in duplicate_updates:
+ extra_updates = PackageUpdate.objects.filter(
+ oldpackage=update['oldpackage'],
+ newpackage=update['newpackage'],
+ security=update['security']
+ ).exclude(id=update['keep_id'])
+ keep_update = PackageUpdate.objects.get(id=update['keep_id'])
+ for extra_update in extra_updates:
+ text = f'Removing duplicate PackageUpdate: {extra_update}'
info_message(text=text)
- update.delete()
- for duplicate in package_updates:
- if update.oldpackage == duplicate.oldpackage and update.newpackage == duplicate.newpackage and \
- update.security == duplicate.security and update.id != duplicate.id:
- text = f'Removing duplicate PackageUpdate: {update}'
- info_message(text=text)
- for host in duplicate.host_set.all():
- host.updates.remove(duplicate)
- host.updates.add(update)
- duplicate.delete()
+ for host in extra_update.host_set.all():
+ host.updates.remove(extra_update)
+ host.updates.add(keep_update)
+ extra_update.delete()
def clean_packages(remove_duplicates=False):
@@ -315,21 +325,23 @@ def clean_packages(remove_duplicates=False):
packages.delete()
if remove_duplicates:
info_message(text='Checking for duplicate Packages...')
- for package in Package.objects.all():
- potential_duplicates = Package.objects.filter(
- name=package.name,
- arch=package.arch,
- epoch=package.epoch,
- version=package.version,
- release=package.release,
- packagetype=package.packagetype,
- category=package.category,
- )
- if potential_duplicates.count() > 1:
- for dupe in potential_duplicates:
- if dupe.id != package.id:
- info_message(text=f'Removing duplicate Package {dupe}')
- dupe.delete()
+ duplicates = Package.objects.values(
+ 'name', 'arch', 'epoch', 'version', 'release', 'packagetype', 'category'
+ ).annotate(count=Count('id'), keep_id=Min('id')).filter(count__gt=1)
+
+ for dup in duplicates:
+ to_delete = Package.objects.filter(
+ name=dup['name'],
+ arch=dup['arch'],
+ epoch=dup['epoch'],
+ version=dup['version'],
+ release=dup['release'],
+ packagetype=dup['packagetype'],
+ category=dup['category']
+ ).exclude(id=dup['keep_id'])
+ for package in to_delete:
+ info_message(text=f'Removing duplicate Package {package}')
+ package.delete()
def clean_packagenames():
diff --git a/packages/views.py b/packages/views.py
index 287c033d..9f75b415 100644
--- a/packages/views.py
+++ b/packages/views.py
@@ -32,7 +32,7 @@
@login_required
def package_list(request):
- packages = Package.objects.select_related()
+ packages = Package.objects.select_related('name', 'arch')
if 'arch_id' in request.GET:
packages = packages.filter(arch=request.GET['arch_id']).distinct()
@@ -121,7 +121,7 @@ def package_list(request):
@login_required
def package_name_list(request):
- packages = PackageName.objects.select_related()
+ packages = PackageName.objects.all()
if 'arch_id' in request.GET:
packages = packages.filter(package__arch=request.GET['arch_id']).distinct()
@@ -165,7 +165,7 @@ def package_detail(request, package_id):
@login_required
def package_name_detail(request, packagename):
package = get_object_or_404(PackageName, name=packagename)
- allversions = Package.objects.select_related().filter(name=package.id)
+ allversions = Package.objects.select_related('name', 'arch').filter(name=package.id)
return render(request,
'packages/package_name_detail.html',
{'package': package,
@@ -185,7 +185,7 @@ class PackageViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows packages to be viewed or edited.
"""
- queryset = Package.objects.all()
+ queryset = Package.objects.select_related('name', 'arch').all()
serializer_class = PackageSerializer
filterset_fields = [
'name',
@@ -201,6 +201,6 @@ class PackageUpdateViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows packages updates to be viewed or edited.
"""
- queryset = PackageUpdate.objects.all()
+ queryset = PackageUpdate.objects.select_related('oldpackage', 'newpackage').all()
serializer_class = PackageUpdateSerializer
filterset_fields = ['oldpackage', 'newpackage', 'security']
diff --git a/reports/views.py b/reports/views.py
index 18f60aa6..e5d1d438 100644
--- a/reports/views.py
+++ b/reports/views.py
@@ -16,8 +16,9 @@
# along with Patchman. If not, see
import json
-from urllib.parse import unquote
+from urllib.parse import parse_qs, unquote
+from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.db.models import Q
@@ -28,24 +29,29 @@
from django.views.decorators.csrf import csrf_exempt
from django_tables2 import RequestConfig
from rest_framework import status, viewsets
+from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly
from rest_framework.response import Response
+from rest_framework_api_key.permissions import HasAPIKey
from tenacity import (
retry, retry_if_exception_type, stop_after_attempt, wait_exponential,
)
from reports.models import Report
-from reports.serializers import ReportUploadSerializer
-from reports.tables import ReportTable
+from reports.serializers import ReportSerializer, ReportUploadSerializer
+from reports.tables import (
+ ReportModuleTable, ReportPackageTable, ReportRepoTable, ReportTable,
+ ReportUpdateTable,
+)
+from reports.tasks import process_report
from util import sanitize_filter_params
from util.filterspecs import Filter, FilterBar
def _get_filtered_reports(filter_params):
"""Helper to reconstruct filtered queryset from filter params."""
- from urllib.parse import parse_qs
params = parse_qs(filter_params)
- reports = Report.objects.select_related()
+ reports = Report.objects.all()
if 'host_id' in params:
reports = reports.filter(hostname=params['host_id'][0])
@@ -78,7 +84,6 @@ def upload(request):
report = Report.objects.create()
report.parse(data, meta)
- from reports.tasks import process_report
process_report.delay(report.id)
if 'report' in data and data['report'] == 'true':
@@ -108,7 +113,7 @@ def upload(request):
@login_required
def report_list(request):
- reports = Report.objects.select_related()
+ reports = Report.objects.all()
if 'host_id' in request.GET:
reports = reports.filter(hostname=request.GET['host_id'])
@@ -159,10 +164,6 @@ def report_detail(request, report_id):
# Add tables for Protocol 2 reports
if report.protocol == '2':
- from reports.tables import (
- ReportModuleTable, ReportPackageTable, ReportRepoTable,
- ReportUpdateTable,
- )
if report.has_packages:
context['packages_table'] = ReportPackageTable(report.packages_parsed)
if report.has_repos:
@@ -183,7 +184,6 @@ def report_detail(request, report_id):
def report_process(request, report_id):
""" Process a report using a celery task
"""
- from reports.tasks import process_report
report = get_object_or_404(Report, id=report_id)
report.processed = False
report.save()
@@ -243,7 +243,6 @@ def report_bulk_action(request):
name = Report._meta.verbose_name if count == 1 else Report._meta.verbose_name_plural
if action == 'process':
- from reports.tasks import process_report
for report in reports:
report.processed = False
report.save()
@@ -273,12 +272,6 @@ class ReportViewSet(viewsets.ViewSet):
"""
def get_permissions(self):
- from django.conf import settings
- from rest_framework.permissions import (
- AllowAny, IsAuthenticatedOrReadOnly,
- )
- from rest_framework_api_key.permissions import HasAPIKey
-
# POST requires API key if configured, otherwise allow any
if self.action == 'create':
if getattr(settings, 'REQUIRE_API_KEY', False):
@@ -289,16 +282,12 @@ def get_permissions(self):
def list(self, request):
"""List all reports."""
- from reports.serializers import ReportSerializer
queryset = Report.objects.all().order_by('-created')
serializer = ReportSerializer(queryset, many=True, context={'request': request})
return Response(serializer.data)
def retrieve(self, request, pk=None):
"""Retrieve a single report."""
- from django.shortcuts import get_object_or_404
-
- from reports.serializers import ReportSerializer
report = get_object_or_404(Report, pk=pk)
serializer = ReportSerializer(report, context={'request': request})
return Response(serializer.data)
@@ -357,7 +346,6 @@ def create(self, request):
)
# Queue for async processing
- from reports.tasks import process_report
process_report.delay(report.id)
return Response(
diff --git a/repos/apps.py b/repos/apps.py
index 4f62987a..d0cf38ed 100644
--- a/repos/apps.py
+++ b/repos/apps.py
@@ -19,3 +19,6 @@
class ReposConfig(AppConfig):
name = 'repos'
+
+ def ready(self):
+ import repos.signals # noqa
diff --git a/repos/forms.py b/repos/forms.py
index f9795b51..57ebae2a 100644
--- a/repos/forms.py
+++ b/repos/forms.py
@@ -28,7 +28,7 @@ class MirrorSelect2Widget(ModelSelect2MultipleWidget):
model = Mirror
search_fields = ['url__icontains', 'repo__name__icontains']
max_results = 50
- queryset = Mirror.objects.select_related().order_by('repo__name', 'url')
+ queryset = Mirror.objects.select_related('repo').order_by('repo__name', 'url')
def __init__(self, *args, **kwargs):
kwargs.setdefault('attrs', {})
@@ -41,7 +41,7 @@ def label_from_instance(self, obj):
class EditRepoForm(ModelForm):
mirrors = ModelMultipleChoiceField(
- queryset=Mirror.objects.select_related().order_by('repo__name', 'url'),
+ queryset=Mirror.objects.select_related('repo').order_by('repo__name', 'url'),
required=False,
widget=MirrorSelect2Widget(attrs={'style': 'width: 100%'}),
)
diff --git a/repos/managers.py b/repos/managers.py
index 78f37a46..739c7ca3 100644
--- a/repos/managers.py
+++ b/repos/managers.py
@@ -20,4 +20,4 @@
class RepositoryManager(models.Manager):
def get_queryset(self):
- return super().get_queryset().select_related()
+ return super().get_queryset().select_related('arch')
diff --git a/repos/migrations/0008_mirror_packages_count.py b/repos/migrations/0008_mirror_packages_count.py
new file mode 100644
index 00000000..41c99fe8
--- /dev/null
+++ b/repos/migrations/0008_mirror_packages_count.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.28 on 2026-02-11 04:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('repos', '0007_alter_mirror_options_alter_mirrorpackage_options_and_more'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='mirror',
+ name='packages_count',
+ field=models.PositiveIntegerField(db_index=True, default=0),
+ ),
+ ]
diff --git a/repos/migrations/0009_backfill_mirror_counts.py b/repos/migrations/0009_backfill_mirror_counts.py
new file mode 100644
index 00000000..244bf64b
--- /dev/null
+++ b/repos/migrations/0009_backfill_mirror_counts.py
@@ -0,0 +1,27 @@
+# Generated by Django 4.2.28 on 2026-02-11
+
+from django.db import migrations
+
+
+def backfill_mirror_counts(apps, schema_editor):
+ """Backfill packages_count for existing mirrors."""
+ Mirror = apps.get_model('repos', 'Mirror')
+ for mirror in Mirror.objects.all():
+ mirror.packages_count = mirror.packages.count()
+ mirror.save(update_fields=['packages_count'])
+
+
+def reverse_backfill(apps, schema_editor):
+ """No-op reverse."""
+ pass
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('repos', '0008_mirror_packages_count'),
+ ]
+
+ operations = [
+ migrations.RunPython(backfill_mirror_counts, reverse_backfill),
+ ]
diff --git a/repos/models.py b/repos/models.py
index 8af9377a..eec5f566 100644
--- a/repos/models.py
+++ b/repos/models.py
@@ -82,11 +82,11 @@ def refresh(self, force=False):
force can be set to force a reset of all the mirrors metadata
"""
if force:
- for mirror in self.mirror_set.all():
- mirror.packages_checksum = None
- mirror.modules_checksum = None
- mirror.errata_checksum = None
- mirror.save()
+ self.mirror_set.all().update(
+ packages_checksum=None,
+ modules_checksum=None,
+ errata_checksum=None
+ )
if not self.auth_required:
if self.repotype == Repository.DEB:
@@ -108,9 +108,7 @@ def refresh_errata(self, force=False):
""" Refresh errata metadata for all of a repos mirrors
"""
if force:
- for mirror in self.mirror_set.all():
- mirror.errata_checksum = None
- mirror.save()
+ self.mirror_set.all().update(errata_checksum=None)
if self.repotype == Repository.RPM:
refresh_repo_errata(self)
@@ -120,10 +118,7 @@ def disable(self):
each mirror so that it doesn't try to update its package metadata.
"""
self.enabled = False
- for mirror in self.mirror_set.all():
- mirror.enabled = False
- mirror.refresh = False
- mirror.save()
+ self.mirror_set.all().update(enabled=False, refresh=False)
def enable(self):
""" Enable a repo. This involves enabling each mirror, which allows it
@@ -131,10 +126,7 @@ def enable(self):
mirror so that it updates its package metadata.
"""
self.enabled = True
- for mirror in self.mirror_set.all():
- mirror.enabled = True
- mirror.refresh = True
- mirror.save()
+ self.mirror_set.all().update(enabled=True, refresh=True)
class Mirror(models.Model):
@@ -151,6 +143,8 @@ class Mirror(models.Model):
enabled = models.BooleanField(default=True)
refresh = models.BooleanField(default=True)
fail_count = models.IntegerField(default=0)
+ # Cached count field for query optimization
+ packages_count = models.PositiveIntegerField(default=0, db_index=True)
class Meta:
verbose_name_plural = 'Mirror'
diff --git a/repos/signals.py b/repos/signals.py
new file mode 100644
index 00000000..aa014736
--- /dev/null
+++ b/repos/signals.py
@@ -0,0 +1,28 @@
+# Copyright 2026 Marcus Furlong
+#
+# This file is part of Patchman.
+#
+# Patchman is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 only.
+#
+# Patchman is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchman. If not, see
+
+from django.db.models.signals import m2m_changed
+from django.dispatch import receiver
+
+from repos.models import Mirror
+
+
+@receiver(m2m_changed, sender=Mirror.packages.through)
+def update_mirror_packages_count(sender, instance, action, **kwargs):
+ """Update packages_count when Mirror.packages M2M changes."""
+ if action in ('post_add', 'post_remove', 'post_clear'):
+ instance.packages_count = instance.packages.count()
+ instance.save(update_fields=['packages_count'])
diff --git a/repos/tables.py b/repos/tables.py
index cff76c08..a0e3aca0 100644
--- a/repos/tables.py
+++ b/repos/tables.py
@@ -36,7 +36,7 @@
MIRROR_PACKAGES_TEMPLATE = (
'{% if not record.mirrorlist %}'
''
- '{{ record.packages.count }}{% endif %}'
+ '{{ record.packages_count }}{% endif %}'
)
MIRROR_ENABLED_TEMPLATE = '{% load common %}{% yes_no_img record.enabled %}'
REFRESH_TEMPLATE = '{% load common %}{% yes_no_img record.refresh %}'
diff --git a/repos/utils.py b/repos/utils.py
index 13cee149..0d81eb25 100644
--- a/repos/utils.py
+++ b/repos/utils.py
@@ -273,11 +273,12 @@ def find_best_repo(package, hostrepos):
repo. Returns the best repo.
"""
best_repo = None
- package_repos = hostrepos.filter(repo__mirror__packages=package).distinct()
+ package_repos = hostrepos.filter(repo__mirror__packages=package).select_related('repo').distinct()
+ package_repos = list(package_repos)
if package_repos:
best_repo = package_repos[0]
- if package_repos.count() > 1:
+ if len(package_repos) > 1:
for hostrepo in package_repos:
if hostrepo.repo.security:
best_repo = hostrepo
diff --git a/repos/views.py b/repos/views.py
index 2a45b668..e49c4972 100644
--- a/repos/views.py
+++ b/repos/views.py
@@ -15,10 +15,12 @@
# You should have received a copy of the GNU General Public License
# along with Patchman. If not, see
+from urllib.parse import parse_qs
+
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.db import IntegrityError
-from django.db.models import Count, Q
+from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
@@ -36,6 +38,7 @@
MirrorPackageSerializer, MirrorSerializer, RepositorySerializer,
)
from repos.tables import MirrorTable, RepositoryTable
+from repos.tasks import refresh_repo
from util import sanitize_filter_params
from util.filterspecs import Filter, FilterBar
@@ -43,7 +46,7 @@
@login_required
def repo_list(request):
- repos = Repository.objects.select_related().order_by('name')
+ repos = Repository.objects.select_related('arch').order_by('name')
if 'repotype' in request.GET:
repos = repos.filter(repotype=request.GET['repotype'])
@@ -146,12 +149,11 @@ def move_mirrors(repo):
hostrepo.delete()
mirror.repo = repo
mirror.save()
- if oldrepo.mirror_set.count() == 0:
+ if not oldrepo.mirror_set.exists():
oldrepo.delete()
- mirrors = Mirror.objects.select_related().annotate(
- packages_count=Count('packages'),
- ).order_by('packages_checksum')
+ # Use cached packages_count instead of expensive annotation
+ mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum')
checksum = None
if 'checksum' in request.GET:
@@ -319,9 +321,7 @@ def repo_edit(request, repo_id):
repo = edit_form.save()
repo.save()
mirrors = edit_form.cleaned_data['mirrors']
- for mirror in mirrors:
- mirror.repo = repo
- mirror.save()
+ mirrors.update(repo=repo)
if repo.enabled:
repo.enable()
else:
@@ -406,7 +406,6 @@ def repo_toggle_security(request, repo_id):
def repo_refresh(request, repo_id):
""" Refresh a repo using a celery task
"""
- from repos.tasks import refresh_repo
repo = get_object_or_404(Repository, id=repo_id)
refresh_repo.delay(repo.id)
text = f'Repostory {repo} is being refreshed'
@@ -416,10 +415,9 @@ def repo_refresh(request, repo_id):
def _get_filtered_repos(filter_params):
"""Helper to reconstruct filtered queryset from filter params."""
- from urllib.parse import parse_qs
params = parse_qs(filter_params)
- repos = Repository.objects.select_related().order_by('name')
+ repos = Repository.objects.select_related('arch').order_by('name')
if 'repotype' in params:
repos = repos.filter(repotype=params['repotype'][0])
@@ -489,7 +487,6 @@ def repo_bulk_action(request):
repos.update(security=False)
messages.success(request, f'Marked {count} {name} as non-security')
elif action == 'refresh':
- from repos.tasks import refresh_repo
for repo in repos:
refresh_repo.delay(repo.id)
messages.success(request, f'Queued {count} {name} for refresh')
@@ -507,10 +504,9 @@ def repo_bulk_action(request):
def _get_filtered_mirrors(filter_params):
"""Helper to reconstruct filtered queryset from filter params."""
- from urllib.parse import parse_qs
params = parse_qs(filter_params)
- mirrors = Mirror.objects.select_related().order_by('packages_checksum')
+ mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum')
if 'checksum' in params:
mirrors = mirrors.filter(packages_checksum=params['checksum'][0])
@@ -592,7 +588,7 @@ class RepositoryViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows repositories to be viewed or edited.
"""
- queryset = Repository.objects.all()
+ queryset = Repository.objects.select_related('arch').all()
serializer_class = RepositorySerializer
@@ -600,7 +596,7 @@ class MirrorViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows mirrors to be viewed or edited.
"""
- queryset = Mirror.objects.all()
+ queryset = Mirror.objects.select_related('repo').all()
serializer_class = MirrorSerializer
@@ -608,5 +604,5 @@ class MirrorPackageViewSet(viewsets.ModelViewSet):
"""
API endpoint that allows mirror packages to be viewed or edited.
"""
- queryset = MirrorPackage.objects.all()
+ queryset = MirrorPackage.objects.select_related('mirror', 'package').all()
serializer_class = MirrorPackageSerializer
diff --git a/sbin/patchman b/sbin/patchman
index 06bef981..0e164c98 100755
--- a/sbin/patchman
+++ b/sbin/patchman
@@ -23,7 +23,6 @@ import sys
from django import setup as django_setup
from django.core.exceptions import MultipleObjectsReturned
-from django.db.models import Count
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings')
from django.conf import settings # noqa
@@ -165,7 +164,7 @@ def host_updates_alt(host=None):
updated_hosts = []
hosts = get_hosts(host, 'Finding updates')
ts = get_datetime_now()
- for host in hosts:
+ for host in hosts.iterator():
info_message(text=str(host))
if host not in updated_hosts:
host.find_updates()
@@ -174,9 +173,8 @@ def host_updates_alt(host=None):
host.save()
# only include hosts with the same number of packages
- filtered_hosts = Host.objects.annotate(
- packages_count=Count('packages')).filter(
- packages_count=host.packages.count())
+ filtered_hosts = Host.objects.filter(
+ packages_count=host.packages_count)
# exclude hosts with the current timestamp
filtered_hosts = filtered_hosts.exclude(updated_at=ts)
@@ -185,7 +183,7 @@ def host_updates_alt(host=None):
updates = host.updates.all()
phosts = []
- for fhost in filtered_hosts:
+ for fhost in filtered_hosts.iterator():
frepos = set(fhost.repos.all())
rdiff = repos.difference(frepos)
@@ -354,7 +352,7 @@ def process_reports(host=None, force=False):
info_message(text=text)
- for report in reports:
+ for report in reports.iterator():
report.process(find_updates=False)
diff --git a/security/managers.py b/security/managers.py
index 4dfcffaf..f8d054a1 100644
--- a/security/managers.py
+++ b/security/managers.py
@@ -19,4 +19,4 @@
class CVEManager(models.Manager):
def get_queryset(self):
- return super().get_queryset().select_related()
+ return super().get_queryset()
diff --git a/security/tasks.py b/security/tasks.py
index 0cfbc2f1..93d48427 100644
--- a/security/tasks.py
+++ b/security/tasks.py
@@ -50,7 +50,7 @@ def update_cves():
if cache.add(lock_key, 'true', lock_expire):
try:
- for cve in CVE.objects.all():
+ for cve in CVE.objects.all().iterator():
update_cve.delay(cve.id)
finally:
cache.delete(lock_key)
@@ -87,7 +87,7 @@ def update_cwes():
if cache.add(lock_key, 'true', lock_expire):
try:
- for cwe in CWE.objects.all():
+ for cwe in CWE.objects.all().iterator():
update_cwe.delay(cwe.id)
finally:
cache.delete(lock_key)
diff --git a/security/utils.py b/security/utils.py
index 127f2c73..745cf1f8 100644
--- a/security/utils.py
+++ b/security/utils.py
@@ -42,7 +42,7 @@ def update_cves(cve_id=None, fetch_nist_data=False):
cve = CVE.objects.get(cve_id=cve_id)
cve.fetch_cve_data(fetch_nist_data, sleep_secs=0)
else:
- for cve in CVE.objects.all():
+ for cve in CVE.objects.all().iterator():
cve.fetch_cve_data(fetch_nist_data)
@@ -56,7 +56,7 @@ def update_cwes(cve_id=None):
cwes = cve.cwes.all()
else:
cwes = CWE.objects.all()
- for cwe in cwes:
+ for cwe in cwes.iterator():
cwe.fetch_cwe_data()
diff --git a/security/views.py b/security/views.py
index ae56a82b..7ae5d851 100644
--- a/security/views.py
+++ b/security/views.py
@@ -32,7 +32,7 @@
@login_required
def cwe_list(request):
- cwes = CWE.objects.select_related()
+ cwes = CWE.objects.all()
if 'search' in request.GET:
terms = request.GET['search'].lower()
@@ -65,7 +65,7 @@ def cwe_detail(request, cwe_id):
@login_required
def cve_list(request):
- cves = CVE.objects.select_related()
+ cves = CVE.objects.all()
if 'erratum_id' in request.GET:
cves = cves.filter(erratum=request.GET['erratum_id'])
@@ -117,7 +117,7 @@ def cve_detail(request, cve_id):
@login_required
def reference_list(request):
- refs = Reference.objects.select_related().order_by('ref_type')
+ refs = Reference.objects.all().order_by('ref_type')
if 'ref_type' in request.GET:
refs = refs.filter(ref_type=request.GET['ref_type']).distinct()
diff --git a/util/context_processors.py b/util/context_processors.py
index a9c46545..97bbcf2c 100644
--- a/util/context_processors.py
+++ b/util/context_processors.py
@@ -16,6 +16,7 @@
import subprocess
from datetime import timedelta
+from importlib.metadata import version as get_pkg_version
from pathlib import Path
from django.db.models import F
@@ -52,8 +53,7 @@ def _get_version():
"""Get version from package metadata or VERSION.txt."""
# Try importlib.metadata first (for installed packages)
try:
- from importlib.metadata import version
- return version('patchman')
+ return get_pkg_version('patchman')
except Exception:
pass
diff --git a/util/management/commands/revoke_api_key.py b/util/management/commands/revoke_api_key.py
index d22955e7..05985c48 100644
--- a/util/management/commands/revoke_api_key.py
+++ b/util/management/commands/revoke_api_key.py
@@ -42,7 +42,7 @@ def handle(self, *args, **options):
if not api_keys.exists():
api_keys = APIKey.objects.filter(name=key_input)
- if api_keys.count() == 0:
+ if not api_keys.exists():
raise CommandError(f'No API key found matching: {key_input}')
elif api_keys.count() > 1:
raise CommandError(f'Multiple keys match "{key_input}". Please be more specific.')
diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html
index 5976adcb..7afc18fb 100644
--- a/util/templates/dashboard.html
+++ b/util/templates/dashboard.html
@@ -13,7 +13,7 @@
{% block content %}
{% with count=noosrelease_osvariants.count %}
- {% if noosrelease_osvariants.count > 0 %}
+ {% if count > 0 %}
@@ -24,7 +24,7 @@
{% endwith %}
{% with count=nohost_osvariants.count %}
- {% if nohost_osvariants.count > 0 %}
+ {% if count > 0 %}
@@ -244,4 +244,8 @@
{% endif %}
{% endwith %}
+{% if not has_issues %}
+ No issues found!
+{% endif %}
+
{% endblock %}
diff --git a/util/templatetags/common.py b/util/templatetags/common.py
index 0fdeaff1..dc9f4fb5 100644
--- a/util/templatetags/common.py
+++ b/util/templatetags/common.py
@@ -18,7 +18,8 @@
from datetime import timedelta
from urllib.parse import urlencode
-from django.template import Library
+from django.db.models import Sum
+from django.template import Library, engines
from django.template.loader import get_template
from django.utils import timezone
from django.utils.html import format_html
@@ -73,7 +74,6 @@ def gen_table(context, object_list, template_name=None):
RequestConfig(request, paginate=False).configure(table)
# Render using the table's configured template
- from django.template import engines
django_engine = engines['django']
template = django_engine.from_string('{% load django_tables2 %}{% render_table table %}')
return template.render({'table': table, 'request': request})
@@ -138,7 +138,4 @@ def reports_timedelta():
@register.simple_tag
def host_count(osrelease):
- host_count = 0
- for osvariant in osrelease.osvariant_set.all():
- host_count += osvariant.host_set.count()
- return host_count
+ return osrelease.osvariant_set.aggregate(total=Sum('hosts_count'))['total'] or 0
diff --git a/util/views.py b/util/views.py
index 6d88f28d..6222546b 100644
--- a/util/views.py
+++ b/util/views.py
@@ -55,8 +55,9 @@ def dashboard(request):
stale_hosts = hosts.filter(lastreport__lt=last_report_delta)
norepo_hosts = hosts.filter(repos__isnull=True, osvariant__osrelease__repos__isnull=True) # noqa
reboot_hosts = hosts.filter(reboot_required=True)
- secupdate_hosts = hosts.filter(updates__security=True, updates__isnull=False).distinct() # noqa
- bugupdate_hosts = hosts.exclude(updates__security=True, updates__isnull=False).distinct().filter(updates__security=False, updates__isnull=False).distinct() # noqa
+ # Use cached count fields instead of expensive M2M JOINs
+ secupdate_hosts = hosts.filter(sec_updates_count__gt=0)
+ bugupdate_hosts = hosts.filter(bug_updates_count__gt=0, sec_updates_count=0)
diff_rdns_hosts = hosts.exclude(reversedns=F('hostname')).filter(check_dns=True) # noqa
# os variant issues
@@ -89,14 +90,16 @@ def dashboard(request):
checksums = {}
possible_mirrors = {}
- for csvalue in Mirror.objects.all().values('packages_checksum').distinct():
+ # Use cached packages_count to avoid N+1 queries
+ for csvalue in Mirror.objects.filter(packages_count__gt=0).values('packages_checksum').distinct():
checksum = csvalue['packages_checksum']
if checksum is not None and checksum != 'yast':
- for mirror in Mirror.objects.filter(packages_checksum=checksum):
- if mirror.packages.count() > 0:
- if checksum not in checksums:
- checksums[checksum] = []
- checksums[checksum].append(mirror)
+ mirrors = list(Mirror.objects.filter(
+ packages_checksum=checksum,
+ packages_count__gt=0
+ ).select_related('repo'))
+ if mirrors:
+ checksums[checksum] = mirrors
for checksum in checksums:
first_mirror = checksums[checksum][0]
@@ -107,10 +110,32 @@ def dashboard(request):
possible_mirrors[checksum] = checksums[checksum]
continue
+ has_issues = (
+ noosrelease_osvariants.exists() or
+ nohost_osvariants.exists() or
+ (norepo_osreleases is not None and norepo_osreleases.exists()) or
+ stale_hosts.exists() or
+ reboot_hosts.exists() or
+ secupdate_hosts.exists() or
+ bugupdate_hosts.exists() or
+ norepo_hosts.exists() or
+ diff_rdns_hosts.exists() or
+ failed_mirrors.exists() or
+ disabled_mirrors.exists() or
+ norefresh_mirrors.exists() or
+ failed_repos.exists() or
+ unused_repos.exists() or
+ nomirror_repos.exists() or
+ nohost_repos.exists() or
+ bool(possible_mirrors) or
+ norepo_packages.exists()
+ )
+
return render(
request,
'dashboard.html',
{'site': site,
+ 'has_issues': has_issues,
'noosrelease_osvariants': noosrelease_osvariants,
'norepo_hosts': norepo_hosts,
'nohost_osvariants': nohost_osvariants,
|